diff --git a/packages/api/package.json b/packages/api/package.json index 5059108e6c..1f3504980a 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -55,8 +55,9 @@ "@tus/server": "^1.0.0", "@types/amqp-connection-manager": "2.0.10", "@types/cors": "^2.8.12", - "ajv": "^6.10.0", + "ajv": "^8.16.0", "ajv-cli": "^3.1.0", + "ajv-formats": "^3.0.1", "amqp-connection-manager": "^4.1.6", "amqplib": "^0.8.0", "analytics-node": "^3.4.0-beta.1", @@ -78,6 +79,7 @@ "express-prom-bundle": "^6.4.1", "fakefilter": "^0.1.880", "fast-stable-stringify": "^1.0.0", + "form-data": "^4.0.0", "fs-extra": "^7.0.1", "google-auth-library": "^5.2.2", "googleapis": "^43.0.0", @@ -96,6 +98,7 @@ "morgan": "^1.9.1", "mqtt": "^4.2.6", "ms": "^2.1.2", + "multer": "^1.4.5-lts.1", "mustache": "^3.0.3", "node-fetch": "^2.6.1", "node-jose": "^1.1.4", @@ -126,11 +129,11 @@ "@types/jsonwebtoken": "^8.5.6", "@types/lodash": "^4.14.191", "@types/ms": "^0.7.31", + "@types/multer": "^1.4.11", "@types/mustache": "^4.1.1", "@types/node-fetch": "^2.5.10", "@types/pg": "^7.14.4", "@types/uuid": "^9.0.0", - "ajv-pack": "^0.3.1", "cloudflare": "^2.7.0", "esbuild": "^0.18.17", "esm": "^3.2.22", diff --git a/packages/api/src/compile-schemas.js b/packages/api/src/compile-schemas.js index bd02d1078d..4b09fd9b6c 100644 --- a/packages/api/src/compile-schemas.js +++ b/packages/api/src/compile-schemas.js @@ -1,11 +1,12 @@ import Ajv from "ajv"; -import pack from "ajv-pack"; -import { safeLoad as parseYaml, safeDump as serializeYaml } from "js-yaml"; +import ajvFormats from "ajv-formats"; +import standaloneCode from "ajv/dist/standalone"; import fs from "fs-extra"; +import { safeLoad as parseYaml, safeDump as serializeYaml } from "js-yaml"; +import $RefParser from "json-schema-ref-parser"; +import { compile as generateTypes } from "json-schema-to-typescript"; import _ from "lodash"; import path from "path"; -import { compile as generateTypes } from "json-schema-to-typescript"; -import $RefParser from "json-schema-ref-parser"; // This takes schema.yaml as its input and produces a few outputs. // 1. types.d.ts, TypeScript definitions of the JSON-schema objects @@ -52,7 +53,14 @@ const data = _.merge({}, apiData, dbData); write(path.resolve(schemaDir, "schema.json"), str); write(path.resolve(schemaDistDir, "schema.json"), str); - const ajv = new Ajv({ sourceCode: true }); + let ajv = new Ajv({ + keywords: [ + ...["example", "minValue"], // OpenAPI keywords not supported by ajv + ...["table", "index", "indexType", "unique"], // our custom keywords + ], + code: { source: true }, + }); + ajv = ajvFormats(ajv, ["binary", "uri"]); const index = []; let types = []; @@ -62,7 +70,7 @@ const data = _.merge({}, apiData, dbData); const type = await generateTypes(schema); types.push(type); var validate = ajv.compile(schema); - var moduleCode = pack(ajv, validate); + var moduleCode = standaloneCode(ajv, validate); const outPath = path.resolve(validatorDir, `${name}.js`); write(outPath, moduleCode); index.push(`'${name}': require('./${name}.js'),`); diff --git a/packages/api/src/controllers/asset.test.ts b/packages/api/src/controllers/asset.test.ts index 8b20658d58..26dc356470 100644 --- a/packages/api/src/controllers/asset.test.ts +++ b/packages/api/src/controllers/asset.test.ts @@ -718,7 +718,7 @@ describe("controllers/asset", () => { await testStoragePatch( { ipfs: { nftMetadata: { a: "b" } } } as any, null, - [expect.stringContaining("should NOT have additional properties")], + [expect.stringContaining("must NOT have additional properties")], ); await testStoragePatch( { ipfs: {} }, diff --git a/packages/api/src/controllers/generate.test.ts b/packages/api/src/controllers/generate.test.ts new file mode 100644 index 0000000000..8db0c8c847 --- /dev/null +++ b/packages/api/src/controllers/generate.test.ts @@ -0,0 +1,193 @@ +import FormData from "form-data"; +import { User } from "../schema/types"; +import { + AuxTestServer, + TestClient, + clearDatabase, + setupUsers, + startAuxTestServer, +} from "../test-helpers"; +import serverPromise, { TestServer } from "../test-server"; + +let server: TestServer; +let mockAdminUserInput: User; +let mockNonAdminUserInput: User; + +// jest.setTimeout(70000) + +beforeAll(async () => { + server = await serverPromise; + + mockAdminUserInput = { + email: "user_admin@gmail.com", + password: "x".repeat(64), + }; + + mockNonAdminUserInput = { + email: "user_non_admin@gmail.com", + password: "y".repeat(64), + }; +}); + +afterEach(async () => { + await clearDatabase(server); +}); + +describe("controllers/generate", () => { + let client: TestClient; + let adminUser: User; + let adminApiKey: string; + let nonAdminUser: User; + let nonAdminToken: string; + + let aiGatewayServer: AuxTestServer; + let aiGatewayCalls: Record; + + beforeAll(async () => { + aiGatewayServer = await startAuxTestServer(30303); // port configured in test-params.ts + const apis = [ + "text-to-image", + "image-to-image", + "image-to-video", + "upscale", + ]; + for (const api of apis) { + aiGatewayServer.app.post(`/${api}`, (req, res) => { + aiGatewayCalls[api] = (aiGatewayCalls[api] || 0) + 1; + return res.status(200).json({ + message: "success", + reqContentType: req.headers["content-type"] ?? "unknown", + }); + }); + } + }); + + afterAll(async () => { + await aiGatewayServer.close(); + }); + + beforeEach(async () => { + ({ client, adminUser, adminApiKey, nonAdminUser, nonAdminToken } = + await setupUsers(server, mockAdminUserInput, mockNonAdminUserInput)); + + client.apiKey = adminApiKey; + await client.post("/experiment", { + name: "ai-generate", + audienceUserIds: [adminUser.id, nonAdminUser.id], + }); + client.apiKey = null; + client.jwtAuth = nonAdminToken; + + aiGatewayCalls = {}; + }); + + const buildMultipartBody = (textFields: Record) => { + const form = new FormData(); + for (const [k, v] of Object.entries(textFields)) { + form.append(k, v); + } + form.append("image", "dummy", { + contentType: "image/png", + }); + return form; + }; + + describe("API proxies", () => { + it("should call the AI Gateway for generate API /text-to-image", async () => { + const res = await client.post("/beta/generate/text-to-image", { + prompt: "a man in a suit and tie", + }); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ + message: "success", + reqContentType: "application/json", + }); + expect(aiGatewayCalls).toEqual({ "text-to-image": 1 }); + }); + + it("should call the AI Gateway for generate API /image-to-image", async () => { + const res = await client.fetch("/beta/generate/image-to-image", { + method: "POST", + body: buildMultipartBody({ + prompt: "replace the suit with a bathing suit", + }), + }); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ + message: "success", + reqContentType: expect.stringMatching("^multipart/form-data"), + }); + expect(aiGatewayCalls).toEqual({ "image-to-image": 1 }); + }); + + it("should call the AI Gateway for generate API /image-to-video", async () => { + const res = await client.fetch("/beta/generate/image-to-video", { + method: "POST", + body: buildMultipartBody({}), + }); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ + message: "success", + reqContentType: expect.stringMatching("^multipart/form-data"), + }); + expect(aiGatewayCalls).toEqual({ "image-to-video": 1 }); + }); + + it("should call the AI Gateway for generate API /upscale", async () => { + const res = await client.fetch("/beta/generate/upscale", { + method: "POST", + body: buildMultipartBody({ prompt: "enhance" }), + }); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ + message: "success", + reqContentType: expect.stringMatching("^multipart/form-data"), + }); + expect(aiGatewayCalls).toEqual({ upscale: 1 }); + }); + }); + + describe("validates multipart schema", () => { + const hugeForm = new FormData(); + const file11mb = "a".repeat(11 * 1024 * 1024); + hugeForm.append("image", file11mb, { + contentType: "image/png", + }); + + const testCases = [ + [ + "should fail with a missing required field", + buildMultipartBody({}), + "must have required property 'prompt'", + ], + [ + "should fail with bad type for a field", + buildMultipartBody({ prompt: "impromptu", seed: "NaN" }), + "must be integer", + ], + [ + "should fail with an unknown field", + buildMultipartBody({ + prompt: "impromptu", + extra_good_image: "yes pls", + }), + "must NOT have additional properties", + ], + ["should limit maximum payload size", hugeForm, "Field value too long"], + ] as const; + + for (const [title, input, error] of testCases) { + it(title, async () => { + const res = await client.fetch("/beta/generate/image-to-image", { + method: "POST", + body: input, + }); + expect(res.status).toBe(422); + expect(await res.json()).toEqual({ + errors: [expect.stringContaining(error)], + }); + expect(aiGatewayCalls).toEqual({}); + }); + } + }); +}); diff --git a/packages/api/src/controllers/generate.ts b/packages/api/src/controllers/generate.ts new file mode 100644 index 0000000000..c46e3fc975 --- /dev/null +++ b/packages/api/src/controllers/generate.ts @@ -0,0 +1,110 @@ +import { RequestHandler, Router } from "express"; +import FormData from "form-data"; +import multer from "multer"; +import { BodyInit } from "node-fetch"; +import logger from "../logger"; +import { authorizer, validateFormData, validatePost } from "../middleware"; +import { fetchWithTimeout } from "../util"; +import { experimentSubjectsOnly } from "./experiment"; +import { pathJoin2 } from "./helpers"; + +const AI_GATEWAY_TIMEOUT = 10 * 60 * 1000; // 10 minutes + +const multipart = multer({ + storage: multer.memoryStorage(), + limits: { fileSize: 10485760 }, // 10MiB +}); + +const app = Router(); + +app.use(experimentSubjectsOnly("ai-generate")); + +function registerGenerateHandler( + name: string, + defaultModel: string, + isJSONReq = false, +): RequestHandler { + const path = `/${name}`; + const middlewares = isJSONReq + ? [validatePost(`${name}-payload`)] + : [multipart.any(), validateFormData(`${name}-payload`)]; + return app.post( + path, + authorizer({}), + ...middlewares, + async function proxyGenerate(req, res) { + const { aiGatewayUrl } = req.config; + if (!aiGatewayUrl) { + res.status(500).json({ errors: ["AI Gateway URL is not set"] }); + return; + } + + const apiUrl = pathJoin2(aiGatewayUrl, path); + + let payload: BodyInit; + if (isJSONReq) { + payload = JSON.stringify({ + model_id: defaultModel, + ...req.body, + }); + } else { + const form = new FormData(); + if (!("model_id" in req.body)) { + form.append("model_id", defaultModel); + } + for (const [key, value] of Object.entries(req.body)) { + form.append(key, value); + } + + if (!Array.isArray(req.files)) { + return res.status(400).json({ + errors: ["Expected an array of files"], + }); + } + for (const file of req.files) { + form.append(file.fieldname, file.buffer, { + filename: file.originalname, + contentType: file.mimetype, + knownLength: file.size, + }); + } + payload = form; + } + + const response = await fetchWithTimeout(apiUrl, { + method: "POST", + body: payload, + timeout: AI_GATEWAY_TIMEOUT, + headers: isJSONReq ? { "content-type": "application/json" } : {}, + }); + + const body = await response.json(); + if (!response.ok) { + logger.error( + `Error from generate API ${path} status=${ + response.status + } body=${JSON.stringify(body)}`, + ); + } + if (response.status >= 500) { + return res.status(500).json({ errors: [`Failed to generate ${name}`] }); + } + + res.status(response.status).json(body); + }, + ); +} + +registerGenerateHandler( + "text-to-image", + "SG161222/RealVisXL_V4.0_Lightning", + true, +); +registerGenerateHandler("image-to-image", "timbrooks/instruct-pix2pix"); +registerGenerateHandler( + "image-to-video", + "stabilityai/stable-video-diffusion-img2vid-xt-1-1", +); +registerGenerateHandler("upscale", "stabilityai/stable-diffusion-x4-upscaler"); + +export default app; diff --git a/packages/api/src/controllers/index.ts b/packages/api/src/controllers/index.ts index b791f5eac6..f9863cd7f1 100644 --- a/packages/api/src/controllers/index.ts +++ b/packages/api/src/controllers/index.ts @@ -1,58 +1,60 @@ +import accessControl from "./access-control"; +import admin from "./admin"; import apiToken from "./api-token"; +import asset from "./asset"; import auth from "./auth"; import broadcaster from "./broadcaster"; +import clip from "./clip"; +import did from "./did"; import experiment from "./experiment"; +import generate from "./generate"; +import geolocate from "./geolocate"; import ingest from "./ingest"; -import objectStore from "./object-store"; -import accessControl from "./access-control"; -import clip from "./clip"; import multistream from "./multistream"; +import objectStore from "./object-store"; import orchestrator from "./orchestrator"; +import playback from "./playback"; +import project from "./project"; +import region from "./region"; +import room from "./room"; +import session from "./session"; import stream from "./stream"; -import user from "./user"; -import geolocate from "./geolocate"; -import webhook from "./webhook"; -import asset from "./asset"; +import stripe from "./stripe"; import task from "./task"; import transcode from "./transcode"; -import stripe from "./stripe"; -import version from "./version"; -import admin from "./admin"; import usage from "./usage"; -import region from "./region"; -import session from "./session"; -import playback from "./playback"; -import did from "./did"; -import room from "./room"; -import project from "./project"; +import user from "./user"; +import version from "./version"; +import webhook from "./webhook"; // Annoying but necessary to get the routing correct export default { + "access-control": accessControl, + admin, "api-token": apiToken, + asset, auth, + "beta/generate": generate, broadcaster, + clip, + did, experiment, - "object-store": objectStore, + geolocate, + ingest, multistream, + "object-store": objectStore, orchestrator, + playback, + project, + region, + room, + session, stream, - user, - geolocate, - ingest, - webhook, - asset, + stripe, task, transcode, - "access-control": accessControl, - region, - stripe, - version, - admin, usage, - session, - playback, - did, - room, - clip, - project, + user, + version, + webhook, }; diff --git a/packages/api/src/controllers/multistream.test.ts b/packages/api/src/controllers/multistream.test.ts index c44206d2af..6130114a93 100644 --- a/packages/api/src/controllers/multistream.test.ts +++ b/packages/api/src/controllers/multistream.test.ts @@ -1,8 +1,8 @@ -import serverPromise, { TestServer } from "../test-server"; -import { TestClient, clearDatabase, setupUsers } from "../test-helpers"; import { v4 as uuid } from "uuid"; import { MultistreamTarget, User } from "../schema/types"; import { db } from "../store"; +import { TestClient, clearDatabase, setupUsers } from "../test-helpers"; +import serverPromise, { TestServer } from "../test-server"; // includes auth file tests @@ -286,7 +286,7 @@ describe("controllers/multistream-target", () => { }); expect(res.status).toBe(422); const body = await res.json(); - expect(body.errors[0]).toContain("Bad URL"); + expect(body.errors[0]).toContain(`must match format \\"uri\\"`); }); }); diff --git a/packages/api/src/middleware/errorHandler.ts b/packages/api/src/middleware/errorHandler.ts index b7e9e9702b..9be41dd17e 100644 --- a/packages/api/src/middleware/errorHandler.ts +++ b/packages/api/src/middleware/errorHandler.ts @@ -1,16 +1,25 @@ import { ErrorRequestHandler } from "express"; +import multer from "multer"; +import logger from "../logger"; import { isAPIError } from "../store/errors"; export default function errorHandler(): ErrorRequestHandler { return (err, _req, res, _next) => { // If we throw any errors with numerical statuses, use them. if (isAPIError(err)) { - res.status(err.status); - return res.json({ errors: [err.message] }); + return res.status(err.status).json({ errors: [err.message] }); } - res.status(500); - console.error(err); - return res.json({ errors: [err.stack] }); + // multipart form errors are always bad input-related + if (err instanceof multer.MulterError) { + return res.status(422).json({ errors: [err.message] }); + } + + logger.error( + `Unhandled error in API path=${_req.path} errType=${err.name} err="${err.message}" stack=${err.stack}`, + ); + return res + .status(500) + .json({ errors: [`Internal server error: ${err.name || err.message}`] }); }; } diff --git a/packages/api/src/middleware/validators.js b/packages/api/src/middleware/validators.js deleted file mode 100644 index bc8bee25a4..0000000000 --- a/packages/api/src/middleware/validators.js +++ /dev/null @@ -1,97 +0,0 @@ -// import { schemaWalk } from '@cloudflare/json-schema-walker' -import validators from "../schema/validators"; - -export const validatePost = (name) => { - const validate = validators[name]; - if (!validate) { - throw new Error(`no validator found for ${name}`); - } - - return (req, res, next) => { - const { body } = req; - if (!validate(body)) { - res.status(422); - return res.json({ - errors: validate.errors.map((err) => JSON.stringify(err)), - }); - } - next(); - }; - - // Enforcement of required read-only properties, currently unused - - // Generate a new version of the schema with all readOnly properties not required - // const postSchema = JSON.parse(JSON.stringify(schema.components.schemas[name])) - // schemaWalk(postSchema, node => { - // if (node.type !== 'object') { - // return - // } - // const oldRequired = node.required - // if (!oldRequired) { - // return - // } - // node.required = oldRequired.filter(key => { - // if (node.properties && node.properties[key].readOnly === true) { - // delete node.properties[key] - // return false - // } - // return true - // }) - // }) -}; - -// Unused, should be rethought. - -// export const validatePut = name => { -// // Set up AJV validator that stores data on first validation then validates -// // that it matches on the second validation. -// const ajv = new Ajv() -// let cache = {} -// let first = true -// ajv.removeKeyword('readOnly') -// ajv.addKeyword('readOnly', { -// validate: function(schema, data, parentSchema, path) { -// if (first) { -// cache[path] = data -// } else { -// if (data !== cache[path]) { -// return false -// } -// } -// return true -// }, -// }) -// const validate = ajv.compile(schema.components.schemas[name]) -// return async (req, res, next) => { -// const data = req.body - -// if (data.id !== `${name}/${req.params.id}`) { -// res.status(409) -// return res.json({ errors: ['id in URL and body must match'] }) -// } - -// // Cache the read-only values in the old data -// const oldData = await req.store.get(data.id) - -// first = true -// cache = {} -// if (!validate(oldData)) { -// throw new Error( -// `Invalid code path, ${ -// data.id -// } in the DB fails validation: ${JSON.stringify(validate.errors)}`, -// ) -// } - -// first = false - -// if (!validate(data)) { -// res.status(422) -// return res.json({ -// errors: validate.errors.map(err => JSON.stringify(err)), -// }) -// } - -// next() -// } -// } diff --git a/packages/api/src/middleware/validators.ts b/packages/api/src/middleware/validators.ts new file mode 100644 index 0000000000..3bc6c9d91a --- /dev/null +++ b/packages/api/src/middleware/validators.ts @@ -0,0 +1,62 @@ +import { RequestHandler } from "express"; +import validators from "../schema/validators"; + +export const validatePost = (name: string): RequestHandler => { + const validate = validators[name]; + if (!validate) { + throw new Error(`no validator found for ${name}`); + } + + return (req, res, next) => { + const { body } = req; + if (!validate(body)) { + res.status(422); + return res.json({ + errors: validate.errors.map((err) => JSON.stringify(err)), + }); + } + next(); + }; +}; + +export const validateFormData = (name: string): RequestHandler => { + const validate = validators[name]; + if (!validate) { + throw new Error(`no validator found for ${name}`); + } + + return (req, res, next) => { + if (!req.is("multipart/form-data")) { + return res.status(422).json({ + errors: ["Expected multipart/form-data content-type"], + }); + } + + const { body, files } = req; + const allFields = {}; + for (const [key, value] of Object.entries(body)) { + const fval = Number.parseFloat(value as string); + if (!Number.isNaN(fval)) { + allFields[key] = fval; + } else if (value === "true" || value === "false") { + allFields[key] = value === "true"; + } else { + allFields[key] = value; + } + } + if (Array.isArray(files)) { + for (const file of files) { + // we don't need to validate the file contents, just that it is there + allFields[file.fieldname] = "dummy"; + } + } + + if (!validate(allFields)) { + res.status(422); + return res.json({ + errors: validate.errors.map((err) => JSON.stringify(err)), + }); + } + next(); + }; +}; diff --git a/packages/api/src/parse-cli.ts b/packages/api/src/parse-cli.ts index c3699a6535..e751df24a6 100755 --- a/packages/api/src/parse-cli.ts +++ b/packages/api/src/parse-cli.ts @@ -87,7 +87,7 @@ function coerceJsonValue(flagName: string) { function coerceJsonProfileArr(flagName: string) { return (str: string): FfmpegProfile[] => { let profiles; - const validator = profileValidator as ValidateFunction; + const validator = profileValidator as any as ValidateFunction; try { profiles = JSON.parse(str); } catch (e) { @@ -216,6 +216,11 @@ export default function parseCli(argv?: string | readonly string[]) { type: "string", default: "https://{{ip}}:8935", }, + "ai-gateway-url": { + describe: + "base URL of the AI Gateway to call for generative AI requests", + type: "string", + }, "ipfs-gateway-url": { describe: "base URL to use for the IPFS content gateway returned on assets.", diff --git a/packages/api/src/schema/api-schema.yaml b/packages/api/src/schema/api-schema.yaml index 6b2ee80cac..cfdf7220f1 100644 --- a/packages/api/src/schema/api-schema.yaml +++ b/packages/api/src/schema/api-schema.yaml @@ -63,7 +63,7 @@ components: height: type: integer minimum: 128 - expample: 720 + example: 720 bitrate: type: integer minimum: 400 @@ -120,7 +120,7 @@ components: height: type: integer minimum: 128 - expample: 720 + example: 720 bitrate: type: integer minimum: 400 @@ -1142,6 +1142,7 @@ components: items: $ref: "#/components/schemas/transcode-profile" storage: + type: object additionalProperties: false properties: ipfs: @@ -1384,6 +1385,7 @@ components: creatorId: $ref: "#/components/schemas/input-creator-id" storage: + type: object additionalProperties: false properties: ipfs: @@ -2854,7 +2856,141 @@ components: targetSegmentSizeSecs: $ref: >- #/components/schemas/new-asset-payload/properties/targetSegmentSizeSecs - + # AI Generate payloads. Keep in mind that these use snake_case instead of camelCase since + # they implement the same interface as the AI Gateway Livepeer node. + text-to-image-payload: + type: object + required: + - prompt + additionalProperties: false + properties: + prompt: + type: string + model_id: + type: string + default: SG161222/RealVisXL_V4.0_Lightning + enum: + - SG161222/RealVisXL_V4.0_Lightning + - ByteDance/SDXL-Lightning + height: + type: integer + width: + type: integer + guidance_scale: + type: number + default: 7.5 + negative_prompt: + type: string + default: "" + safety_check: + type: boolean + default: true + seed: + type: integer + num_inference_steps: + type: integer + default: 50 + num_images_per_prompt: + type: integer + default: 1 + image-to-image-payload: + type: object + required: + - prompt + - image + additionalProperties: false + properties: + prompt: + type: string + image: + type: string + format: binary + maxLength: 10485760 # 10MiB + model_id: + type: string + default: timbrooks/instruct-pix2pix + enum: + - timbrooks/instruct-pix2pix + - ByteDance/SDXL-Lightning + - SG161222/RealVisXL_V4.0_Lightning + strength: + type: number + default: 0.8 + guidance_scale: + type: number + default: 7.5 + image_guidance_scale: + type: number + default: 1.5 + negative_prompt: + type: string + default: "" + safety_check: + type: boolean + default: true + seed: + type: integer + num_images_per_prompt: + type: integer + default: 1 + image-to-video-payload: + type: object + required: + - image + additionalProperties: false + properties: + image: + type: string + format: binary + maxLength: 10485760 # 10MiB + model_id: + type: string + default: stabilityai/stable-video-diffusion-img2vid-xt-1-1 + enum: + - stabilityai/stable-video-diffusion-img2vid-xt-1-1 + height: + type: integer + default: 576 + width: + type: integer + default: 1024 + fps: + type: integer + default: 6 + motion_bucket_id: + type: integer + default: 127 + noise_aug_strength: + type: number + default: 0.02 + seed: + type: integer + safety_check: + type: boolean + default: true + upscale-payload: + type: object + required: + - prompt + - image + additionalProperties: false + properties: + prompt: + type: string + image: + type: string + format: binary + maxLength: 10485760 # 10MiB + model_id: + type: string + default: stabilityai/stable-diffusion-x4-upscaler + enum: + - stabilityai/stable-diffusion-x4-upscaler + safety_check: + type: boolean + default: true + seed: + type: integer paths: /stream: post: diff --git a/packages/api/src/schema/db-schema.yaml b/packages/api/src/schema/db-schema.yaml index 5e2ec388f0..833fef7064 100644 --- a/packages/api/src/schema/db-schema.yaml +++ b/packages/api/src/schema/db-schema.yaml @@ -527,6 +527,7 @@ components: type: string description: Refresh token to be used to generate a new JWT transcode-asset-payload: + type: object additionalProperties: false required: - name diff --git a/packages/api/src/test-params.ts b/packages/api/src/test-params.ts index 46333c022a..5279e5ecfd 100644 --- a/packages/api/src/test-params.ts +++ b/packages/api/src/test-params.ts @@ -38,6 +38,7 @@ params.trustedIpfsGateways = [ "https://ipfs.example.com/ipfs/", /https:\/\/.+\.ipfs-provider.io\/ipfs\//, ]; +params.aiGatewayUrl = "http://localhost:30303/"; params.ingest = [ { ingest: "rtmp://test/live", diff --git a/yarn.lock b/yarn.lock index 3c6e593864..26a0311f9c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7778,6 +7778,13 @@ resolved "https://registry.npmjs.org/@types/ms/-/ms-0.7.31.tgz" integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA== +"@types/multer@^1.4.11": + version "1.4.11" + resolved "https://registry.yarnpkg.com/@types/multer/-/multer-1.4.11.tgz#c70792670513b4af1159a2b60bf48cc932af55c5" + integrity sha512-svK240gr6LVWvv3YGyhLlA+6LRRWA4mnGIU7RcNmgjBYFl6665wcXrRfxGp5tEPVHUNm5FMcmq7too9bxCwX/w== + dependencies: + "@types/express" "*" + "@types/mustache@^4.1.1": version "4.2.2" resolved "https://registry.npmjs.org/@types/mustache/-/mustache-4.2.2.tgz" @@ -8463,12 +8470,19 @@ ajv-cli@^3.1.0: json5 "^2.1.3" minimist "^1.2.0" +ajv-formats@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/ajv-formats/-/ajv-formats-3.0.1.tgz#3d5dc762bca17679c3c2ea7e90ad6b7532309578" + integrity sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ== + dependencies: + ajv "^8.0.0" + ajv-keywords@^3.5.2: version "3.5.2" resolved "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz" integrity sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ== -ajv-pack@^0.3.0, ajv-pack@^0.3.1: +ajv-pack@^0.3.0: version "0.3.1" resolved "https://registry.npmjs.org/ajv-pack/-/ajv-pack-0.3.1.tgz" integrity sha512-psFkqg+ItqBXjQ0kbdP/Y72Jmz+wHt8MD7bVGdzdxjKsp988QTK5YMQoBsPUotbhnYO8VKPU3vPALYlhO/2gtg== @@ -8486,7 +8500,7 @@ ajv@^5.0.0, ajv@^5.5.2: fast-json-stable-stringify "^2.0.0" json-schema-traverse "^0.3.0" -ajv@^6.10.0, ajv@^6.12.3, ajv@^6.12.5, ajv@^6.7.0: +ajv@^6.12.3, ajv@^6.12.5, ajv@^6.7.0: version "6.12.6" resolved "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz" integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== @@ -8496,6 +8510,16 @@ ajv@^6.10.0, ajv@^6.12.3, ajv@^6.12.5, ajv@^6.7.0: json-schema-traverse "^0.4.1" uri-js "^4.2.2" +ajv@^8.0.0, ajv@^8.16.0: + version "8.16.0" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.16.0.tgz#22e2a92b94f005f7e0f9c9d39652ef0b8f6f0cb4" + integrity sha512-F0twR8U1ZU67JIEtekUcLkXkoO5mMMmgGD8sK/xUFzJ805jxHQl92hImFAqqXMyMYjSPOyUPAwHYhB72g5sTXw== + dependencies: + fast-deep-equal "^3.1.3" + json-schema-traverse "^1.0.0" + require-from-string "^2.0.2" + uri-js "^4.4.1" + ajv@^8.11.0: version "8.12.0" resolved "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz" @@ -8690,6 +8714,11 @@ apg-js@^4.1.1: resolved "https://registry.npmjs.org/apg-js/-/apg-js-4.1.2.tgz" integrity sha512-2OALKUe82NLVPe4NTooom8NykWIa2D7YxO7jG1pgnYWnkfhTUriXpITmLvVD8k8TzDfa9G5O4y8rPe2/uUB1Bg== +append-field@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/append-field/-/append-field-1.0.0.tgz#1e3440e915f0b1203d23748e78edd7b9b5b43e56" + integrity sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw== + append-transform@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/append-transform/-/append-transform-1.0.0.tgz" @@ -9780,7 +9809,7 @@ builtins@^5.0.0: dependencies: semver "^7.0.0" -busboy@1.6.0: +busboy@1.6.0, busboy@^1.0.0: version "1.6.0" resolved "https://registry.yarnpkg.com/busboy/-/busboy-1.6.0.tgz#966ea36a9502e43cdb9146962523b92f531f6893" integrity sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA== @@ -10613,24 +10642,24 @@ concat-map@0.0.1: resolved "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== -concat-stream@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz" - integrity sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A== +concat-stream@^1.5.2, concat-stream@~1.6.0: + version "1.6.2" + resolved "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz" + integrity sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw== dependencies: buffer-from "^1.0.0" inherits "^2.0.3" - readable-stream "^3.0.2" + readable-stream "^2.2.2" typedarray "^0.0.6" -concat-stream@~1.6.0: - version "1.6.2" - resolved "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz" - integrity sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw== +concat-stream@^2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz" + integrity sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A== dependencies: buffer-from "^1.0.0" inherits "^2.0.3" - readable-stream "^2.2.2" + readable-stream "^3.0.2" typedarray "^0.0.6" config-chain@^1.1.13: @@ -12878,7 +12907,7 @@ fast-deep-equal@^2.0.1: resolved "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz" integrity sha512-bCK/2Z4zLidyB4ReuIsvALH6w31YfAQDmXMqMx6FyfHqvBxtjC0eRumeSu4Bs3XtXwpyIywtSTrVT99BxY1f9w== -fast-deep-equal@^3.1.1: +fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: version "3.1.3" resolved "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz" integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== @@ -17843,7 +17872,7 @@ mixin-deep@^1.2.0: for-in "^1.0.2" is-extendable "^1.0.1" -mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@^0.5.5, mkdirp@~0.5.1: +mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@^0.5.4, mkdirp@^0.5.5, mkdirp@~0.5.1: version "0.5.6" resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz" integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw== @@ -17955,6 +17984,19 @@ msrcrypto@^1.5.6: resolved "https://registry.npmjs.org/msrcrypto/-/msrcrypto-1.5.8.tgz" integrity sha512-ujZ0TRuozHKKm6eGbKHfXef7f+esIhEckmThVnz7RNyiOJd7a6MXj2JGBoL9cnPDW+JMG16MoTUh5X+XXjI66Q== +multer@^1.4.5-lts.1: + version "1.4.5-lts.1" + resolved "https://registry.yarnpkg.com/multer/-/multer-1.4.5-lts.1.tgz#803e24ad1984f58edffbc79f56e305aec5cfd1ac" + integrity sha512-ywPWvcDMeH+z9gQq5qYHCCy+ethsk4goepZ45GLD63fOu0YcNecQxi64nDs3qluZB+murG3/D4dJ7+dGctcCQQ== + dependencies: + append-field "^1.0.0" + busboy "^1.0.0" + concat-stream "^1.5.2" + mkdirp "^0.5.4" + object-assign "^4.1.1" + type-is "^1.6.4" + xtend "^4.0.0" + multiformats@9.9.0: version "9.9.0" resolved "https://registry.npmjs.org/multiformats/-/multiformats-9.9.0.tgz" @@ -22183,11 +22225,6 @@ socks@^2.7.1: ip-address "^9.0.5" smart-buffer "^4.2.0" -sonner@^1.5.0: - version "1.5.0" - resolved "https://registry.yarnpkg.com/sonner/-/sonner-1.5.0.tgz#af359f817063318415326b33aab54c5d17c747b7" - integrity sha512-FBjhG/gnnbN6FY0jaNnqZOMmB73R+5IiyYAw8yBj7L54ER7HB3fOSE5OFiQiE2iXWxeXKvg6fIP4LtVppHEdJA== - sort-keys@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/sort-keys/-/sort-keys-2.0.0.tgz" @@ -23595,7 +23632,7 @@ type-fest@^1.2.1: resolved "https://registry.npmjs.org/type-fest/-/type-fest-1.4.0.tgz" integrity sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA== -type-is@^1.6.16, type-is@~1.6.18: +type-is@^1.6.16, type-is@^1.6.4, type-is@~1.6.18: version "1.6.18" resolved "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz" integrity sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g== @@ -23960,7 +23997,7 @@ update-notifier@^2.5.0: semver-diff "^2.0.0" xdg-basedir "^3.0.0" -uri-js@^4.2.2: +uri-js@^4.2.2, uri-js@^4.4.1: version "4.4.1" resolved "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz" integrity sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==