From 8a33ed48151d7a7d6b339bb9bf8f0a4404f42b59 Mon Sep 17 00:00:00 2001 From: Kanad Gupta Date: Thu, 12 Dec 2024 16:14:39 -0600 Subject: [PATCH] feat: adding `openapi upload` (#1116) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit | 🚥 Resolves RM-11384 | | :------------------- | ## 🧰 Changes `rdme openapi` has now been replaced by `rdme openapi upload` 🚀 highlights: - much simpler flag structure - more intuitive slug-based identifying system, thanks to API v2 - better error messages, thanks to API v2 code review notes - a lot of the diff is related to setting up boilerplate for interacting with API v2 - feedback appreciated on the docs — feel free to review `documentation/commands/openapi.md` or `src/commands/openapi/upload.ts` - feedback appreciated on whether the optional flag should be called `--slug` or `--identifier` or something else. went with `--slug` since it's short and it was easy to write clear explainer docs around
outstanding tasks (archived) - [x] command - [x] tests - [x] ~~optional `--action` arg?~~ **edit**: might be a good enhancement further down the line, but going to pass on this for now! - [x] tests for `useSpecVersion` - [x] test for `pending` timeout (there's a nock API for duplicating an interceptor, use that!) - [x] finalize command docs - [x] set up [test project](https://rdme-refactored-test.readme.io) for syncing in CI (i.e., adding back what i removed in https://github.com/readmeio/rdme/pull/1111/commits/d69d1aa69fbb577afbcf76301f37229d3f576419)
follow-up enhancements - tests for `readmeAPIv2Fetch` - optional `--action` arg for use in CI to prevent accidental overwrites - optional `--timeout` arg to allow user to configure timeout - GHA onboarding support
## 🧬 QA & Testing this can only be tested locally for now: 1. in your local dev server, load up a project that uses readme refactored 2. set up an API key for said project 3. change the value on [this line](https://github.com/readmeio/rdme/blob/af0e0dca7f715847bd5359d5bf60e1df63a81246/src/lib/config.ts#L4) to be `http://api.readme.local:3000/v2` 4. check out this branch and run the following: ```sh # setup npm ci && npm run build # try running any or all of the following: bin/dev.js openapi upload # try logging in bin/dev.js openapi upload --key # try selecting a file bin/dev.js openapi upload --key __tests__/__fixtures__/petstore-simple-weird-version.json # without a version arg bin/dev.js openapi upload --key __tests__/__fixtures__/petstore-simple-weird-version.json --version=1.0 # with a version arg bin/dev.js openapi upload --key __tests__/__fixtures__/petstore-simple-weird-version.json --useSpecVersion # use the version defined in the spec ``` --------- Co-authored-by: Jon Ursenbach --- .github/workflows/ci.yml | 22 ++ __tests__/commands/openapi/upload.test.ts | 409 ++++++++++++++++++++++ __tests__/helpers/get-api-mock.ts | 27 ++ __tests__/helpers/oclif.ts | 8 +- documentation/commands/openapi.md | 75 ++++ package-lock.json | 29 ++ package.json | 2 + src/commands/openapi/upload.ts | 214 +++++++++++ src/index.ts | 2 + src/lib/apiError.ts | 49 +++ src/lib/baseCommand.ts | 19 + src/lib/config.ts | 1 + src/lib/readmeAPIFetch.ts | 137 +++++++- 13 files changed, 989 insertions(+), 5 deletions(-) create mode 100644 __tests__/commands/openapi/upload.test.ts create mode 100644 src/commands/openapi/upload.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46011ec01..11326b9b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,3 +110,25 @@ jobs: if: ${{ github.ref }} == 'refs/heads/next' with: rdme: openapi validate oas-examples-repo/3.1/json/petstore.json + + # Docs: https://rdme-test.readme.io + - name: Run `openapi` command + uses: ./rdme-repo/ + with: + rdme: openapi upload oas-examples-repo/3.1/json/petstore.json --key=${{ secrets.RDME_REFACTORED_TEST_PROJECT_API_KEY }} + - name: Run `openapi` command with env var + uses: ./rdme-repo/ + with: + rdme: openapi upload oas-examples-repo/3.1/json/petstore.json + env: + RDME_API_KEY: ${{ secrets.RDME_REFACTORED_TEST_PROJECT_API_KEY }} + - name: Run `openapi` command with other env var + uses: ./rdme-repo/ + with: + rdme: openapi upload oas-examples-repo/3.1/json/petstore.json + env: + README_API_KEY: ${{ secrets.RDME_REFACTORED_TEST_PROJECT_API_KEY }} + - name: Run `openapi` command with weird arg syntax + uses: ./rdme-repo/ + with: + rdme: openapi upload "oas-examples-repo/3.1/json/petstore.json" --key "${{ secrets.RDME_REFACTORED_TEST_PROJECT_API_KEY }}" diff --git a/__tests__/commands/openapi/upload.test.ts b/__tests__/commands/openapi/upload.test.ts new file mode 100644 index 000000000..7e405999e --- /dev/null +++ b/__tests__/commands/openapi/upload.test.ts @@ -0,0 +1,409 @@ +import nock from 'nock'; +import prompts from 'prompts'; +import slugify from 'slugify'; +import { describe, beforeAll, beforeEach, afterEach, it, expect } from 'vitest'; + +import Command from '../../../src/commands/openapi/upload.js'; +import petstore from '../../__fixtures__/petstore-simple-weird-version.json' with { type: 'json' }; +import { getAPIv2Mock, getAPIv2MockForGHA } from '../../helpers/get-api-mock.js'; +import { runCommand, type OclifOutput } from '../../helpers/oclif.js'; +import { after, before } from '../../helpers/setup-gha-env.js'; + +const key = 'rdme_123'; +const version = '1.0.0'; +const filename = '__tests__/__fixtures__/petstore-simple-weird-version.json'; +const fileUrl = 'https://example.com/openapi.json'; +const slugifiedFilename = slugify.default(filename); + +describe('rdme openapi upload', () => { + let run: (args?: string[]) => OclifOutput; + + beforeAll(() => { + nock.disableNetConnect(); + run = runCommand(Command); + }); + + afterEach(() => { + nock.cleanAll(); + }); + + describe('flag error handling', () => { + it('should throw if an error if both `--version` and `--useSpecVersion` flags are passed', async () => { + const result = await run(['--useSpecVersion', '--version', version, filename, '--key', key]); + expect(result.error.message).toContain('--version=1.0.0 cannot also be provided when using --useSpecVersion'); + }); + }); + + describe('given that the API definition is a local file', () => { + it('should create a new API definition in ReadMe', async () => { + const mock = getAPIv2Mock({ authorization: `Bearer ${key}` }) + .get(`/versions/${version}/apis`) + .reply(200, { data: [] }) + .post(`/versions/${version}/apis`, body => + body.match(`form-data; name="schema"; filename="${slugifiedFilename}"`), + ) + .reply(200, { + data: { + upload: { status: 'done' }, + uri: `/versions/${version}/apis/${slugifiedFilename}`, + }, + }); + + const result = await run(['--version', version, filename, '--key', key]); + expect(result.stdout).toContain('was successfully created in ReadMe!'); + + mock.done(); + }); + + it('should update an existing API definition in ReadMe', async () => { + prompts.inject([true]); + + const mock = getAPIv2Mock({ authorization: `Bearer ${key}` }) + .get(`/versions/${version}/apis`) + .reply(200, { data: [{ filename: slugifiedFilename }] }) + .put(`/versions/1.0.0/apis/${slugifiedFilename}`, body => + body.match(`form-data; name="schema"; filename="${slugifiedFilename}"`), + ) + .reply(200, { + data: { + upload: { status: 'done' }, + uri: `/versions/${version}/apis/${slugifiedFilename}`, + }, + }); + + const result = await run(['--version', version, filename, '--key', key]); + expect(result.stdout).toContain('was successfully updated in ReadMe!'); + + mock.done(); + }); + + it('should handle upload failures', async () => { + const mock = getAPIv2Mock({ authorization: `Bearer ${key}` }) + .get(`/versions/${version}/apis`) + .reply(200, { data: [] }) + .post(`/versions/${version}/apis`, body => + body.match(`form-data; name="schema"; filename="${slugifiedFilename}"`), + ) + .reply(200, { + data: { + upload: { status: 'fail' }, + uri: `/versions/${version}/apis/${slugifiedFilename}`, + }, + }); + + const result = await run(['--version', version, filename, '--key', key]); + expect(result.error.message).toBe( + 'Your API definition upload failed with an unexpected error. Please get in touch with us at support@readme.io.', + ); + + mock.done(); + }); + + describe('and the `--slug` flag is passed', () => { + it('should use the provided slug (no file extension) as the filename', async () => { + const customSlug = 'custom-slug'; + const customSlugWithExtension = `${customSlug}.json`; + const mock = getAPIv2Mock({ authorization: `Bearer ${key}` }) + .get(`/versions/${version}/apis`) + .reply(200, { data: [] }) + .post(`/versions/${version}/apis`, body => + body.match(`form-data; name="schema"; filename="${customSlugWithExtension}"`), + ) + .reply(200, { + data: { + upload: { status: 'done' }, + uri: `/versions/${version}/apis/${customSlugWithExtension}`, + }, + }); + + const result = await run(['--version', version, filename, '--key', key, '--slug', customSlug]); + expect(result.stdout).toContain( + `Your API definition (${customSlugWithExtension}) was successfully created in ReadMe!`, + ); + + mock.done(); + }); + + it('should use the provided slug (includes file extension) as the filename', async () => { + const customSlug = 'custom-slug.json'; + const mock = getAPIv2Mock({ authorization: `Bearer ${key}` }) + .get(`/versions/${version}/apis`) + .reply(200, { data: [] }) + .post(`/versions/${version}/apis`, body => body.match(`form-data; name="schema"; filename="${customSlug}"`)) + .reply(200, { + data: { + upload: { status: 'done' }, + uri: `/versions/${version}/apis/${customSlug}`, + }, + }); + + const result = await run(['--version', version, filename, '--key', key, '--slug', customSlug]); + expect(result.stdout).toContain(`Your API definition (${customSlug}) was successfully created in ReadMe!`); + + mock.done(); + }); + + it('should handle a slug with an invalid file extension', async () => { + const customSlug = 'custom-slug.yikes'; + + const result = await run(['--version', version, filename, '--key', key, '--slug', customSlug]); + expect(result.error.message).toBe( + 'Please provide a valid file extension that matches the extension on the file you provided. Must be `.json`, `.yaml`, or `.yml`.', + ); + }); + + it('should handle a slug with a valid but mismatching file extension', async () => { + const customSlug = 'custom-slug.yml'; + + const result = await run(['--version', version, filename, '--key', key, '--slug', customSlug]); + expect(result.error.message).toBe( + 'Please provide a valid file extension that matches the extension on the file you provided. Must be `.json`, `.yaml`, or `.yml`.', + ); + }); + }); + + describe('and the upload status initially is a pending state', () => { + it('should poll the API until the upload is complete', async () => { + const mock = getAPIv2Mock({ authorization: `Bearer ${key}` }) + .get(`/versions/${version}/apis`) + .reply(200, { data: [] }) + .post(`/versions/${version}/apis`, body => + body.match(`form-data; name="schema"; filename="${slugifiedFilename}"`), + ) + .reply(200, { + data: { + upload: { status: 'pending' }, + uri: `/versions/${version}/apis/${slugifiedFilename}`, + }, + }) + .get(`/versions/${version}/apis/${slugifiedFilename}`) + .times(9) + .reply(200, { + data: { + upload: { status: 'pending' }, + uri: `/versions/${version}/apis/${slugifiedFilename}`, + }, + }) + .get(`/versions/${version}/apis/${slugifiedFilename}`) + .reply(200, { + data: { + upload: { status: 'done' }, + uri: `/versions/${version}/apis/${slugifiedFilename}`, + }, + }); + + const result = await run(['--version', version, filename, '--key', key]); + expect(result.stdout).toContain('was successfully created in ReadMe!'); + + mock.done(); + }); + + it('should poll the API and handle timeouts', async () => { + const mock = getAPIv2Mock({ authorization: `Bearer ${key}` }) + .get(`/versions/${version}/apis`) + .reply(200, { data: [] }) + .post(`/versions/${version}/apis`, body => + body.match(`form-data; name="schema"; filename="${slugifiedFilename}"`), + ) + .reply(200, { + data: { + upload: { status: 'pending' }, + uri: `/versions/${version}/apis/${slugifiedFilename}`, + }, + }) + .get(`/versions/${version}/apis/${slugifiedFilename}`) + .times(10) + .reply(200, { + data: { + upload: { status: 'pending' }, + uri: `/versions/${version}/apis/${slugifiedFilename}`, + }, + }); + + const result = await run(['--version', version, filename, '--key', key]); + expect(result.error.message).toBe('Sorry, this upload timed out. Please try again later.'); + + mock.done(); + }); + + it('should poll the API once and handle a failure state with a 4xx', async () => { + const mock = getAPIv2Mock({ authorization: `Bearer ${key}` }) + .get(`/versions/${version}/apis`) + .reply(200, { data: [] }) + .post(`/versions/${version}/apis`, body => + body.match(`form-data; name="schema"; filename="${slugifiedFilename}"`), + ) + .reply(200, { + data: { + upload: { status: 'pending' }, + uri: `/versions/${version}/apis/${slugifiedFilename}`, + }, + }) + .get(`/versions/${version}/apis/${slugifiedFilename}`) + .reply(400); + + const result = await run(['--version', version, filename, '--key', key]); + expect(result.error.message).toBe( + 'The ReadMe API responded with an unexpected error. Please try again and if this issue persists, get in touch with us at support@readme.io.', + ); + + mock.done(); + }); + + it('should poll the API once and handle an unexpected state with a 2xx', async () => { + const mock = getAPIv2Mock({ authorization: `Bearer ${key}` }) + .get(`/versions/${version}/apis`) + .reply(200, { data: [] }) + .post(`/versions/${version}/apis`, body => + body.match(`form-data; name="schema"; filename="${slugifiedFilename}"`), + ) + .reply(200, { + data: { + upload: { status: 'pending' }, + uri: `/versions/${version}/apis/${slugifiedFilename}`, + }, + }) + .get(`/versions/${version}/apis/${slugifiedFilename}`) + .reply(200, { + data: { + upload: { status: 'something-unexpected' }, + uri: `/versions/${version}/apis/${slugifiedFilename}`, + }, + }); + + const result = await run(['--version', version, filename, '--key', key]); + expect(result.error).toStrictEqual( + new Error( + 'Your API definition upload failed with an unexpected error. Please get in touch with us at support@readme.io.', + ), + ); + + mock.done(); + }); + }); + + describe('and the command is being run in a CI environment', () => { + beforeEach(before); + + afterEach(after); + + it('should overwrite an existing API definition without asking for confirmation', async () => { + const mock = getAPIv2MockForGHA({ authorization: `Bearer ${key}` }) + .get(`/versions/${version}/apis`) + .reply(200, { data: [{ filename: slugifiedFilename }] }) + .put(`/versions/1.0.0/apis/${slugifiedFilename}`, body => + body.match(`form-data; name="schema"; filename="${slugifiedFilename}"`), + ) + .reply(200, { + data: { + upload: { status: 'done' }, + uri: `/versions/${version}/apis/${slugifiedFilename}`, + }, + }); + + const result = await run(['--version', version, filename, '--key', key]); + expect(result.stdout).toContain('was successfully updated in ReadMe!'); + + mock.done(); + }); + }); + + describe('given that the `--version` flag is not set', () => { + it('should default to the `stable` version', async () => { + const mock = getAPIv2Mock({ authorization: `Bearer ${key}` }) + .get('/versions/stable/apis') + .reply(200, { data: [] }) + .post('/versions/stable/apis', body => + body.match(`form-data; name="schema"; filename="${slugifiedFilename}"`), + ) + .reply(200, { + data: { + upload: { status: 'done' }, + uri: `/versions/stable/apis/${slugifiedFilename}`, + }, + }); + + const result = await run([filename, '--key', key]); + expect(result.stdout).toContain('was successfully created in ReadMe!'); + + mock.done(); + }); + + it('should use the version from the spec file if --`useSpecVersion` is passed', async () => { + const altVersion = '1.2.3'; + const mock = getAPIv2Mock({ authorization: `Bearer ${key}` }) + .get(`/versions/${altVersion}/apis`) + .reply(200, { data: [] }) + .post(`/versions/${altVersion}/apis`, body => + body.match(`form-data; name="schema"; filename="${slugifiedFilename}"`), + ) + .reply(200, { + data: { + upload: { status: 'done' }, + uri: `/versions/${altVersion}/apis/${slugifiedFilename}`, + }, + }); + + const result = await run(['--useSpecVersion', filename, '--key', key]); + expect(result.stdout).toContain('was successfully created in ReadMe!'); + + mock.done(); + }); + }); + }); + + describe('given that the API definition is a URL', () => { + it('should create a new API definition in ReadMe', async () => { + const fileMock = nock('https://example.com').get('/openapi.json').reply(200, petstore); + + const mock = getAPIv2Mock({ authorization: `Bearer ${key}` }) + .get(`/versions/${version}/apis`) + .reply(200, {}) + .post(`/versions/${version}/apis`, body => body.match(`form-data; name="url"\r\n\r\n${fileUrl}`)) + .reply(200, { + data: { + upload: { status: 'done' }, + uri: `/versions/${version}/apis/openapi.json`, + }, + }); + + const result = await run(['--version', version, fileUrl, '--key', key]); + expect(result.stdout).toContain('was successfully created in ReadMe!'); + + fileMock.done(); + mock.done(); + }); + + it('should update an existing API definition in ReadMe', async () => { + prompts.inject([true]); + + const fileMock = nock('https://example.com').get('/openapi.json').reply(200, petstore); + + const mock = getAPIv2Mock({ authorization: `Bearer ${key}` }) + .get(`/versions/${version}/apis`) + .reply(200, { data: [{ filename: 'openapi.json' }] }) + .put('/versions/1.0.0/apis/openapi.json', body => body.match(`form-data; name="url"\r\n\r\n${fileUrl}`)) + .reply(200, { + data: { + upload: { status: 'done' }, + uri: `/versions/${version}/apis/openapi.json`, + }, + }); + + const result = await run(['--version', version, fileUrl, '--key', key]); + expect(result.stdout).toContain('was successfully updated in ReadMe!'); + + fileMock.done(); + mock.done(); + }); + + it('should handle issues fetching from the URL', async () => { + const fileMock = nock('https://example.com').get('/openapi.json').reply(400, {}); + + const result = await run(['--version', version, fileUrl, '--key', key]); + expect(result.error.message).toBe('Unknown file detected.'); + + fileMock.done(); + }); + }); +}); diff --git a/__tests__/helpers/get-api-mock.ts b/__tests__/helpers/get-api-mock.ts index 8ffd07f36..687ddabe5 100644 --- a/__tests__/helpers/get-api-mock.ts +++ b/__tests__/helpers/get-api-mock.ts @@ -3,6 +3,8 @@ import nock from 'nock'; import config from '../../src/lib/config.js'; import { getUserAgent } from '../../src/lib/readmeAPIFetch.js'; +import { mockVersion } from './oclif.js'; + /** * Nock wrapper for ReadMe API v1 that adds required * `user-agent` request header so it gets properly picked up by nock. @@ -15,3 +17,28 @@ export function getAPIv1Mock(reqHeaders = {}) { }, }); } + +/** + * Nock wrapper for ReadMe API v2 that adds required + * `user-agent` request header so it gets properly picked up by nock. + */ +export function getAPIv2Mock(reqHeaders: nock.Options['reqheaders'] = {}) { + return nock(config.host.v2, { + reqheaders: { + 'User-Agent': ua => ua.startsWith(`rdme/${mockVersion}`), + 'x-readme-source': 'cli', + ...reqHeaders, + }, + }); +} + +/** + * Variant of `getAPIv2Mock` for mocking a GitHub Actions environment. + */ +export function getAPIv2MockForGHA(reqHeaders: nock.Options['reqheaders'] = {}) { + return getAPIv2Mock({ + 'User-Agent': ua => ua.startsWith(`rdme-github/${mockVersion}`), + 'x-readme-source': 'cli-gh', + ...reqHeaders, + }); +} diff --git a/__tests__/helpers/oclif.ts b/__tests__/helpers/oclif.ts index a6ec356c4..9462dbed5 100644 --- a/__tests__/helpers/oclif.ts +++ b/__tests__/helpers/oclif.ts @@ -5,8 +5,12 @@ import path from 'node:path'; import { Config } from '@oclif/core'; import { captureOutput, runCommand as oclifRunCommand } from '@oclif/test'; +export type OclifOutput = ReturnType>; + const testNodeEnv = process.env.NODE_ENV; +export const mockVersion = '7.0.0'; + /** * Used for setting up the oclif configuration for simulating commands in tests. * This is a really barebones approach so we can continue using vitest + nock @@ -21,7 +25,7 @@ export function setupOclifConfig() { return Config.load({ root, - version: '7.0.0', + version: mockVersion, }); } @@ -32,7 +36,7 @@ export function setupOclifConfig() { * * @example runCommand(LoginCommand)(['--email', 'owlbert@example.com', '--password', 'password']) */ -function runCommand(Command: T) { +export function runCommand(Command: T) { return async function runCommandAgainstArgs(args?: string[]) { const oclifConfig = await setupOclifConfig(); // @ts-expect-error this is the pattern recommended by the @oclif/test docs. diff --git a/documentation/commands/openapi.md b/documentation/commands/openapi.md index e16d02fd8..f74d2e950 100644 --- a/documentation/commands/openapi.md +++ b/documentation/commands/openapi.md @@ -6,6 +6,7 @@ Manage your API definition (e.g., syncing, validation, analysis, conversion, etc * [`rdme openapi convert [SPEC]`](#rdme-openapi-convert-spec) * [`rdme openapi inspect [SPEC]`](#rdme-openapi-inspect-spec) * [`rdme openapi reduce [SPEC]`](#rdme-openapi-reduce-spec) +* [`rdme openapi upload [SPEC]`](#rdme-openapi-upload-spec) * [`rdme openapi validate [SPEC]`](#rdme-openapi-validate-spec) ## `rdme openapi convert [SPEC]` @@ -133,6 +134,80 @@ EXAMPLES $ rdme openapi reduce petstore.json --path /pet/{id} --method get --method put --out petstore.reduced.json ``` +## `rdme openapi upload [SPEC]` + +Upload (or reupload) your API definition to ReadMe. + +``` +USAGE + $ rdme openapi upload [SPEC] --key [--slug ] [--useSpecVersion | --version ] + +ARGUMENTS + SPEC A path to your API definition — either a local file path or a URL. If your working directory and all + subdirectories contain a single OpenAPI file, you can omit the path. + +FLAGS + --key= (required) ReadMe project API key + --slug= Override the slug (i.e., the unique identifier) for your API definition. + --useSpecVersion Use the OpenAPI `info.version` field for your ReadMe project version + --version= [default: stable] ReadMe project version + +DESCRIPTION + Upload (or reupload) your API definition to ReadMe. + + By default, the slug (i.e., the unique identifier for your API definition resource in ReadMe) will be inferred from + the spec name and path. As long as you maintain these directory/file names and run `rdme` from the same location + relative to your file, the inferred slug will be preserved and any updates you make to this file will be synced to the + same resource in ReadMe. + + If the spec is a local file, the inferred slug takes the relative path and slugifies it (e.g., the slug for + `docs/api/petstore.json` will be `docs-api-petstore.json`). + + If the spec is a URL, the inferred slug is the base file name from the URL (e.g., the slug for + `https://example.com/docs/petstore.json` will be `petstore.json`). + +EXAMPLES + You can pass in a file name like so: + + $ rdme openapi upload --version=1.0.0 openapi.json + + You can also pass in a file in a subdirectory (we recommend always running the CLI from the root of your + repository): + + $ rdme openapi upload --version=v1.0.0 example-directory/petstore.json + + You can also pass in a URL: + + $ rdme openapi upload --version=1.0.0 https://example.com/openapi.json + + If you specify your ReadMe project version in the `info.version` field in your OpenAPI definition, you can use that: + + $ rdme openapi upload --useSpecVersion https://example.com/openapi.json + +FLAG DESCRIPTIONS + --key= ReadMe project API key + + An API key for your ReadMe project. Note that API authentication is required despite being omitted from the example + usage. See our docs for more information: https://github.com/readmeio/rdme/tree/v10#authentication + + --slug= Override the slug (i.e., the unique identifier) for your API definition. + + Allows you to override the slug (i.e., the unique identifier for your API definition resource in ReadMe) that's + inferred from the API definition's file/URL path. + + You do not need to include a file extension (i.e., either `custom-slug.json` or `custom-slug` will work). If you do, + it must match the file extension of the file you're uploading. + + --useSpecVersion Use the OpenAPI `info.version` field for your ReadMe project version + + If included, use the version specified in the `info.version` field in your OpenAPI definition for your ReadMe + project version. This flag is mutually exclusive with `--version`. + + --version= ReadMe project version + + Defaults to `stable` (i.e., your main project version). This flag is mutually exclusive with `--useSpecVersion`. +``` + ## `rdme openapi validate [SPEC]` Validate your OpenAPI/Swagger definition. diff --git a/package-lock.json b/package-lock.json index 328dbc9b1..88286d32d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,8 +29,10 @@ "prompts": "^2.4.2", "semver": "^7.5.3", "simple-git": "^3.19.1", + "slugify": "^1.6.6", "string-argv": "^0.3.2", "table": "^6.8.1", + "tmp-promise": "^3.0.3", "toposort": "^2.0.2", "undici": "^5.28.4", "validator": "^13.7.0" @@ -15882,6 +15884,15 @@ "dev": true, "license": "MIT" }, + "node_modules/slugify": { + "version": "1.6.6", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", + "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/smob": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/smob/-/smob-1.5.0.tgz", @@ -16823,6 +16834,24 @@ "node": ">=0.6.0" } }, + "node_modules/tmp-promise": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.3.tgz", + "integrity": "sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==", + "license": "MIT", + "dependencies": { + "tmp": "^0.2.0" + } + }, + "node_modules/tmp-promise/node_modules/tmp": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz", + "integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==", + "license": "MIT", + "engines": { + "node": ">=14.14" + } + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", diff --git a/package.json b/package.json index de2b203ee..e69718d5d 100644 --- a/package.json +++ b/package.json @@ -61,8 +61,10 @@ "prompts": "^2.4.2", "semver": "^7.5.3", "simple-git": "^3.19.1", + "slugify": "^1.6.6", "string-argv": "^0.3.2", "table": "^6.8.1", + "tmp-promise": "^3.0.3", "toposort": "^2.0.2", "undici": "^5.28.4", "validator": "^13.7.0" diff --git a/src/commands/openapi/upload.ts b/src/commands/openapi/upload.ts new file mode 100644 index 000000000..874f911ba --- /dev/null +++ b/src/commands/openapi/upload.ts @@ -0,0 +1,214 @@ +import fs from 'node:fs'; +import nodePath from 'node:path'; + +import { Flags } from '@oclif/core'; +import ora from 'ora'; +import prompts from 'prompts'; +import slugify from 'slugify'; +import { file as tmpFile } from 'tmp-promise'; + +import BaseCommand from '../../lib/baseCommand.js'; +import { keyFlag, specArg } from '../../lib/flags.js'; +import isCI, { isTest } from '../../lib/isCI.js'; +import { oraOptions } from '../../lib/logger.js'; +import prepareOas from '../../lib/prepareOas.js'; +import promptTerminal from '../../lib/promptWrapper.js'; + +export default class OpenAPIUploadCommand extends BaseCommand { + static summary = 'Upload (or reupload) your API definition to ReadMe.'; + + static description = [ + 'By default, the slug (i.e., the unique identifier for your API definition resource in ReadMe) will be inferred from the spec name and path. As long as you maintain these directory/file names and run `rdme` from the same location relative to your file, the inferred slug will be preserved and any updates you make to this file will be synced to the same resource in ReadMe.', + 'If the spec is a local file, the inferred slug takes the relative path and slugifies it (e.g., the slug for `docs/api/petstore.json` will be `docs-api-petstore.json`).', + 'If the spec is a URL, the inferred slug is the base file name from the URL (e.g., the slug for `https://example.com/docs/petstore.json` will be `petstore.json`).', + ].join('\n\n'); + + static args = { + spec: specArg, + }; + + static flags = { + key: keyFlag, + slug: Flags.string({ + summary: 'Override the slug (i.e., the unique identifier) for your API definition.', + description: [ + "Allows you to override the slug (i.e., the unique identifier for your API definition resource in ReadMe) that's inferred from the API definition's file/URL path.", + "You do not need to include a file extension (i.e., either `custom-slug.json` or `custom-slug` will work). If you do, it must match the file extension of the file you're uploading.", + ].join('\n\n'), + }), + useSpecVersion: Flags.boolean({ + summary: 'Use the OpenAPI `info.version` field for your ReadMe project version', + description: + 'If included, use the version specified in the `info.version` field in your OpenAPI definition for your ReadMe project version. This flag is mutually exclusive with `--version`.', + exclusive: ['version'], + }), + version: Flags.string({ + summary: 'ReadMe project version', + description: + 'Defaults to `stable` (i.e., your main project version). This flag is mutually exclusive with `--useSpecVersion`.', + default: 'stable', + }), + }; + + static examples = [ + { + description: 'You can pass in a file name like so:', + command: '<%= config.bin %> <%= command.id %> --version=1.0.0 openapi.json', + }, + { + description: + 'You can also pass in a file in a subdirectory (we recommend always running the CLI from the root of your repository):', + command: '<%= config.bin %> <%= command.id %> --version=v1.0.0 example-directory/petstore.json', + }, + { + description: 'You can also pass in a URL:', + command: '<%= config.bin %> <%= command.id %> --version=1.0.0 https://example.com/openapi.json', + }, + { + description: + 'If you specify your ReadMe project version in the `info.version` field in your OpenAPI definition, you can use that:', + command: '<%= config.bin %> <%= command.id %> --useSpecVersion https://example.com/openapi.json', + }, + ]; + + /** + * Poll the ReadMe API until the upload is complete. + */ + private async pollAPIUntilUploadIsComplete(slug: string, headers: Headers) { + let count = 0; + let status = 'pending'; + + while (status === 'pending' && count < 10) { + // eslint-disable-next-line no-await-in-loop, no-loop-func + await new Promise(resolve => { + // exponential backoff — wait 1s, 2s, 4s, 8s, 16s, 32s, 30s, 30s, 30s, 30s, etc. + setTimeout(resolve, Math.min(isTest() ? 1 : 1000 * 2 ** count, 30000)); + }); + this.debug(`polling API for status of ${slug}, count is ${count}`); + // eslint-disable-next-line no-await-in-loop + const response = await this.readmeAPIFetch(slug, { headers }).then(res => this.handleAPIRes(res)); + status = response?.data?.upload?.status; + count += 1; + } + + if (status === 'pending') { + throw new Error('Sorry, this upload timed out. Please try again later.'); + } + + return status; + } + + async run() { + const { spec } = this.args; + + const { preparedSpec, specFileType, specPath, specVersion } = await prepareOas(spec, 'openapi'); + + const version = this.flags.useSpecVersion ? specVersion : this.flags.version; + + let filename = specFileType === 'url' ? nodePath.basename(specPath) : slugify.default(specPath); + + if (this.flags.slug) { + const fileExtension = nodePath.extname(filename); + const slugExtension = nodePath.extname(this.flags.slug); + if (slugExtension && (!['.json', '.yaml', '.yml'].includes(slugExtension) || fileExtension !== slugExtension)) { + throw new Error( + 'Please provide a valid file extension that matches the extension on the file you provided. Must be `.json`, `.yaml`, or `.yml`.', + ); + } + + // the API expects a file extension, so keep it if it's there, add it if it's not + filename = `${this.flags.slug.replace(slugExtension, '')}${fileExtension}`; + } + + const headers = new Headers({ authorization: `Bearer ${this.flags.key}` }); + + const existingAPIDefinitions = await this.readmeAPIFetch(`/versions/${version}/apis`, { headers }).then(res => + this.handleAPIRes(res), + ); + + // if the current slug already exists, we'll use PUT to update it. otherwise, we'll use POST to create it + const method = existingAPIDefinitions?.data?.some((d: { filename: string }) => d.filename === filename) + ? 'PUT' + : 'POST'; + + this.debug(`making a ${method} request`); + + // if the file already exists, ask the user if they want to overwrite it + if (method === 'PUT') { + // bypass the prompt if we're in a CI environment + prompts.override({ + confirm: isCI() ? true : undefined, + }); + + const { confirm } = await promptTerminal({ + type: 'confirm', + name: 'confirm', + message: `This will overwrite the existing API definition for ${filename}. Are you sure you want to continue?`, + }); + + if (!confirm) { + throw new Error('Aborting, no changes were made.'); + } + } + + const body = new FormData(); + + if (specFileType === 'url') { + this.debug('attaching URL to form data payload'); + body.append('url', specPath); + } else { + // Create a temporary file to write the bundled spec to, + // which we will then stream into the form data body + const { path } = await tmpFile({ prefix: 'rdme-openapi-', postfix: '.json' }); + this.debug(`creating temporary file at ${path}`); + fs.writeFileSync(path, preparedSpec); + const stream = fs.createReadStream(path); + + this.debug('file and stream created, streaming into form data payload'); + body.append('schema', { + [Symbol.toStringTag]: 'File', + name: filename, + stream: () => stream, + type: 'application/json', + }); + } + + const options: RequestInit = { headers, method, body }; + + const spinner = ora({ ...oraOptions() }).start( + `${method === 'POST' ? 'Creating' : 'Updating'} your API definition to ReadMe...`, + ); + + const response = await this.readmeAPIFetch( + `/versions/${version}/apis${method === 'POST' ? '' : `/${filename}`}`, + options, + ) + .then(res => this.handleAPIRes(res)) + .catch((err: Error) => { + spinner.fail(); + throw err; + }); + + if (response?.data?.upload?.status && response?.data?.uri) { + let status = response.data.upload.status; + + if (status === 'pending') { + spinner.text = `${spinner.text} uploaded but not yet processed by ReadMe. Polling for completion...`; + status = await this.pollAPIUntilUploadIsComplete(response.data.uri, headers); + } + + if (status === 'done') { + spinner.succeed(`${spinner.text} done!`); + this.log( + `🚀 Your API definition (${filename}) was successfully ${method === 'POST' ? 'created' : 'updated'} in ReadMe!`, + ); + return { uri: response.data.uri, status }; + } + } + + spinner.fail(); + throw new Error( + 'Your API definition upload failed with an unexpected error. Please get in touch with us at support@readme.io.', + ); + } +} diff --git a/src/index.ts b/src/index.ts index ebba17c98..891ad0f30 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,6 +4,7 @@ import LogoutCommand from './commands/logout.js'; import OpenAPIConvertCommand from './commands/openapi/convert.js'; import OpenAPIInspectCommand from './commands/openapi/inspect.js'; import OpenAPIReduceCommand from './commands/openapi/reduce.js'; +import OpenAPIUploadCommand from './commands/openapi/upload.js'; import OpenAPIValidateCommand from './commands/openapi/validate.js'; import WhoAmICommand from './commands/whoami.js'; @@ -29,6 +30,7 @@ export const COMMANDS = { 'openapi:convert': OpenAPIConvertCommand, 'openapi:inspect': OpenAPIInspectCommand, 'openapi:reduce': OpenAPIReduceCommand, + 'openapi:upload': OpenAPIUploadCommand, 'openapi:validate': OpenAPIValidateCommand, whoami: WhoAmICommand, diff --git a/src/lib/apiError.ts b/src/lib/apiError.ts index 0aab9ab87..70a26af81 100644 --- a/src/lib/apiError.ts +++ b/src/lib/apiError.ts @@ -1,6 +1,10 @@ +/* eslint-disable max-classes-per-file */ /** * APIv1ErrorResponse is the shape of the error response we get from ReadMe API v1. */ + +import chalk from 'chalk'; + interface APIv1ErrorResponse { docs?: string; error: string; @@ -10,6 +14,21 @@ interface APIv1ErrorResponse { suggestion?: string; } +/** + * APIv2ErrorResponse is the shape of the error response we get from ReadMe API v2. + */ +export type APIv2ErrorResponse = Partial<{ + detail: string; + errors?: { + key: string; + message: string; + }[]; + poem: string[]; + status: number; + title: string; + type: string; +}>; + /** * Error class for handling ReadMe API v1 errors. */ @@ -54,3 +73,33 @@ export class APIv1Error extends Error { } } } + +/** + * Error class for handling ReadMe API v2 errors. + */ +export class APIv2Error extends Error { + response: APIv2ErrorResponse; + + constructor(res: APIv2ErrorResponse) { + let stringified = + 'The ReadMe API responded with an unexpected error. Please try again and if this issue persists, get in touch with us at support@readme.io.'; + + if (res.title) { + stringified = `The ReadMe API returned the following error:\n\n${chalk.bold(res.title)}`; + } + + if (res.detail) { + stringified += `\n\n${res.detail}`; + } + + if (res.errors?.length) { + stringified += `\n\n${res.errors.map((e, i) => `${i + 1}. ${chalk.bold(e.key)}: ${e.message}`).join('\n')}`; + } + + super(stringified); + + this.name = 'APIv2Error'; + + this.response = res; + } +} diff --git a/src/lib/baseCommand.ts b/src/lib/baseCommand.ts index 2d552f177..d965f3526 100644 --- a/src/lib/baseCommand.ts +++ b/src/lib/baseCommand.ts @@ -9,6 +9,7 @@ import chalk from 'chalk'; import debugPkg from 'debug'; import { isGHA, isTest } from './isCI.js'; +import { handleAPIv2Res, readmeAPIv2Fetch, type FilePathDetails } from './readmeAPIFetch.js'; type Flags = Interfaces.InferredFlags<(typeof BaseCommand)['baseFlags'] & T['flags']>; type Args = Interfaces.InferredArgs; @@ -112,6 +113,24 @@ export default abstract class BaseCommand extends this.args = args as Args; } + /** + * Wrapper around `handleAPIv2Res` that binds the context of the class to the function. + */ + public async handleAPIRes(res: Response) { + return handleAPIv2Res.call(this, res); + } + + /** + * Wrapper around `readmeAPIv2Fetch` that binds the context of the class to the function. + */ + public async readmeAPIFetch( + pathname: string, + options: RequestInit = { headers: new Headers() }, + fileOpts: FilePathDetails = { filePath: '', fileType: false }, + ) { + return readmeAPIv2Fetch.call(this, pathname, options, fileOpts); + } + async runCreateGHAHook(opts: CreateGHAHookOptsInClass) { return this.config .runHook('createGHA', { diff --git a/src/lib/config.ts b/src/lib/config.ts index d07afd01a..384a39519 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -1,6 +1,7 @@ const config = { host: { v1: 'https://dash.readme.com', + v2: 'https://api.readme.com/v2', }, hub: 'https://{project}.readme.io', // this is only used for the `rdme open` command } as const; diff --git a/src/lib/readmeAPIFetch.ts b/src/lib/readmeAPIFetch.ts index 67789479f..4bd504274 100644 --- a/src/lib/readmeAPIFetch.ts +++ b/src/lib/readmeAPIFetch.ts @@ -1,11 +1,12 @@ import type { SpecFileType } from './prepareOas.js'; +import type { Command } from '@oclif/core'; import path from 'node:path'; import mime from 'mime-types'; import { ProxyAgent } from 'undici'; -import { APIv1Error } from './apiError.js'; +import { APIv1Error, APIv2Error, type APIv2ErrorResponse } from './apiError.js'; import config from './config.js'; import { git } from './createGHA/index.js'; import { getPkgVersion } from './getPkg.js'; @@ -18,7 +19,7 @@ const SUCCESS_NO_CONTENT = 204; * This contains a few pieces of information about a file so * we can properly construct a source URL for it. */ -interface FilePathDetails { +export interface FilePathDetails { /** The URL or local file path */ filePath: string; /** This is derived from the `oas-normalize` `type` property. */ @@ -84,7 +85,7 @@ function parseWarningHeader(header: string): WarningHeader[] { /** * Getter function for a string to be used in the user-agent header based on the current - * environment. + * environment. Used for API v1 requests. * */ export function getUserAgent() { @@ -208,6 +209,102 @@ export async function readmeAPIv1Fetch( }); } +/** + * Wrapper for the `fetch` API so we can add rdme-specific headers to all ReadMe API v2 requests. + * + * @param pathname the pathname to make the request to. Must have a leading slash. + * @param fileOpts optional object containing information about the file being sent. + * We use this to construct a full source URL for the file. + */ +export async function readmeAPIv2Fetch( + this: T, + pathname: string, + options: RequestInit = { headers: new Headers() }, + fileOpts: FilePathDetails = { filePath: '', fileType: false }, +) { + let source = 'cli'; + let headers = options.headers as Headers; + + if (!(options.headers instanceof Headers)) { + headers = new Headers(options.headers); + } + + headers.set( + 'User-Agent', + this.config.userAgent.replace(this.config.name, `${this.config.name}${isGHA() ? '-github' : ''}`), + ); + + if (!headers.get('accept')) { + headers.set('accept', 'application/json'); + } + + if (isGHA()) { + source = 'cli-gh'; + if (process.env.GITHUB_REPOSITORY) headers.set('x-github-repository', process.env.GITHUB_REPOSITORY); + if (process.env.GITHUB_RUN_ATTEMPT) headers.set('x-github-run-attempt', process.env.GITHUB_RUN_ATTEMPT); + if (process.env.GITHUB_RUN_ID) headers.set('x-github-run-id', process.env.GITHUB_RUN_ID); + if (process.env.GITHUB_RUN_NUMBER) headers.set('x-github-run-number', process.env.GITHUB_RUN_NUMBER); + if (process.env.GITHUB_SHA) headers.set('x-github-sha', process.env.GITHUB_SHA); + + const filePath = await normalizeFilePath(fileOpts); + + if (filePath) { + /** + * Constructs a full URL to the file using GitHub Actions runner variables + * @see {@link https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables} + * @example https://github.com/readmeio/rdme/blob/cb4129d5c7b51ff3b50f933a9c7d0c3d0d33d62c/documentation/rdme.md + */ + try { + const sourceUrl = new URL( + `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/blob/${process.env.GITHUB_SHA}/${filePath}`, + ).href; + headers.set('x-readme-source-url', sourceUrl); + } catch (e) { + this.debug(`error constructing github source url: ${e.message}`); + } + } + } + + if (isCI()) { + headers.set('x-rdme-ci', ciName()); + } + + headers.set('x-readme-source', source); + + if (fileOpts.filePath && fileOpts.fileType === 'url') { + headers.set('x-readme-source-url', fileOpts.filePath); + } + + const fullUrl = `${config.host.v2}${pathname}`; + const proxy = getProxy(); + + this.debug( + `making ${(options.method || 'get').toUpperCase()} request to ${fullUrl} ${proxy ? `with proxy ${proxy} and ` : ''}with headers: ${sanitizeHeaders(headers)}`, + ); + + return fetch(fullUrl, { + ...options, + headers, + // @ts-expect-error we need to clean up our undici usage here ASAP + dispatcher: proxy ? new ProxyAgent(proxy) : undefined, + }) + .then(res => { + const warningHeader = res.headers.get('Warning'); + if (warningHeader) { + this.debug(`received warning header: ${warningHeader}`); + const warnings = parseWarningHeader(warningHeader); + warnings.forEach(warning => { + warn(warning.message, 'ReadMe API Warning:'); + }); + } + return res; + }) + .catch(e => { + this.debug(`error making fetch request: ${e}`); + throw e; + }); +} + /** * Small handler for handling responses from ReadMe API v1. * @@ -244,6 +341,40 @@ export async function handleAPIv1Res(res: Response, rejectOnJsonError = true) { return Promise.reject(body); } +/** + * Small handler for handling responses from ReadMe API v2. + * + * If we receive JSON errors, we throw an APIError exception. + * + * If we receive non-JSON responses, we consider them errors and throw them. + */ +export async function handleAPIv2Res(this: T, res: Response) { + const contentType = res.headers.get('content-type') || ''; + const extension = mime.extension(contentType) || contentType.includes('json') ? 'json' : false; + if (extension === 'json') { + // TODO: type this better + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const body = (await res.json()) as any; + this.debug(`received status code ${res.status} from ${res.url} with JSON response: ${JSON.stringify(body)}`); + if (!res.ok) { + throw new APIv2Error(body as APIv2ErrorResponse); + } + return body; + } + if (res.status === SUCCESS_NO_CONTENT) { + this.debug(`received status code ${res.status} from ${res.url} with no content`); + return {}; + } + + // If we receive a non-JSON response, it's likely an error. + // Let's debug the raw response body and throw it. + const body = await res.text(); + this.debug(`received status code ${res.status} from ${res.url} with non-JSON response: ${body}`); + throw new Error( + 'The ReadMe API responded with an unexpected error. Please try again and if this issue persists, get in touch with us at support@readme.io.', + ); +} + /** * If you supply `undefined` or `null` to the `Headers` API, * it'll convert those to a string by default,