diff --git a/docs/zzapi-bundle-description.md b/docs/zzapi-bundle-description.md index 2592190..5e58fe6 100644 --- a/docs/zzapi-bundle-description.md +++ b/docs/zzapi-bundle-description.md @@ -173,6 +173,7 @@ Operators supported in the RHS are: * `$type`: string|number|object|array|null: to check the type of the field * `$tests`: perform assertions (recursively) on the value, as if it were the `$.` root * `$skip`: skip the assertions under this test. Useful in case some tests are failing, but we want the output to keep reminding us of this fact. +* `$multi`: use `jasonpath.query` (all matches) instead of `jasonpath.value` (first match) to evaluate the JSONPath expresson. This is useful if you need to evaluate multiple nested elements of an object array all at once. ### jsonpath tests diff --git a/examples/tests-bundle.zzb b/examples/tests-bundle.zzb index 1d56664..83eb4c5 100644 --- a/examples/tests-bundle.zzb +++ b/examples/tests-bundle.zzb @@ -33,7 +33,7 @@ requests: params: foo1: bar1 foo2: bar2 - tests: # old way of specifying json tests + tests: json: $.args.foo1: bar1 $.args.foo2: bar2 @@ -44,7 +44,7 @@ requests: params: foo1: bar1 foo2: - tests: # new way of specifying json + tests: $.url: "https://postman-echo.com/get?foo1=bar1&foo2" get-with-params-as-array: diff --git a/schemas/zzapi-bundle.schema.json b/schemas/zzapi-bundle.schema.json index e3a562d..d25fb36 100644 --- a/schemas/zzapi-bundle.schema.json +++ b/schemas/zzapi-bundle.schema.json @@ -112,7 +112,6 @@ "enum": ["number", "string", "boolean", "object", "array", "null", "undefined"] }, "$regex": { "type": "string" }, - "$options": { "type": "string" }, "$exists": { "type": "boolean" }, "$size": { "anyOf": [ @@ -132,7 +131,9 @@ ] }, "$tests": { "$ref": "#/$defs/tests" }, - "$skip": { "type": "boolean" } + "$options": { "type": "string" }, + "$skip": { "type": "boolean" }, + "$multi": { "type": "boolean" } } } ] diff --git a/src/constructCurl.ts b/src/constructCurl.ts index a35d381..9935a8a 100644 --- a/src/constructCurl.ts +++ b/src/constructCurl.ts @@ -1,7 +1,7 @@ import { getStringValueIfDefined } from "./utils/typeUtils"; -import { getParamsForUrl, getURL } from "./executeRequest"; import { RequestSpec } from "./models"; +import { getParamsForUrl, getURL } from "./executeRequest"; function replaceSingleQuotes(value: T): T { if (typeof value !== "string") return value; diff --git a/src/replaceVars.ts b/src/replaceVars.ts index 1accb4a..d494f5d 100644 --- a/src/replaceVars.ts +++ b/src/replaceVars.ts @@ -1,7 +1,7 @@ import { getStrictStringValue, isArrayOrDict } from "./utils/typeUtils"; -import { Variables } from "./variables"; import { RequestSpec } from "./models"; +import { Variables } from "./variables"; function replaceVariablesInArray( data: any[], diff --git a/src/runTests.ts b/src/runTests.ts index 1b784a8..096a461 100644 --- a/src/runTests.ts +++ b/src/runTests.ts @@ -2,9 +2,13 @@ import jp from "jsonpath"; import { getStringIfNotScalar, isDict } from "./utils/typeUtils"; -import { Tests, ResponseData, Assertion, SpecResult } from "./models"; +import { Tests, ResponseData, Assertion, SpecResult, TestResult } from "./models"; import { mergePrefixBasedTests } from "./mergeData"; +const SKIP_CLAUSE = "$skip", + OPTIONS_CLAUSE = "$options", + MULTI_CLAUSE = "$multi"; + function hasFailure(res: SpecResult): boolean { return res.results.some((r) => !r.pass) || res.subResults.some(hasFailure); } @@ -36,15 +40,35 @@ export function runAllTests( res.subResults.push(headerResults); } - for (const spec in tests.json) { - let expected = tests.json[spec], - received; + res.subResults.push(...runJsonTests(tests.json, responseData.json, skip)); + + if (tests.body) { + const expected = tests.body; + const received = responseData.body; + const bodyResults = runTest("body", expected, received, skip); + + res.subResults.push(bodyResults); + } + + return res; +} + +function runJsonTests(tests: { [key: string]: Assertion }, jsonData: any, skip?: boolean): SpecResult[] { + const results: SpecResult[] = []; + + for (const spec in tests) { + const expected = tests[spec]; + let received; try { - received = getValueForJSONTests(responseData.json, spec); + received = getValueForJSONTests( + jsonData, + spec, + typeof expected === "object" && expected !== null && expected[MULTI_CLAUSE], + ); } catch (err: any) { - res.subResults.push({ + results.push({ spec, - skipped: skip || (typeof expected === "object" && expected !== null && expected["$skip"]), + skipped: skip || (typeof expected === "object" && expected !== null && expected[SKIP_CLAUSE]), results: [{ pass: false, expected, received: "", op: spec, message: err }], subResults: [], }); @@ -52,18 +76,10 @@ export function runAllTests( } const jsonResults = runTest(spec, expected, received, skip); - res.subResults.push(jsonResults); + results.push(jsonResults); } - if (tests.body) { - const expected = tests.body; - const received = responseData.body; - const bodyResults = runTest("body", expected, received, skip); - - res.subResults.push(bodyResults); - } - - return res; + return results; } function runTest(spec: string, expected: Assertion, received: any, skip?: boolean): SpecResult { @@ -78,133 +94,286 @@ function runTest(spec: string, expected: Assertion, received: any, skip?: boolea return { spec, skipped: skip, results: [{ pass, expected, received, op: ":" }], subResults: [] }; } -function getValueForJSONTests(responseContent: object, key: string): any { +function getValueForJSONTests(responseContent: object, key: string, multi?: boolean): any { try { - return jp.value(responseContent, key); + return multi ? jp.query(responseContent, key) : jp.value(responseContent, key); } catch (err: any) { throw new Error(`Error while evaluating JSONPath ${key}: ${err.description || err.message || err}`); } } -function runObjectTests( +function getType(data: any): string { + if (data === null) return "null"; + if (Array.isArray(data)) return "array"; + return typeof data; +} + +const tests: { + [name: string]: ( + expectedObj: any, + receivedObj: any, + spec: string, + op: string, + options: { [key: string]: any }, + ) => SpecResult; +} = { + $eq: function (expectedObj, receivedObj, spec, op, options): SpecResult { + const received: Exclude = getStringIfNotScalar(receivedObj), + expected: Exclude = getStringIfNotScalar(expectedObj); + return { + spec, + subResults: [], + results: [{ pass: received === expected, expected, received, op }], + }; + }, + $ne: function (expectedObj, receivedObj, spec, op, options): SpecResult { + const received: Exclude = getStringIfNotScalar(receivedObj), + expected: Exclude = getStringIfNotScalar(expectedObj); + return { + spec, + subResults: [], + results: [{ pass: received !== expected, expected, received, op }], + }; + }, + $gt: function (expectedObj, receivedObj, spec, op, options): SpecResult { + const received: Exclude = getStringIfNotScalar(receivedObj), + expected: Exclude = getStringIfNotScalar(expectedObj); + return { + spec, + subResults: [], + results: [{ pass: received > expected, expected, received, op }], + }; + }, + $lt: function (expectedObj, receivedObj, spec, op, options): SpecResult { + const received: Exclude = getStringIfNotScalar(receivedObj), + expected: Exclude = getStringIfNotScalar(expectedObj); + return { + spec, + subResults: [], + results: [{ pass: received < expected, expected, received, op }], + }; + }, + $lte: function (expectedObj, receivedObj, spec, op, options): SpecResult { + const received: Exclude = getStringIfNotScalar(receivedObj), + expected: Exclude = getStringIfNotScalar(expectedObj); + return { + spec, + subResults: [], + results: [{ pass: received <= expected, expected, received, op }], + }; + }, + $gte: function (expectedObj, receivedObj, spec, op, options): SpecResult { + const received: Exclude = getStringIfNotScalar(receivedObj), + expected: Exclude = getStringIfNotScalar(expectedObj); + return { + spec, + subResults: [], + results: [{ pass: received >= expected, expected, received, op }], + }; + }, + $size: function (expectedObj, receivedObj, spec, op, options): SpecResult { + const res: SpecResult = { spec, results: [], subResults: [] }; + + const receivedLen: number | undefined = + typeof receivedObj === "object" && receivedObj !== null + ? Object.keys(receivedObj).length + : typeof receivedObj === "string" || Array.isArray(receivedObj) + ? receivedObj.length + : undefined; + + const received: Exclude = getStringIfNotScalar(receivedObj); + const expected: Exclude = getStringIfNotScalar(expectedObj); + + if (typeof expectedObj === "number") { + const compResult: TestResult = { + pass: expected === receivedLen, + op: op, + expected: expected, + received: `(length: ${receivedLen}) -> ${received}`, + }; + res.results.push(compResult); + + return res; + } + + if (isDict(expectedObj)) { + // the spec remains the same, so we add it to the current layer + const compRes: SpecResult = runObjectTests(expectedObj, receivedLen, spec); + res.results.push(...compRes.results); + res.subResults.push(...compRes.subResults); + + return res; + } + + const compResult: TestResult = { + pass: false, + op: op, + expected: expected, + received: received, + message: "value for $size is not a number or valid JSON", + }; + res.results.push(compResult); + + return res; + }, + $exists: function (expectedObj, receivedObj, spec, op, options): SpecResult { + const received: Exclude = getStringIfNotScalar(receivedObj), + expected: Exclude = getStringIfNotScalar(expectedObj); + const exists = received !== undefined; + return { + spec, + subResults: [], + results: [{ pass: exists === expected, expected, received, op }], + }; + }, + $type: function (expectedObj, receivedObj, spec, op, options): SpecResult { + const receivedType = getType(receivedObj); + const receivedStr: string = `${getStringIfNotScalar(receivedObj)} (type ${receivedType})`; + const expected: Exclude = getStringIfNotScalar(expectedObj); + return { + spec, + subResults: [], + results: [{ pass: expected === receivedType, expected, received: receivedStr, op }], + }; + }, + $regex: function (expectedObj, receivedObj, spec, op, options): SpecResult { + const received: Exclude = getStringIfNotScalar(receivedObj), + expected: Exclude = getStringIfNotScalar(expectedObj); + + const regexOpts = options[OPTIONS_CLAUSE]; + const regex = new RegExp(expected, regexOpts); + let pass: boolean = false, + message: string = ""; + try { + pass = typeof received === "string" && regex.test(received); + } catch (err: any) { + message = err.message; + } + + return { + spec, + subResults: [], + results: [{ pass, expected, received, op, message }], + }; + }, + $sw: function (expectedObj, receivedObj, spec, op, options): SpecResult { + const received: Exclude = getStringIfNotScalar(receivedObj), + expected: Exclude = getStringIfNotScalar(expectedObj); + return { + spec, + subResults: [], + results: [ + { pass: typeof received === "string" && received.startsWith(expected), expected, received, op }, + ], + }; + }, + $ew: function (expectedObj, receivedObj, spec, op, options): SpecResult { + const received: Exclude = getStringIfNotScalar(receivedObj), + expected: Exclude = getStringIfNotScalar(expectedObj); + return { + spec, + subResults: [], + results: [ + { pass: typeof received === "string" && received.endsWith(expected), expected, received, op }, + ], + }; + }, + $co: function (expectedObj, receivedObj, spec, op, options): SpecResult { + const received: Exclude = getStringIfNotScalar(receivedObj), + expected: Exclude = getStringIfNotScalar(expectedObj); + return { + spec, + subResults: [], + results: [ + { pass: typeof received === "string" && received.includes(expected), expected, received, op }, + ], + }; + }, + [OPTIONS_CLAUSE]: function (expectedObj, receivedObj, spec, op, options): SpecResult { + return { + spec, + subResults: [], + results: [], + }; + }, + [SKIP_CLAUSE]: function (expectedObj, receivedObj, spec, op, options): SpecResult { + return { + spec, + subResults: [], + results: [], + }; + }, + [MULTI_CLAUSE]: function (expectedObj, receivedObj, spec, op, options): SpecResult { + return { + spec, + subResults: [], + results: [], + }; + }, + $tests: function (expectedObj, receivedObj, spec, op, options): SpecResult { + const res: SpecResult = { spec, results: [], subResults: [] }; + + const expected: Exclude = getStringIfNotScalar(expectedObj); + const received: Exclude = getStringIfNotScalar(receivedObj); + + const recursiveTests = expectedObj; + if (!isDict(recursiveTests)) { + const compResult: TestResult = { + pass: false, + op: op, + expected: expected, + received: received, + message: "recursive tests must be dicts", + }; + res.results.push(compResult); + + return res; + } + + mergePrefixBasedTests(recursiveTests); + + // the spec remains the same, so we add it to the current layer + const compRes = runJsonTests(recursiveTests.json, receivedObj, options[SKIP_CLAUSE]); + res.subResults.push(...compRes); + + return res; + }, +}; + +export function runObjectTests( opVals: { [key: string]: any }, receivedObject: any, spec: string, skip?: boolean, ): SpecResult { - let objRes: SpecResult = { spec, results: [], subResults: [] }; - if (skip || opVals["$skip"]) objRes.skipped = true; + const objRes: SpecResult = { + spec, + results: [], + subResults: [], + skipped: skip || opVals[SKIP_CLAUSE], + }; + + const options: { [key: string]: any } = { + [OPTIONS_CLAUSE]: opVals[OPTIONS_CLAUSE], + [SKIP_CLAUSE]: objRes.skipped, + }; + const testNames = Object.keys(tests); for (const op in opVals) { - let expected = getStringIfNotScalar(opVals[op]); - let received = getStringIfNotScalar(receivedObject); - - let pass = false; - let message = ""; - if (op === "$eq") { - pass = received === expected; - } else if (op === "$ne") { - pass = received !== expected; - } else if (op === "$lt") { - pass = received < expected; - } else if (op === "$gt") { - pass = receivedObject > expected; - } else if (op === "$lte") { - pass = receivedObject <= expected; - } else if (op === "$gte") { - pass = receivedObject >= expected; - } else if (op === "$size") { - let receivedLen: number | undefined = undefined; - if (typeof receivedObject === "object") { - receivedLen = Object.keys(receivedObject).length; - } else if (typeof receivedObject === "string" || Array.isArray(receivedObject)) { - receivedLen = receivedObject.length; - } - if (typeof expected === "number") { - pass = receivedLen === expected; - } else { - try { - expected = JSON.parse(expected); - - // the spec remains the same, so we add it to the current layer - const res = runObjectTests(expected, receivedLen, spec); - objRes.results.push(...res.results); - objRes.subResults.push(...res.subResults); - continue; - } catch (err: any) { - pass = false; - message = `$size val is not num or valid JSON`; - } - } - } else if (op === "$exists") { - const exists = received !== undefined; - pass = exists === expected; - } else if (op === "$type") { - const receivedType = getType(receivedObject); - pass = expected === receivedType; - received = `${received} (type ${receivedType})`; - } else if (op === "$regex") { - const options = opVals["$options"]; - const regex = new RegExp(expected, options); - try { - pass = typeof received === "string" && regex.test(received); - } catch (err: any) { - message = err.message; - } - } else if (op === "$sw") { - pass = typeof received === "string" && received.startsWith(expected); - } else if (op === "$ew") { - pass = typeof received === "string" && received.endsWith(expected); - } else if (op === "$co") { - pass = typeof received === "string" && received.includes(expected); - } else if (op === "$options") { - continue; // do nothing. $regex will address it. - } else if (op === "$tests") { - const recursiveTests = opVals[op]; - - if (!isDict(recursiveTests)) { - pass = false; - message = "recursive tests must be dicts"; - } else { - mergePrefixBasedTests(recursiveTests); - const receivedObj: ResponseData = { - executionTime: 0, - body: "", - rawHeaders: "", - headers: {}, - json: receivedObject, - }; - - // the spec remains the same, so we add it to the current layer - const res = runAllTests(recursiveTests, receivedObj, false, spec, objRes.skipped); - objRes.results.push(...res.results); - objRes.subResults.push(...res.subResults); - continue; - } - } else if (op === "$skip") { - continue; // do nothing. If it wasn't already addressed, that means the test is not to be skipped. + if (testNames.includes(op)) { + const res: SpecResult = tests[op](opVals[op], receivedObject, spec, op, options); + objRes.results.push(...res.results); + objRes.subResults.push(...res.subResults); } else { objRes.results.push({ pass: false, - expected: "one of $eq, $ne etc.", + expected: `one of ${testNames.join(", ")}`, received: op, op: "", - message: "To compare objects, use $eq", + message: "Note: use $eq to compare objects", }); - continue; } - objRes.results.push({ pass, expected, received, op, message }); } return objRes; } - -function getType(data: any): string { - if (data === null) { - return "null"; - } else if (Array.isArray(data)) { - return "array"; - } else { - return typeof data; - } -} diff --git a/src/utils/typeUtils.ts b/src/utils/typeUtils.ts index 6c05ca9..8dd38be 100644 --- a/src/utils/typeUtils.ts +++ b/src/utils/typeUtils.ts @@ -14,7 +14,7 @@ export function getDescriptiveType(obj: any): string { return typeof obj; } -export function getStringIfNotScalar(data: any) { +export function getStringIfNotScalar(data: any): Exclude { if (typeof data !== "object") return data; return JSON.stringify(data); } diff --git a/tests/bundles/auto-negative-schema.zzb b/tests/bundles/auto-negative-schema.zzb new file mode 100644 index 0000000..00f98ee --- /dev/null +++ b/tests/bundles/auto-negative-schema.zzb @@ -0,0 +1,23 @@ +common: + baseUrl: https://postman-echo.com + headers: + Content-type: application/json + +requests: + # This request tests should all fail due to bad tests schema + # Ensure we don't crash on these. + tests-negative-schema: + method: POST + url: /post + body: + address: 1, example street + numbers: [444, 222] + object: { foo: bar } + tests: + status: { $ne: 200 } + headers: + content-type: { $exists: false } + $.data.operator: { badop: any } # invalid operator badop. If you want to match an entire object/array, use it as the value of the $eq operator. + $.data.numbers: [444, 222] + $.data.address: { $type: invalid } + $.data.object: { $exists: 4 } diff --git a/tests/bundles/auto-tests.zzb b/tests/bundles/auto-tests.zzb index 8ed4365..5a108c4 100644 --- a/tests/bundles/auto-tests.zzb +++ b/tests/bundles/auto-tests.zzb @@ -42,7 +42,7 @@ requests: params: foo1: bar1 foo2: bar2 - tests: # old way of specifying json tests + tests: status: { $eq: 0, $skip: true } $h.Content-type: { $eq: random-test, $skip: true } json: @@ -58,7 +58,7 @@ requests: params: foo1: bar1 foo2: - tests: # new way of specifying json + tests: $.url: "https://postman-echo.com/get?foo1=bar1&foo2" get-with-params-as-array-positive: @@ -255,8 +255,9 @@ requests: $.data.phoneNumbers[0].type: mobile $.data.phoneNumbers.1.type: home $.data.phoneNumbers[?(@.type=="home")].number: 0123-4567-8910 - $.data.phoneNumbers[*].type: mobile - $.data.phoneNumbers[*].available: { $eq: [7, 22] } + $.data.phoneNumbers[*].number: 0123-4567-8888 # without multi option, it compares only the first + $.data.phoneNumbers[*].available: { $eq: [7, 22] } # without multi option, it compares only the first + $.data.phoneNumbers[*].type: { $eq: ["mobile", "home"], $multi: true } $.data.phoneNumbers.0.available: { $eq: [7, 22], $type: array } $.data.phoneNumbers.1.available: { $eq: "[18,22]", $type: array } $.data.phoneNumbers.0: @@ -297,25 +298,6 @@ requests: $.data.age.something: 55 # jsonpath should take care of this. $.data.numbers[5]: 0 # jsonpath should take care of this - # This request tests should all fail due to bad tests schema - # Ensure we don't crash on these. - tests-negative-schema: - method: POST - url: /post - body: - address: 1, example street - numbers: [444, 222] - object: { foo: bar } - tests: - status: { $ne: 200 } - headers: - content-type: { $exists: false } - # Uncomment the following to run tests. Schema validation makes these invalid. - # $.data.operator: { badop: any } # invalid operator badop. If you want to match an entire object/array, use it as the value of the $eq operator. - # $.data.numbers: [444, 222] - # $.data.address: { $type: invalid } - # $.data.object: { $exists: 4 } - capture-response-positive: method: POST url: /post diff --git a/tests/curl.test.ts b/tests/curl.test.ts new file mode 100644 index 0000000..1501080 --- /dev/null +++ b/tests/curl.test.ts @@ -0,0 +1,41 @@ +import * as fs from "fs"; +import util from "util"; +import { exec } from "child_process"; + +import { getCurlRequest, getRequestSpec } from "../src/index"; + +const execPromise = util.promisify(exec); + +async function runCurlRequest( + bundlePath: string, + requestName: string, +): Promise<{ stdout: string; stderr: string }> { + const content = fs.readFileSync(bundlePath, "utf-8"); + const request = getRequestSpec(content, requestName); + + const curlReq = getCurlRequest(request); + + return await execPromise(curlReq); +} + +test("execute simple-get cURL", async () => { + const { stdout, stderr } = await runCurlRequest( + "./tests/bundles/auto-tests.zzb", + "simple-get-positive", + ); + + const response = JSON.parse(stdout); + expect(response.url).toBe("https://postman-echo.com/get"); +}); + +test("execute post-header-merge cURL", async () => { + const { stdout, stderr } = await runCurlRequest( + "./tests/bundles/auto-tests.zzb", + "post-header-merge-positive", + ); + + const response = JSON.parse(stdout); + expect(response.url).toBe("https://postman-echo.com/post"); + expect(response.headers["x-custom-header"]).toBe("Custom Value"); + expect(response.json["foo1"]).toBe("bar1"); +}); diff --git a/tests/integration.test.ts b/tests/integration.test.ts index 9fcd142..037230a 100644 --- a/tests/integration.test.ts +++ b/tests/integration.test.ts @@ -6,3 +6,8 @@ test("execute auto-tests.zzb in default env", async () => { const rawReq = new RawRequest("./tests/bundles/auto-tests.zzb", "default"); expect(await callRequests(rawReq)).toBe(0); }); + +test("execute auto-negative-schema.zzb in default env", async () => { + const rawReq = new RawRequest("./tests/bundles/auto-negative-schema.zzb", "default"); + expect(await callRequests(rawReq)).toBe(0); +}); diff --git a/tests/runTests.ts b/tests/runTests.ts index 1b858fd..0485867 100644 --- a/tests/runTests.ts +++ b/tests/runTests.ts @@ -115,7 +115,7 @@ function allPositive(res: SpecResult, numTests: number): string[] { const SKIP_CLAUSE = "$skip"; const TEST_CLAUSE = "$tests"; -const NON_TEST_KEYS = ["$options", "multi", TEST_CLAUSE, SKIP_CLAUSE]; +const NON_TEST_KEYS = ["$options", "$multi", TEST_CLAUSE, SKIP_CLAUSE]; function getNumTests(tests: Tests) { let numTests = 0;