From 4d28e7454dfa2d0e51ca7fcd315780d2930c2831 Mon Sep 17 00:00:00 2001 From: Varun0157 Date: Tue, 4 Jun 2024 14:34:57 +0530 Subject: [PATCH 01/12] feat: adding skipped attribute, not running object tests in case of skip invocation --- src/models.ts | 1 + src/runTests.ts | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/models.ts b/src/models.ts index 1d675d3..0aaddb0 100644 --- a/src/models.ts +++ b/src/models.ts @@ -133,6 +133,7 @@ export interface TestResult { export interface SpecResult { spec: string | null; + skipped?: boolean; results: TestResult[]; subResults: SpecResult[]; } diff --git a/src/runTests.ts b/src/runTests.ts index 6a4bbe4..d2246e9 100644 --- a/src/runTests.ts +++ b/src/runTests.ts @@ -9,7 +9,7 @@ export function runAllTests( tests: Tests, responseData: ResponseData, stopOnFailure: boolean, - rootSpec: string | null = null, + rootSpec: string | null = null ): SpecResult { const res: SpecResult = { spec: rootSpec, results: [], subResults: [] }; if (!tests) return res; @@ -80,6 +80,10 @@ function getValueForJSONTests(responseContent: object, key: string): any { function runObjectTests(opVals: { [key: string]: any }, receivedObject: any, spec: string): SpecResult { let objRes: SpecResult = { spec, results: [], subResults: [] }; + if (opVals["$skip"]) { + objRes.skipped = true; + return objRes; + } for (const op in opVals) { let expected = getStringIfNotScalar(opVals[op]); @@ -167,6 +171,8 @@ function runObjectTests(opVals: { [key: string]: any }, receivedObject: any, spe objRes.subResults.push(...res.subResults); continue; } + } else if (op === "$skip") { + continue; // do nothing. If it wasn't already addressed, that means the test is not to be skipped. } else { objRes.results.push({ pass: false, From 11cb3cc79b588ceab69b4de60bb9937a8c76bce4 Mon Sep 17 00:00:00 2001 From: Varun0157 Date: Tue, 4 Jun 2024 14:35:24 +0530 Subject: [PATCH 02/12] reformat: prettier --- src/runTests.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runTests.ts b/src/runTests.ts index d2246e9..cd85636 100644 --- a/src/runTests.ts +++ b/src/runTests.ts @@ -9,7 +9,7 @@ export function runAllTests( tests: Tests, responseData: ResponseData, stopOnFailure: boolean, - rootSpec: string | null = null + rootSpec: string | null = null, ): SpecResult { const res: SpecResult = { spec: rootSpec, results: [], subResults: [] }; if (!tests) return res; From 67c2bfa22cbe1e3265af32459a9929a12f26074c Mon Sep 17 00:00:00 2001 From: Varun0157 Date: Tue, 4 Jun 2024 15:14:17 +0530 Subject: [PATCH 03/12] feat: still displaying tests in case of skipping a spec --- src/runTests.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/runTests.ts b/src/runTests.ts index cd85636..79e7e6d 100644 --- a/src/runTests.ts +++ b/src/runTests.ts @@ -9,7 +9,7 @@ export function runAllTests( tests: Tests, responseData: ResponseData, stopOnFailure: boolean, - rootSpec: string | null = null, + rootSpec: string | null = null ): SpecResult { const res: SpecResult = { spec: rootSpec, results: [], subResults: [] }; if (!tests) return res; @@ -80,10 +80,7 @@ function getValueForJSONTests(responseContent: object, key: string): any { function runObjectTests(opVals: { [key: string]: any }, receivedObject: any, spec: string): SpecResult { let objRes: SpecResult = { spec, results: [], subResults: [] }; - if (opVals["$skip"]) { - objRes.skipped = true; - return objRes; - } + if (opVals["$skip"]) objRes.skipped = true; for (const op in opVals) { let expected = getStringIfNotScalar(opVals[op]); From 8f670df76186957e9ac12cef118f3a3d36a0c689 Mon Sep 17 00:00:00 2001 From: Varun0157 Date: Tue, 4 Jun 2024 15:30:59 +0530 Subject: [PATCH 04/12] reformat: prettier --- src/runTests.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runTests.ts b/src/runTests.ts index 79e7e6d..19f57ee 100644 --- a/src/runTests.ts +++ b/src/runTests.ts @@ -9,7 +9,7 @@ export function runAllTests( tests: Tests, responseData: ResponseData, stopOnFailure: boolean, - rootSpec: string | null = null + rootSpec: string | null = null, ): SpecResult { const res: SpecResult = { spec: rootSpec, results: [], subResults: [] }; if (!tests) return res; From e6949bc415ac5174b17cbacc3d385ea609b31c43 Mon Sep 17 00:00:00 2001 From: Varun0157 Date: Tue, 4 Jun 2024 15:46:23 +0530 Subject: [PATCH 05/12] feat: skipping tests in sub-specs in case of $skip clause in parent --- src/runTests.ts | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/runTests.ts b/src/runTests.ts index 19f57ee..2066b61 100644 --- a/src/runTests.ts +++ b/src/runTests.ts @@ -10,6 +10,7 @@ export function runAllTests( responseData: ResponseData, stopOnFailure: boolean, rootSpec: string | null = null, + skip?: boolean ): SpecResult { const res: SpecResult = { spec: rootSpec, results: [], subResults: [] }; if (!tests) return res; @@ -17,7 +18,7 @@ export function runAllTests( if (tests.status) { const expected = tests.status; const received = responseData.status; - const statusResults = runTest("status", expected, received); + const statusResults = runTest("status", expected, received, skip); if (stopOnFailure && statusResults.results.some((r) => !r.pass)) return res; } @@ -25,7 +26,7 @@ export function runAllTests( for (const spec in tests.headers) { const expected = tests.headers[spec]; const received = responseData.headers ? responseData.headers[spec] : ""; - const headerResults = runTest(spec, expected, received); + const headerResults = runTest(spec, expected, received, skip); res.subResults.push(headerResults); } @@ -44,14 +45,14 @@ export function runAllTests( continue; } - const jsonResults = runTest(spec, expected, received); + const jsonResults = runTest(spec, expected, received, skip); res.subResults.push(jsonResults); } if (tests.body) { const expected = tests.body; const received = responseData.body; - const bodyResults = runTest("body", expected, received); + const bodyResults = runTest("body", expected, received, skip); res.subResults.push(bodyResults); } @@ -59,15 +60,16 @@ export function runAllTests( return res; } -function runTest(spec: string, expected: Assertion, received: any): SpecResult { +function runTest(spec: string, expected: Assertion, received: any, skip?: boolean): SpecResult { // typeof null is also 'object' - if (expected !== null && typeof expected === "object") return runObjectTests(expected, received, spec); + if (expected !== null && typeof expected === "object") + return runObjectTests(expected, received, spec, skip); expected = getStringIfNotScalar(expected); received = getStringIfNotScalar(received); const pass = expected === received; - return { spec, results: [{ pass, expected, received, op: ":" }], subResults: [] }; + return { spec, skipped: skip, results: [{ pass, expected, received, op: ":" }], subResults: [] }; } function getValueForJSONTests(responseContent: object, key: string): any { @@ -78,9 +80,14 @@ function getValueForJSONTests(responseContent: object, key: string): any { } } -function runObjectTests(opVals: { [key: string]: any }, receivedObject: any, spec: string): SpecResult { +function runObjectTests( + opVals: { [key: string]: any }, + receivedObject: any, + spec: string, + skip?: boolean +): SpecResult { let objRes: SpecResult = { spec, results: [], subResults: [] }; - if (opVals["$skip"]) objRes.skipped = true; + if (skip || opVals["$skip"]) objRes.skipped = true; for (const op in opVals) { let expected = getStringIfNotScalar(opVals[op]); From b79d4302d64d3d365b6a2f560da7290c27d54504 Mon Sep 17 00:00:00 2001 From: Varun0157 Date: Tue, 4 Jun 2024 15:52:06 +0530 Subject: [PATCH 06/12] bug: passing skipping condition to recursive call --- src/runTests.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runTests.ts b/src/runTests.ts index 2066b61..e93d94a 100644 --- a/src/runTests.ts +++ b/src/runTests.ts @@ -170,7 +170,7 @@ function runObjectTests( }; // the spec remains the same, so we add it to the current layer - const res = runAllTests(recursiveTests, receivedObj, false, spec); + const res = runAllTests(recursiveTests, receivedObj, false, spec, objRes.skipped); objRes.results.push(...res.results); objRes.subResults.push(...res.subResults); continue; From e84f3c22bcada2c28b9ba5a7920883e2771f3f10 Mon Sep 17 00:00:00 2001 From: Varun0157 Date: Tue, 4 Jun 2024 15:52:35 +0530 Subject: [PATCH 07/12] reformat: prettier --- examples/tests-bundle.zzb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/tests-bundle.zzb b/examples/tests-bundle.zzb index c8a1b21..1d56664 100644 --- a/examples/tests-bundle.zzb +++ b/examples/tests-bundle.zzb @@ -67,8 +67,8 @@ requests: - multi-1 - multi-2 tests: - $.args.foo1: bar1 - $.args.foo2: { $eq: ["multi-1", "multi-2"] } + $.args.foo1: bar + $.args.foo2: { $eq: ["multi-1", "multi-2"], $skip: true } post-header-merge: url: /post @@ -232,7 +232,7 @@ requests: number: 0123-4567-8910 available: [18, 22] tests: - $.data.firstName: John + $.data.firstName: Joh $.data.age: { $type: number, $eq: 26, $ne: 30, $gt: 25, $lt: 28 } $.data.address: { $type: object, $size: 3, $exists: true } $.data.address.city: Nara @@ -246,8 +246,9 @@ requests: $.data.phoneNumbers.0.available: { $eq: [7, 22], $type: array } $.data.phoneNumbers.1.available: { $eq: "[18,22]", $type: array } $.data.phoneNumbers.0: + $skip: true $tests: - $.type: mobiles + $.type: mobile $.number: { $ne: 0123-4567-8910 } $.available: { $eq: [7, 22] } $.data.lastName: { $exists: true } From 44dc57424840f2d15fd0392e7c90af4733d7b6fa Mon Sep 17 00:00:00 2001 From: Varun0157 Date: Tue, 4 Jun 2024 16:05:55 +0530 Subject: [PATCH 08/12] reformat: prettier --- src/runTests.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runTests.ts b/src/runTests.ts index e93d94a..7e66dd0 100644 --- a/src/runTests.ts +++ b/src/runTests.ts @@ -10,7 +10,7 @@ export function runAllTests( responseData: ResponseData, stopOnFailure: boolean, rootSpec: string | null = null, - skip?: boolean + skip?: boolean, ): SpecResult { const res: SpecResult = { spec: rootSpec, results: [], subResults: [] }; if (!tests) return res; @@ -84,7 +84,7 @@ function runObjectTests( opVals: { [key: string]: any }, receivedObject: any, spec: string, - skip?: boolean + skip?: boolean, ): SpecResult { let objRes: SpecResult = { spec, results: [], subResults: [] }; if (skip || opVals["$skip"]) objRes.skipped = true; From 2dcdf28f2a47d4db22a7bdc012ff231da6f7e901 Mon Sep 17 00:00:00 2001 From: Varun0157 Date: Tue, 4 Jun 2024 16:39:22 +0530 Subject: [PATCH 09/12] bug: accounting for skipped tests in case of failed jsonpath eval --- src/runTests.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/runTests.ts b/src/runTests.ts index 7e66dd0..01944d2 100644 --- a/src/runTests.ts +++ b/src/runTests.ts @@ -10,7 +10,7 @@ export function runAllTests( responseData: ResponseData, stopOnFailure: boolean, rootSpec: string | null = null, - skip?: boolean, + skip?: boolean ): SpecResult { const res: SpecResult = { spec: rootSpec, results: [], subResults: [] }; if (!tests) return res; @@ -39,6 +39,7 @@ export function runAllTests( } catch (err: any) { res.subResults.push({ spec, + skipped: skip || (typeof expected === "object" && expected !== null && expected["$skip"]), results: [{ pass: false, expected, received: "", op: spec, message: err }], subResults: [], }); @@ -84,7 +85,7 @@ function runObjectTests( opVals: { [key: string]: any }, receivedObject: any, spec: string, - skip?: boolean, + skip?: boolean ): SpecResult { let objRes: SpecResult = { spec, results: [], subResults: [] }; if (skip || opVals["$skip"]) objRes.skipped = true; From dd90919d13587f53386236919867ab6e014b5669 Mon Sep 17 00:00:00 2001 From: Varun0157 Date: Tue, 4 Jun 2024 16:39:48 +0530 Subject: [PATCH 10/12] refactor: prettier --- src/runTests.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runTests.ts b/src/runTests.ts index 01944d2..d5f58fb 100644 --- a/src/runTests.ts +++ b/src/runTests.ts @@ -10,7 +10,7 @@ export function runAllTests( responseData: ResponseData, stopOnFailure: boolean, rootSpec: string | null = null, - skip?: boolean + skip?: boolean, ): SpecResult { const res: SpecResult = { spec: rootSpec, results: [], subResults: [] }; if (!tests) return res; @@ -85,7 +85,7 @@ function runObjectTests( opVals: { [key: string]: any }, receivedObject: any, spec: string, - skip?: boolean + skip?: boolean, ): SpecResult { let objRes: SpecResult = { spec, results: [], subResults: [] }; if (skip || opVals["$skip"]) objRes.skipped = true; From ee4436277c80cae884e4cf445fa117895aea4ab0 Mon Sep 17 00:00:00 2001 From: Vasan Subramanian Date: Sat, 8 Jun 2024 12:19:10 +0530 Subject: [PATCH 11/12] Added $skip to bundle schema --- schemas/zzapi-bundle.schema.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schemas/zzapi-bundle.schema.json b/schemas/zzapi-bundle.schema.json index 82fc8af..e3a562d 100644 --- a/schemas/zzapi-bundle.schema.json +++ b/schemas/zzapi-bundle.schema.json @@ -131,7 +131,8 @@ } ] }, - "$tests": { "$ref": "#/$defs/tests" } + "$tests": { "$ref": "#/$defs/tests" }, + "$skip": { "type": "boolean" } } } ] From 517495b71937bfb2e2734e3414133338645cf481 Mon Sep 17 00:00:00 2001 From: Vasan Subramanian Date: Sat, 8 Jun 2024 12:27:18 +0530 Subject: [PATCH 12/12] Added $skip to the documentation. --- docs/zzapi-bundle-description.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/zzapi-bundle-description.md b/docs/zzapi-bundle-description.md index 7b98808..2592190 100644 --- a/docs/zzapi-bundle-description.md +++ b/docs/zzapi-bundle-description.md @@ -90,6 +90,7 @@ These are options that can be switches on/off, both at the common level as well * `follow`: whether to follow redirects (default is false) * `verifySSL`: whether to enfoce SSL certificate validation (default is false) * `showHeaders`: whether to show the response headers for each request (default is false) +* `rawParams`: whether to consider the params as is (default is false, ie use URL encoding) * `keepRawJSON`: whether to keep the original response JSON without formatting it. (default is false) * `stopOnFailure`: whether to skip other tests when any of the status tests fail. @@ -171,6 +172,7 @@ Operators supported in the RHS are: * `$exists`: true|false, to check existance of a field * `$type`: string|number|object|array|null: to check the type of the field * `$tests`: perform assertions (recursively) on the value, as if it were the `$.` root +* `$skip`: skip the assertions under this test. Useful in case some tests are failing, but we want the output to keep reminding us of this fact. ### jsonpath tests