Merge pull request #8 from agrostar/vsc#7_skip-tests

skip tests of a spec when the $skip clause is set to a truthy value

docs/zzapi-bundle-description.md

@@ -90,6 +90,7 @@ These are options that can be switches on/off, both at the common level as well
 * `follow`: whether to follow redirects (default is false)
 * `verifySSL`: whether to enfoce SSL certificate validation (default is false)
 * `showHeaders`: whether to show the response headers for each request (default is false)
+* `rawParams`: whether to consider the params as is (default is false, ie use URL encoding)
 * `keepRawJSON`: whether to keep the original response JSON without formatting it. (default is false)
 * `stopOnFailure`: whether to skip other tests when any of the status tests fail.
 
@@ -171,6 +172,7 @@ Operators supported in the RHS are:
 * `$exists`: true|false, to check existance of a field
 * `$type`: string|number|object|array|null: to check the type of the field
 * `$tests`: perform assertions (recursively) on the value, as if it were the `$.` root
+* `$skip`: skip the assertions under this test. Useful in case some tests are failing, but we want the output to keep reminding us of this fact.
 
 ### jsonpath tests
 

examples/tests-bundle.zzb

@@ -67,8 +67,8 @@ requests:
         - multi-1
         - multi-2
     tests:
-      $.args.foo1: bar1
-      $.args.foo2: { $eq: ["multi-1", "multi-2"] }
+      $.args.foo1: bar
+      $.args.foo2: { $eq: ["multi-1", "multi-2"], $skip: true }
 
   post-header-merge:
     url: /post
@@ -232,7 +232,7 @@ requests:
           number: 0123-4567-8910
           available: [18, 22]
     tests:
-      $.data.firstName: John
+      $.data.firstName: Joh
       $.data.age: { $type: number, $eq: 26, $ne: 30, $gt: 25, $lt: 28 }
       $.data.address: { $type: object, $size: 3, $exists: true }
       $.data.address.city: Nara
@@ -246,8 +246,9 @@ requests:
       $.data.phoneNumbers.0.available: { $eq: [7, 22], $type: array }
       $.data.phoneNumbers.1.available: { $eq: "[18,22]", $type: array }
       $.data.phoneNumbers.0:
+        $skip: true
         $tests:
-          $.type: mobiles
+          $.type: mobile
           $.number: { $ne: 0123-4567-8910 }
           $.available: { $eq: [7, 22] }
       $.data.lastName: { $exists: true }

schemas/zzapi-bundle.schema.json

@@ -131,7 +131,8 @@
                 }                        
               ]
             },
-            "$tests": { "$ref": "#/$defs/tests" }
+            "$tests": { "$ref": "#/$defs/tests" },
+            "$skip": { "type": "boolean" }
           }
         }
       ]

src/models.ts

@@ -133,6 +133,7 @@ export interface TestResult {
 
 export interface SpecResult {
   spec: string | null;
+  skipped?: boolean;
   results: TestResult[];
   subResults: SpecResult[];
 }

src/runTests.ts

@@ -10,22 +10,23 @@ export function runAllTests(
   responseData: ResponseData,
   stopOnFailure: boolean,
   rootSpec: string | null = null,
+  skip?: boolean,
 ): SpecResult {
   const res: SpecResult = { spec: rootSpec, results: [], subResults: [] };
   if (!tests) return res;
 
   if (tests.status) {
     const expected = tests.status;
     const received = responseData.status;
-    const statusResults = runTest("status", expected, received);
+    const statusResults = runTest("status", expected, received, skip);
 
     if (stopOnFailure && statusResults.results.some((r) => !r.pass)) return res;
   }
 
   for (const spec in tests.headers) {
     const expected = tests.headers[spec];
     const received = responseData.headers ? responseData.headers[spec] : "";
-    const headerResults = runTest(spec, expected, received);
+    const headerResults = runTest(spec, expected, received, skip);
 
     res.subResults.push(headerResults);
   }
@@ -38,36 +39,38 @@ export function runAllTests(
     } catch (err: any) {
       res.subResults.push({
         spec,
+        skipped: skip || (typeof expected === "object" && expected !== null && expected["$skip"]),
         results: [{ pass: false, expected, received: "", op: spec, message: err }],
         subResults: [],
       });
       continue;
     }
 
-    const jsonResults = runTest(spec, expected, received);
+    const jsonResults = runTest(spec, expected, received, skip);
     res.subResults.push(jsonResults);
   }
 
   if (tests.body) {
     const expected = tests.body;
     const received = responseData.body;
-    const bodyResults = runTest("body", expected, received);
+    const bodyResults = runTest("body", expected, received, skip);
 
     res.subResults.push(bodyResults);
   }
 
   return res;
 }
 
-function runTest(spec: string, expected: Assertion, received: any): SpecResult {
+function runTest(spec: string, expected: Assertion, received: any, skip?: boolean): SpecResult {
   // typeof null is also 'object'
-  if (expected !== null && typeof expected === "object") return runObjectTests(expected, received, spec);
+  if (expected !== null && typeof expected === "object")
+    return runObjectTests(expected, received, spec, skip);
 
   expected = getStringIfNotScalar(expected);
   received = getStringIfNotScalar(received);
   const pass = expected === received;
 
-  return { spec, results: [{ pass, expected, received, op: ":" }], subResults: [] };
+  return { spec, skipped: skip, results: [{ pass, expected, received, op: ":" }], subResults: [] };
 }
 
 function getValueForJSONTests(responseContent: object, key: string): any {
@@ -78,8 +81,14 @@ function getValueForJSONTests(responseContent: object, key: string): any {
   }
 }
 
-function runObjectTests(opVals: { [key: string]: any }, receivedObject: any, spec: string): SpecResult {
+function runObjectTests(
+  opVals: { [key: string]: any },
+  receivedObject: any,
+  spec: string,
+  skip?: boolean,
+): SpecResult {
   let objRes: SpecResult = { spec, results: [], subResults: [] };
+  if (skip || opVals["$skip"]) objRes.skipped = true;
 
   for (const op in opVals) {
     let expected = getStringIfNotScalar(opVals[op]);
@@ -162,11 +171,13 @@ function runObjectTests(opVals: { [key: string]: any }, receivedObject: any, spe
         };
 
         // the spec remains the same, so we add it to the current layer
-        const res = runAllTests(recursiveTests, receivedObj, false, spec);
+        const res = runAllTests(recursiveTests, receivedObj, false, spec, objRes.skipped);
         objRes.results.push(...res.results);
         objRes.subResults.push(...res.subResults);
         continue;
       }
+    } else if (op === "$skip") {
+      continue; // do nothing. If it wasn't already addressed, that means the test is not to be skipped.
     } else {
       objRes.results.push({
         pass: false,