[@typespec/spector] - Handle and report failed scenarios at the end (#…

…4872)

In the `server-test` scenario, if there is an error, then the script
immediately reports the error and comes out of the execution. But, the
service generator has not implemented all the scenarios yet. So, If all
the scenarios are executed, then the script might fail even before
executing the intended script. So, the correct approach would be to
report at the end.

Please review and approve the PR. Thanks

.chronus/changes/FormatErrorOutput-2024-9-25-14-43-7.md

@@ -0,0 +1,7 @@
+---
+changeKind: internal
+packages:
+  - "@typespec/spector"
+---
+
+Handle Failing scenarios at the end

packages/spector/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@typespec/spector",
-  "version": "0.1.0-alpha.0",
+  "version": "0.1.0-alpha.1",
   "description": "Typespec Core Tool to validate, run mock api, collect coverage.",
   "exports": {
     ".": {

packages/spector/src/actions/server-test.ts

@@ -1,12 +1,19 @@
 import { MockApiDefinition } from "@typespec/spec-api";
 import * as fs from "fs";
 import * as path from "path";
+import pc from "picocolors";
 import { logger } from "../logger.js";
 import { loadScenarioMockApis } from "../scenarios-resolver.js";
 import { makeServiceCall, uint8ArrayToString } from "./helper.js";
 
 const DEFAULT_BASE_URL = "http://localhost:3000";
 
+export interface ServerTestDiagnostics {
+  scenario_name: string;
+  status: "success" | "failure";
+  message: any;
+}
+
 class ServerTestsGenerator {
   private name: string = "";
   private mockApiDefinition: MockApiDefinition;
@@ -184,15 +191,49 @@ export async function serverTest(scenariosPath: string, options: ServerTestOptio
   }
   // 2. Load all the scenarios
   const scenarios = await loadScenarioMockApis(scenariosPath);
+  const success_diagnostics: ServerTestDiagnostics[] = [];
+  const failure_diagnostics: ServerTestDiagnostics[] = [];
+
   // 3. Execute each scenario
   for (const [name, scenario] of Object.entries(scenarios)) {
     if (!Array.isArray(scenario.apis)) continue;
     for (const api of scenario.apis) {
       if (api.kind !== "MockApiDefinition") continue;
       if (testCasesToRun.length === 0 || testCasesToRun.includes(name)) {
         const obj: ServerTestsGenerator = new ServerTestsGenerator(name, api, baseUrl);
-        await obj.executeScenario();
+        try {
+          await obj.executeScenario();
+          success_diagnostics.push({
+            scenario_name: name,
+            status: "success",
+            message: "executed successfully",
+          });
+        } catch (e: any) {
+          failure_diagnostics.push({
+            scenario_name: name,
+            status: "failure",
+            message: `code = ${e.code} \n message = ${e.message} \n name = ${e.name} \n stack = ${e.stack} \n status = ${e.status}`,
+          });
+        }
       }
     }
   }
+
+  // 4. Print diagnostics
+  logger.info("Server Tests Diagnostics Summary");
+
+  if (success_diagnostics.length > 0) logger.info("Success Scenarios");
+  success_diagnostics.forEach((diagnostic) => {
+    logger.info(`${pc.green("✓")} Scenario: ${diagnostic.scenario_name} - ${diagnostic.message}`);
+  });
+
+  if (failure_diagnostics.length > 0) logger.error("Failure Scenarios");
+  if (failure_diagnostics.length > 0) {
+    logger.error("Failed Scenario details");
+    failure_diagnostics.forEach((diagnostic) => {
+      logger.error(`${pc.red("✘")} Scenario: ${diagnostic.scenario_name}`);
+      logger.error(`${diagnostic.message}`);
+    });
+    process.exit(-1);
+  }
 }