diff --git a/.chronus/changes/FormatErrorOutput-2024-9-25-14-43-7.md b/.chronus/changes/FormatErrorOutput-2024-9-25-14-43-7.md new file mode 100644 index 0000000000..bbde55a822 --- /dev/null +++ b/.chronus/changes/FormatErrorOutput-2024-9-25-14-43-7.md @@ -0,0 +1,7 @@ +--- +changeKind: internal +packages: + - "@typespec/spector" +--- + +Handle Failing scenarios at the end \ No newline at end of file diff --git a/packages/spector/package.json b/packages/spector/package.json index b0fe27e99f..947bc69f60 100644 --- a/packages/spector/package.json +++ b/packages/spector/package.json @@ -1,6 +1,6 @@ { "name": "@typespec/spector", - "version": "0.1.0-alpha.0", + "version": "0.1.0-alpha.1", "description": "Typespec Core Tool to validate, run mock api, collect coverage.", "exports": { ".": { diff --git a/packages/spector/src/actions/server-test.ts b/packages/spector/src/actions/server-test.ts index c22e6b83b4..c023c8f1b1 100644 --- a/packages/spector/src/actions/server-test.ts +++ b/packages/spector/src/actions/server-test.ts @@ -1,12 +1,19 @@ import { MockApiDefinition } from "@typespec/spec-api"; import * as fs from "fs"; import * as path from "path"; +import pc from "picocolors"; import { logger } from "../logger.js"; import { loadScenarioMockApis } from "../scenarios-resolver.js"; import { makeServiceCall, uint8ArrayToString } from "./helper.js"; const DEFAULT_BASE_URL = "http://localhost:3000"; +export interface ServerTestDiagnostics { + scenario_name: string; + status: "success" | "failure"; + message: any; +} + class ServerTestsGenerator { private name: string = ""; private mockApiDefinition: MockApiDefinition; @@ -184,6 +191,9 @@ export async function serverTest(scenariosPath: string, options: ServerTestOptio } // 2. Load all the scenarios const scenarios = await loadScenarioMockApis(scenariosPath); + const success_diagnostics: ServerTestDiagnostics[] = []; + const failure_diagnostics: ServerTestDiagnostics[] = []; + // 3. Execute each scenario for (const [name, scenario] of Object.entries(scenarios)) { if (!Array.isArray(scenario.apis)) continue; @@ -191,8 +201,39 @@ export async function serverTest(scenariosPath: string, options: ServerTestOptio if (api.kind !== "MockApiDefinition") continue; if (testCasesToRun.length === 0 || testCasesToRun.includes(name)) { const obj: ServerTestsGenerator = new ServerTestsGenerator(name, api, baseUrl); - await obj.executeScenario(); + try { + await obj.executeScenario(); + success_diagnostics.push({ + scenario_name: name, + status: "success", + message: "executed successfully", + }); + } catch (e: any) { + failure_diagnostics.push({ + scenario_name: name, + status: "failure", + message: `code = ${e.code} \n message = ${e.message} \n name = ${e.name} \n stack = ${e.stack} \n status = ${e.status}`, + }); + } } } } + + // 4. Print diagnostics + logger.info("Server Tests Diagnostics Summary"); + + if (success_diagnostics.length > 0) logger.info("Success Scenarios"); + success_diagnostics.forEach((diagnostic) => { + logger.info(`${pc.green("✓")} Scenario: ${diagnostic.scenario_name} - ${diagnostic.message}`); + }); + + if (failure_diagnostics.length > 0) logger.error("Failure Scenarios"); + if (failure_diagnostics.length > 0) { + logger.error("Failed Scenario details"); + failure_diagnostics.forEach((diagnostic) => { + logger.error(`${pc.red("✘")} Scenario: ${diagnostic.scenario_name}`); + logger.error(`${diagnostic.message}`); + }); + process.exit(-1); + } }