Skip to content

Commit

Permalink
Add "Run Test Multiple Times" (#1009)
Browse files Browse the repository at this point in the history 
Adds two new items to the context menu when you right click a test in
the test explorer:

- Run Multiple Times
- Run Until Failure

Selecting either of these promts with a text input where the user can
input how many times they want to run the test(s).

If the user selected Run Until Failure the tests will be run a maximum
number of times, stopping at the first iteration that produces a
failure.

A current limitation in VS Code is if you have multiple tests selected
in the Test Explorer (by shift or ctrl clicking them) only the first is
passed to the command.

Issue: #832
  • Loading branch information
@plemarquand
plemarquand authored Aug 15, 2024
1 parent e0a0fba commit 0747917
Show file tree
Hide file tree
Showing 5 changed files with 219 additions and 28 deletions.
22 changes: 22 additions & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,16 @@
"command": "swift.clearDiagnosticsCollection",
"title": "Clear Diagnostics Collection",
"category": "Swift"
},
{
"command": "swift.runTestsMultipleTimes",
"title": "Run Multiple Times...",
"category": "Swift"
},
{
"command": "swift.runTestsUntilFailure",
"title": "Run Until Failure...",
"category": "Swift"
}
],
"configuration": [
Expand Down Expand Up @@ -598,6 +608,18 @@
}
],
"menus": {
"testing/item/context": [
{
"command": "swift.runTestsMultipleTimes",
"group": "testExtras",
"when": "testId"
},
{
"command": "swift.runTestsUntilFailure",
"group": "testExtras",
"when": "testId"
}
],
"commandPalette": [
{
"command": "swift.createNewProject",
Expand Down
2 changes: 1 addition & 1 deletion src/TestExplorer/TestExplorer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ export class TestExplorer {
private onTestItemsDidChangeEmitter = new vscode.EventEmitter<vscode.TestController>();
public onTestItemsDidChange: vscode.Event<vscode.TestController>;

private onDidCreateTestRunEmitter = new vscode.EventEmitter<TestRunProxy>();
public onDidCreateTestRunEmitter = new vscode.EventEmitter<TestRunProxy>();
public onCreateTestRun: vscode.Event<TestRunProxy>;

constructor(public folderContext: FolderContext) {
Expand Down
76 changes: 59 additions & 17 deletions src/TestExplorer/TestRunner.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,20 +52,28 @@ export class TestRunProxy {
private runStarted: boolean = false;
private queuedOutput: string[] = [];
private _testItems: vscode.TestItem[];
private iteration: number | undefined;
public coverage: TestCoverage;

public testRunCompleteEmitter = new vscode.EventEmitter<void>();
public onTestRunComplete: vscode.Event<void>;

// Allows for introspection on the state of TestItems after a test run.
public runState = {
failed: [] as {
test: vscode.TestItem;
message: vscode.TestMessage | readonly vscode.TestMessage[];
}[],
passed: [] as vscode.TestItem[],
skipped: [] as vscode.TestItem[],
errored: [] as vscode.TestItem[],
unknown: 0,
output: [] as string[],
};
public runState = TestRunProxy.initialTestRunState();

private static initialTestRunState() {
return {
failed: [] as {
test: vscode.TestItem;
message: vscode.TestMessage | readonly vscode.TestMessage[];
}[],
passed: [] as vscode.TestItem[],
skipped: [] as vscode.TestItem[],
errored: [] as vscode.TestItem[],
unknown: 0,
output: [] as string[],
};
}

public get testItems(): vscode.TestItem[] {
return this._testItems;
Expand All @@ -79,6 +87,7 @@ export class TestRunProxy {
) {
this._testItems = args.testItems;
this.coverage = new TestCoverage(folderContext);
this.onTestRunComplete = this.testRunCompleteEmitter.event;
}

public testRunStarted = () => {
Expand Down Expand Up @@ -194,20 +203,37 @@ export class TestRunProxy {

public async end() {
this.testRun?.end();
this.testRunCompleteEmitter.fire();
}

public setIteration(iteration: number) {
this.runState = TestRunProxy.initialTestRunState();
this.iteration = iteration;
}

public appendOutput(output: string) {
const tranformedOutput = this.prependIterationToOutput(output);
if (this.testRun) {
this.testRun.appendOutput(output);
this.runState.output.push(output);
this.testRun.appendOutput(tranformedOutput);
this.runState.output.push(tranformedOutput);
} else {
this.queuedOutput.push(output);
this.queuedOutput.push(tranformedOutput);
}
}

public appendOutputToTest(output: string, test: vscode.TestItem, location?: vscode.Location) {
this.testRun?.appendOutput(output, location, test);
this.runState.output.push(output);
const tranformedOutput = this.prependIterationToOutput(output);
this.testRun?.appendOutput(tranformedOutput, location, test);
this.runState.output.push(tranformedOutput);
}

private prependIterationToOutput(output: string): string {
if (this.iteration === undefined) {
return output;
}
const itr = this.iteration + 1;
const lines = output.match(/[^\r\n]*[\r\n]*/g);
return lines?.map(line => (line ? `\x1b[34mRun ${itr}\x1b[0m ${line}` : "")).join("") ?? "";
}

public async computeCoverage() {
Expand All @@ -222,7 +248,7 @@ export class TestRunProxy {

/** Class used to run tests */
export class TestRunner {
private testRun: TestRunProxy;
public testRun: TestRunProxy;
private testArgs: TestRunArguments;
private xcTestOutputParser: IXCTestOutputParser;
private swiftTestOutputParser: SwiftTestingOutputParser;
Expand Down Expand Up @@ -256,6 +282,20 @@ export class TestRunner {
);
}

/**
* When performing a "Run test multiple times" run set the iteration
* so it can be shown in the logs.
* @param iteration The iteration counter
*/
public setIteration(iteration: number) {
// The SwiftTestingOutputParser holds state and needs to be reset between iterations.
this.swiftTestOutputParser = new SwiftTestingOutputParser(
this.testRun.testRunStarted,
this.testRun.addParameterizedTestCase
);
this.testRun.setIteration(iteration);
}

/**
* If the request has no test items to include in the run,
* default to usig all the items in the `TestController`.
Expand Down Expand Up @@ -587,6 +627,7 @@ export class TestRunner {
task.execution.onDidWrite(str => {
const replaced = str
.replace("[1/1] Planning build", "") // Work around SPM still emitting progress when doing --no-build.
.replace(/\[1\/1\] Write swift-version-.*/gm, "")
.replace(
/LLVM Profile Error: Failed to write file "default.profraw": Operation not permitted\r\n/gm,
""
Expand Down Expand Up @@ -1023,6 +1064,7 @@ export class TestRunnerTestRunState implements ITestRunState {
return;
}
const testItem = this.testRun.testItems[index];
this.issues.delete(index);
this.testRun.started(testItem);
this.currentTestItem = testItem;
this.startTimes.set(index, startTime);
Expand Down
56 changes: 56 additions & 0 deletions src/commands.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ import { SwiftProjectTemplate } from "./toolchain/toolchain";
import { showToolchainSelectionQuickPick, showToolchainError } from "./ui/ToolchainSelection";
import { captureDiagnostics } from "./commands/captureDiagnostics";
import { reindexProjectRequest } from "./sourcekit-lsp/lspExtensions";
import { TestRunner, TestRunnerTestRunState } from "./TestExplorer/TestRunner";
import { TestKind } from "./TestExplorer/TestKind";

/**
* References:
Expand Down Expand Up @@ -734,6 +736,54 @@ function openInExternalEditor(packageNode: PackageNode) {
}
}

async function runTestMultipleTimes(
ctx: WorkspaceContext,
test: vscode.TestItem,
untilFailure: boolean
) {
const str = await vscode.window.showInputBox({
prompt: "Label: ",
placeHolder: `${untilFailure ? "Maximum " : ""}# of times to run`,
validateInput: value => (/^[1-9]\d*$/.test(value) ? undefined : "Enter an integer value"),
});

if (!str || !ctx.currentFolder?.testExplorer) {
return;
}

const numExecutions = parseInt(str);
const testExplorer = ctx.currentFolder.testExplorer;
const runner = new TestRunner(
TestKind.standard,
new vscode.TestRunRequest([test]),
ctx.currentFolder,
testExplorer.controller
);

testExplorer.onDidCreateTestRunEmitter.fire(runner.testRun);

const testRunState = new TestRunnerTestRunState(runner.testRun);
const token = new vscode.CancellationTokenSource();

vscode.commands.executeCommand("workbench.panel.testResults.view.focus");

for (let i = 0; i < numExecutions; i++) {
runner.setIteration(i);
runner.testRun.appendOutput(`\x1b[36mBeginning Test Iteration #${i + 1}\x1b[0m\n`);

await runner.runSession(token.token, testRunState);

if (
untilFailure &&
(runner.testRun.runState.failed.length > 0 ||
runner.testRun.runState.errored.length > 0)
) {
break;
}
}
runner.testRun.end();
}

/**
* Switches the target SDK to the platform selected in a QuickPick UI.
*/
Expand Down Expand Up @@ -841,6 +891,12 @@ export function register(ctx: WorkspaceContext): vscode.Disposable[] {
vscode.commands.registerCommand("swift.run", () => runBuild(ctx)),
vscode.commands.registerCommand("swift.debug", () => debugBuild(ctx)),
vscode.commands.registerCommand("swift.cleanBuild", () => cleanBuild(ctx)),
vscode.commands.registerCommand("swift.runTestsMultipleTimes", item =>
runTestMultipleTimes(ctx, item, false)
),
vscode.commands.registerCommand("swift.runTestsUntilFailure", item =>
runTestMultipleTimes(ctx, item, true)
),
// Note: This is only available on macOS (gated in `package.json`) because its the only OS that has the iOS SDK available.
vscode.commands.registerCommand("swift.switchPlatform", () => switchPlatform()),
vscode.commands.registerCommand("swift.resetPackage", () => resetPackage(ctx)),
Expand Down
91 changes: 81 additions & 10 deletions test/suite/testexplorer/TestExplorerIntegration.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import * as vscode from "vscode";
import * as assert from "assert";
import { beforeEach } from "mocha";
import { when, anything } from "ts-mockito";
import { testAssetUri } from "../../fixtures";
import { globalWorkspaceContextPromise } from "../extension.test";
import { TestExplorer } from "../../../src/TestExplorer/TestExplorer";
Expand All @@ -29,6 +30,7 @@ import { WorkspaceContext } from "../../../src/WorkspaceContext";
import { TestRunProxy } from "../../../src/TestExplorer/TestRunner";
import { Version } from "../../../src/utilities/version";
import { TestKind } from "../../../src/TestExplorer/TestKind";
import { mockNamespace } from "../../unit-tests/MockUtils";

suite("Test Explorer Suite", function () {
const MAX_TEST_RUN_TIME_MINUTES = 5;
Expand All @@ -49,18 +51,10 @@ suite("Test Explorer Suite", function () {
)[0];
}

async function runTest(
async function gatherTests(
controller: vscode.TestController,
runProfile: TestKind,
...tests: string[]
): Promise<TestRunProxy> {
const targetProfile = testExplorer.testRunProfiles.find(
profile => profile.label === runProfile
);
if (!targetProfile) {
throw new Error(`Unable to find run profile named ${runProfile}`);
}

): Promise<vscode.TestItem[]> {
const testItems = tests.map(test => {
const testItem = getTestItem(controller, test);
if (!testItem) {
Expand All @@ -79,6 +73,21 @@ suite("Test Explorer Suite", function () {
return testItem;
});

return testItems;
}

async function runTest(
controller: vscode.TestController,
runProfile: TestKind,
...tests: string[]
): Promise<TestRunProxy> {
const targetProfile = testExplorer.testRunProfiles.find(
profile => profile.label === runProfile
);
if (!targetProfile) {
throw new Error(`Unable to find run profile named ${runProfile}`);
}
const testItems = await gatherTests(controller, ...tests);
const request = new vscode.TestRunRequest(testItems);

return (
Expand Down Expand Up @@ -233,6 +242,37 @@ suite("Test Explorer Suite", function () {
],
});
});

suite("Runs multiple", function () {
const numIterations = 5;
const windowMock = mockNamespace(vscode, "window");

test("@slow runs an swift-testing test multiple times", async function () {
const testItems = await gatherTests(
testExplorer.controller,
"PackageTests.MixedXCTestSuite/testPassing"
);

await testExplorer.folderContext.workspaceContext.focusFolder(
testExplorer.folderContext
);

// Stub the showInputBox method to return the input text
when(windowMock.showInputBox(anything())).thenReturn(
Promise.resolve(`${numIterations}`)
);

vscode.commands.executeCommand("swift.runTestsMultipleTimes", testItems[0]);

const testRun = await eventPromise(testExplorer.onCreateTestRun);

await eventPromise(testRun.onTestRunComplete);

assertTestResults(testRun, {
passed: ["PackageTests.MixedXCTestSuite/testPassing"],
});
});
});
});

suite("XCTest", () => {
Expand Down Expand Up @@ -262,6 +302,37 @@ suite("Test Explorer Suite", function () {
passed: ["PackageTests.DebugReleaseTestSuite/testRelease"],
});
});

suite("Runs multiple", function () {
const numIterations = 5;
const windowMock = mockNamespace(vscode, "window");

test("@slow runs an XCTest multiple times", async function () {
const testItems = await gatherTests(
testExplorer.controller,
"PackageTests.topLevelTestPassing()"
);

await testExplorer.folderContext.workspaceContext.focusFolder(
testExplorer.folderContext
);

// Stub the showInputBox method to return the input text
when(windowMock.showInputBox(anything())).thenReturn(
Promise.resolve(`${numIterations}`)
);

vscode.commands.executeCommand("swift.runTestsMultipleTimes", testItems[0]);

const testRun = await eventPromise(testExplorer.onCreateTestRun);

await eventPromise(testRun.onTestRunComplete);

assertTestResults(testRun, {
passed: ["PackageTests.topLevelTestPassing()"],
});
});
});
});

// Do coverage last as it does a full rebuild, causing the stage after it to have to rebuild as well.
Expand Down

0 comments on commit 0747917

Please sign in to comment.