improve performance of large validation suites

packages/vest/src/core/Runtime.ts

@@ -7,9 +7,10 @@ import {
   seq,
   tinyState,
 } from 'vest-utils';
-import { IRecociler, VestRuntime } from 'vestjs-runtime';
+import { IRecociler, TIsolate, VestRuntime } from 'vestjs-runtime';
 
 import { TIsolateSuite } from 'IsolateSuite';
+import { Severity } from 'Severity';
 import {
   SuiteName,
   SuiteResult,
@@ -20,15 +21,25 @@ import {
 export type DoneCallback = (res: SuiteResult<TFieldName, TGroupName>) => void;
 type FieldCallbacks = Record<string, DoneCallbacks>;
 type DoneCallbacks = Array<DoneCallback>;
+type FailuresCache = {
+  [Severity.ERRORS]: Record<TFieldName, TIsolate[]>;
+  [Severity.WARNINGS]: Record<TFieldName, TIsolate[]>;
+};
+export type PreAggCache = {
+  pending: TIsolate[];
+  failures: FailuresCache;
+};
 
 type StateExtra = {
   doneCallbacks: TinyState<DoneCallbacks>;
   fieldCallbacks: TinyState<FieldCallbacks>;
   suiteName: Maybe<string>;
   suiteId: string;
   suiteResultCache: CacheApi<SuiteResult<TFieldName, TGroupName>>;
+  preAggCache: CacheApi<PreAggCache>;
 };
 const suiteResultCache = cache<SuiteResult<TFieldName, TGroupName>>();
+const preAggCache = cache<PreAggCache>();
 
 export function useCreateVestState({
   suiteName,
@@ -40,6 +51,7 @@ export function useCreateVestState({
   const stateRef: StateExtra = {
     doneCallbacks: tinyState.createTinyState<DoneCallbacks>(() => []),
     fieldCallbacks: tinyState.createTinyState<FieldCallbacks>(() => ({})),
+    preAggCache,
     suiteId: seq(),
     suiteName,
     suiteResultCache,
@@ -69,16 +81,27 @@ export function useSuiteId() {
 }
 
 export function useSuiteResultCache<F extends TFieldName, G extends TGroupName>(
-  action: CB<SuiteResult<F, G>>
+  action: CB<SuiteResult<F, G>>,
 ): SuiteResult<F, G> {
   const suiteResultCache = useX().suiteResultCache;
 
   return suiteResultCache([useSuiteId()], action) as SuiteResult<F, G>;
 }
 
+export function usePreAggCache(action: CB<PreAggCache>) {
+  const preAggCache = useX().preAggCache;
+
+  return preAggCache([useSuiteId()], action);
+}
+
 export function useExpireSuiteResultCache() {
   const suiteResultCache = useX().suiteResultCache;
   suiteResultCache.invalidate([useSuiteId()]);
+
+  // whenever we invalidate the entire result, we also want to invalidate the preagg cache
+  // so that we do not get stale results there.
+  // there may be a better place to do this, but for now, this should work.
+  preAggCache.invalidate([useSuiteId()]);
 }
 
 export function useResetCallbacks() {

packages/vest/src/core/VestBus/VestBus.ts

@@ -2,7 +2,7 @@ import { CB, ValueOf } from 'vest-utils';
 import { Bus, RuntimeEvents, TIsolate } from 'vestjs-runtime';
 
 import { Events } from 'BusEvents';
-import { TIsolateTest } from 'IsolateTest';
+// import { TIsolateTest } from 'IsolateTest';
 import {
   useExpireSuiteResultCache,
   useResetCallbacks,
@@ -20,39 +20,36 @@ import { useRunDoneCallbacks, useRunFieldCallbacks } from 'runCallbacks';
 export function useInitVestBus() {
   const VestBus = Bus.useBus();
 
-  // Report a the completion of a test. There may be other tests with the same
-  // name that are still running, or not yet started.
-  on(Events.TEST_COMPLETED, (testObject: TIsolateTest) => {
-    if (VestTest.isCanceled(testObject)) {
-      return;
-    }
-
-    const { fieldName } = VestTest.getData(testObject);
-
-    useRunFieldCallbacks(fieldName);
-  });
+  on(Events.TEST_COMPLETED, () => {});
+  // on(Events.TEST_RUN_STARTED, () => {});
 
-  on(Events.TEST_RUN_STARTED, () => {
-    /* Let's just invalidate the suite cache for now */
-  });
-
-  on(RuntimeEvents.ISOLATE_PENDING, (isolate: TIsolate) => {
+  VestBus.on(RuntimeEvents.ISOLATE_PENDING, (isolate: TIsolate) => {
     if (VestTest.is(isolate)) {
       VestTest.setPending(isolate);
     }
 
     VestIsolate.setPending(isolate);
   });
 
-  on(RuntimeEvents.ISOLATE_DONE, (isolate: TIsolate) => {
+  VestBus.on(RuntimeEvents.ISOLATE_DONE, (isolate: TIsolate) => {
     if (VestTest.is(isolate)) {
       VestBus.emit(Events.TEST_COMPLETED, isolate);
     }
 
     VestIsolate.setDone(isolate);
+  });
 
-    if (!SuiteWalker.hasPending()) {
-      // When no more tests are running, emit the done event
+  VestBus.on(RuntimeEvents.ASYNC_ISOLATE_DONE, (isolate: TIsolate) => {
+    if (VestTest.is(isolate)) {
+      if (!VestTest.isCanceled(isolate)) {
+        const { fieldName } = VestTest.getData(isolate);
+
+        useRunFieldCallbacks(fieldName);
+      }
+    }
+
+    if (!SuiteWalker.useHasPending()) {
+      // When no more async tests are running, emit the done event
       VestBus.emit(Events.ALL_RUNNING_TESTS_FINISHED);
     }
   });
@@ -62,7 +59,7 @@ export function useInitVestBus() {
   });
 
   // Called when all the tests, including async, are done running
-  on(Events.ALL_RUNNING_TESTS_FINISHED, () => {
+  VestBus.on(Events.ALL_RUNNING_TESTS_FINISHED, () => {
     // Small optimization. We don't need to run this if there are no async tests
     // The reason is that we run this function immediately after the suite callback
     // is run, so if the suite is only comprised of sync tests, we don't need to
@@ -82,6 +79,11 @@ export function useInitVestBus() {
   });
 
   on(Events.SUITE_CALLBACK_RUN_FINISHED, () => {
+    if (!SuiteWalker.useHasPending()) {
+      // When no more async tests are running, emit the done event
+      VestBus.emit(Events.ALL_RUNNING_TESTS_FINISHED);
+    }
+
     useOmitOptionalFields();
   });
 

packages/vest/src/core/isolate/IsolateTest/__tests__/hasRemainingTests.test.ts

@@ -3,7 +3,7 @@ import wait from 'wait';
 import { SuiteWalker } from 'SuiteWalker';
 import * as vest from 'vest';
 
-describe('SuiteWalker.hasRemainingWithTestNameMatching', () => {
+describe('SuiteWalker.useHasRemainingWithTestNameMatching', () => {
   let hasRemaining: boolean | null = null;
   let count = 0;
 
@@ -14,23 +14,20 @@ describe('SuiteWalker.hasRemainingWithTestNameMatching', () => {
   describe('When no field specified', () => {
     describe('When no remaining tests', () => {
       it('should return false', () => {
-        vest.create(() => {
-          hasRemaining = SuiteWalker.hasRemainingWithTestNameMatching();
-        })();
-        expect(hasRemaining).toBe(false);
+        const suite = vest.create(() => {})();
+        expect(suite.isPending()).toBe(false);
       });
     });
 
     describe('When there are remaining tests', () => {
       it('pending tests return true', () => {
-        vest.create(() => {
+        const suite = vest.create(() => {
           vest.test('f1', async () => {
             await wait(100);
           });
-          hasRemaining = SuiteWalker.hasRemainingWithTestNameMatching();
         })();
 
-        expect(hasRemaining).toBe(true);
+        expect(suite.isPending()).toBe(true);
       });
 
       it('lagging tests return true', () => {
@@ -40,12 +37,11 @@ describe('SuiteWalker.hasRemainingWithTestNameMatching', () => {
             await wait(100);
           });
           count++;
-          hasRemaining = SuiteWalker.hasRemainingWithTestNameMatching();
         });
         suite();
         suite();
 
-        expect(hasRemaining).toBe(true);
+        expect(suite.isPending()).toBe(true);
       });
 
       it('lagging and pending tests return true', () => {
@@ -58,36 +54,32 @@ describe('SuiteWalker.hasRemainingWithTestNameMatching', () => {
             await wait(100);
           });
           count++;
-          hasRemaining = SuiteWalker.hasRemainingWithTestNameMatching();
         });
 
         suite();
         suite();
 
-        expect(hasRemaining).toBe(true);
+        expect(suite.isPending()).toBe(true);
       });
     });
   });
 
   describe('When field specified', () => {
     describe('When no remaining tests', () => {
       it('Should return false', () => {
-        vest.create(() => {
-          hasRemaining = SuiteWalker.hasRemainingWithTestNameMatching('f1');
-        })();
-        expect(hasRemaining).toBe(false);
+        const suite = vest.create(() => {})();
+        expect(suite.isPending('f1')).toBe(false);
       });
     });
 
     describe('When remaining tests', () => {
       it('pending tests return true', () => {
-        vest.create(() => {
+        const suite = vest.create(() => {
           vest.test('f1', async () => {
             await wait(100);
           });
-          hasRemaining = SuiteWalker.hasRemainingWithTestNameMatching('f1');
         })();
-        expect(hasRemaining).toBe(true);
+        expect(suite.isPending('f1')).toBe(true);
       });
 
       it('lagging tests return true', () => {
@@ -97,12 +89,11 @@ describe('SuiteWalker.hasRemainingWithTestNameMatching', () => {
             await wait(100);
           });
           count++;
-          hasRemaining = SuiteWalker.hasRemainingWithTestNameMatching('f1');
         });
         suite();
         suite();
 
-        expect(hasRemaining).toBe(true);
+        expect(suite.isPending('f1')).toBe(true);
       });
 
       it('lagging and pending tests return true', () => {
@@ -115,15 +106,13 @@ describe('SuiteWalker.hasRemainingWithTestNameMatching', () => {
             await wait(100);
           });
           count++;
-          hasRemaining =
-            SuiteWalker.hasRemainingWithTestNameMatching('f1') &&
-            SuiteWalker.hasRemainingWithTestNameMatching('f2');
         });
 
         suite();
         suite();
 
-        expect(hasRemaining).toBe(true);
+        expect(suite.isPending('f1')).toBe(true);
+        expect(suite.isPending('f2')).toBe(true);
       });
     });
   });

packages/vest/src/core/test/testLevelFlowControl/runTest.ts

@@ -24,7 +24,7 @@ export function useAttemptRunTest(testObject: TIsolateTest) {
     deferThrow(
       text(ErrorStrings.UNEXPECTED_TEST_REGISTRATION_ERROR, {
         testObject: JSON.stringify(testObject),
-      })
+      }),
     );
   }
 }
@@ -77,7 +77,7 @@ function useRunTest(testObject: TIsolateTest): Promise<void> | undefined {
       text(ErrorStrings.UNEXPECTED_TEST_REGISTRATION_ERROR, {
         testObject: JSON.stringify(testObject),
         error: e,
-      })
+      }),
     );
   }
 }