Merge branch 'master' of https://github.com/IDEMSInternational/parent…

…ing-app-ui into chore/default-node-20

cspell.config.yml

@@ -20,6 +20,7 @@
     "linenums",
     "lottie",
     "matomo",
+    "memfs",
     "metabase",
     "mkdocs",
     "mycompany",

packages/scripts/.eslintrc.json

@@ -2,5 +2,14 @@
   "parserOptions": {
     "sourceType": "module",
     "ecmaVersion": "latest"
-  }
+  },
+  "plugins": ["jest"],
+  "overrides": [
+    {
+      "files": ["**/*.spec.ts"],
+      "plugins": ["jest"],
+      "extends": ["plugin:jest/recommended"],
+      "rules": {}
+    }
+  ]
 }

packages/scripts/jest.config.js

@@ -0,0 +1,9 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} **/
+module.exports = {
+  testEnvironment: "node",
+  transform: {
+    "^.+.tsx?$": ["ts-jest", {}],
+  },
+  globalSetup: "<rootDir>/test/setup.ts",
+};
+

packages/scripts/package.json

@@ -11,8 +11,8 @@
     "start": "ts-node src/commands/index.ts",
     "build": "ts-node build.ts",
     "dev": "ts-node-dev --transpile-only --respawn --watch src src/commands/index.ts",
-    "test": "jasmine-ts --config=jasmine.json",
-    "test:watch": "nodemon --ext ts --exec 'jasmine-ts --config=jasmine.json'"
+    "test": "jest --runInBand --",
+    "test:watch": "jest --runInBand --watchAll --"
   },
   "author": "",
   "license": "ISC",
@@ -47,14 +47,13 @@
     "@swc/core": "^1.3.29",
     "@types/fs-extra": "^9.0.4",
     "@types/inquirer": "^7.3.1",
-    "@types/jasmine": "^3.10.6",
+    "@types/jest": "^29.5.12",
     "@types/node-rsa": "^1.1.1",
     "@types/semver": "^7.3.9",
-    "jasmine": "^3.99.0",
-    "jasmine-spec-reporter": "^7.0.0",
-    "jasmine-ts": "^0.4.0",
-    "mock-fs": "^5.2.0",
-    "nodemon": "^2.0.19",
+    "eslint-plugin-jest": "^28.8.0",
+    "jest": "^29.7.0",
+    "memfs": "^4.11.1",
+    "ts-jest": "^29.2.5",
     "ts-node": "^10.8.0",
     "ts-node-dev": "^2.0.0",
     "tsup": "^7.2.0"

packages/scripts/src/commands/app-data/convert/cacheStrategy/jsonFile.spec.ts

@@ -89,11 +89,11 @@ describe("Json File Cache", () => {
     // Add entry
     const data = Math.random();
     const { entryName, filePath } = cache.add(data);
-    expect(existsSync(filePath)).toBeTrue();
+    expect(existsSync(filePath)).toEqual(true);
     expect(cache.get(entryName)).toEqual(data);
     // Remove entry
     cache.remove(entryName);
-    expect(existsSync(filePath)).toBeFalse();
+    expect(existsSync(filePath)).toEqual(false);
     expect(cache.get(entryName)).toBeUndefined();
   });
 

packages/scripts/src/commands/app-data/convert/convert.spec.ts

@@ -1,46 +1,58 @@
 import { AppDataConverter } from "./index";
 
-import path, { resolve } from "path";
+import { resolve } from "path";
 
-import { SCRIPTS_TEST_DATA_DIR } from "../../../paths";
-import { emptyDirSync, existsSync, readdirSync, readJSONSync } from "fs-extra";
-import { clearLogs, getLogs } from "./utils";
-import { useMockErrorLogger } from "../../../../test/helpers/utils";
+import { emptyDirSync, existsSync, readdirSync, readJSONSync, ensureDirSync } from "fs-extra";
+import { clearLogs } from "shared";
 
-// Folders used for tests
+import { TEST_DATA_PATHS } from "../../../../test/helpers/utils";
+import { ActiveDeployment } from "../../deployment/get";
+
+const { SHEETS_CACHE_FOLDER, SHEETS_INPUT_FOLDER, SHEETS_OUTPUT_FOLDER } = TEST_DATA_PATHS;
 const paths = {
-  inputFolders: [path.resolve(SCRIPTS_TEST_DATA_DIR, "input", "sheets")],
-  outputFolder: path.resolve(SCRIPTS_TEST_DATA_DIR, "output", "sheets"),
-  cacheFolder: path.resolve(SCRIPTS_TEST_DATA_DIR, "cache"),
+  inputFolders: [resolve(SHEETS_INPUT_FOLDER, "sheets")],
+  outputFolder: resolve(SHEETS_OUTPUT_FOLDER, "sheets"),
+  cacheFolder: resolve(SHEETS_CACHE_FOLDER),
 };
 
+// HACK - avoid loading active deployment
+jest.spyOn(ActiveDeployment, "get").mockReturnValue({
+  app_data: {
+    sheets_filter_function: () => true,
+    assets_filter_function: () => true,
+    output_path: paths.outputFolder,
+  },
+} as any);
+
+/** yarn workspace scripts test -t convert.spec.ts */
 describe("App Data Converter", () => {
   let converter: AppDataConverter;
   beforeAll(() => {
-    if (existsSync(paths.outputFolder)) {
-      path.resolve(SCRIPTS_TEST_DATA_DIR, "output");
-    }
+    ensureDirSync(paths.outputFolder);
+    emptyDirSync(paths.outputFolder);
+    ensureDirSync(paths.cacheFolder);
+    emptyDirSync(paths.cacheFolder);
   });
   beforeEach(() => {
-    clearLogs();
     converter = new AppDataConverter(paths);
   });
-  afterAll(() => {
-    emptyDirSync(path.resolve(SCRIPTS_TEST_DATA_DIR, "output"));
-  });
+
   it("Uses child caches", async () => {
+    await converter.run();
     const cacheFolders = readdirSync(paths.cacheFolder);
+    // expect contents file and cached conversions
     expect(cacheFolders.length).toBeGreaterThan(1);
   });
   it("Clears child caches on version change", async () => {
-    const updatedConverter = new AppDataConverter(paths, { version: -1 });
+    const updatedConverter = new AppDataConverter(paths, { version: new Date().getTime() });
+    // no need to run the converter, simply creating should clear the cache
     const cacheFolders = readdirSync(paths.cacheFolder);
-    expect(cacheFolders.length).toEqual(1); // only contents file
+    expect(cacheFolders).toHaveLength(1); // only contents file
   });
   it("Processes test_input xlsx without error", async () => {
-    await converter.run();
-    const errorLogs = getLogs("error");
-    expect(errorLogs.length).toEqual(0);
+    const { errors, result } = await converter.run();
+    expect(errors).toHaveLength(0);
+    expect(Object.values(result).length).toBeGreaterThan(0);
   });
   it("Populates output to folder by data type", async () => {
     await converter.run();
@@ -50,10 +62,7 @@ describe("App Data Converter", () => {
   it("Supports input from multiple source folders", async () => {
     const multipleSourceConverter = new AppDataConverter({
       ...paths,
-      inputFolders: [
-        ...paths.inputFolders,
-        path.resolve(SCRIPTS_TEST_DATA_DIR, "input", "sheets_additional"),
-      ],
+      inputFolders: [...paths.inputFolders, resolve(SHEETS_INPUT_FOLDER, "sheets_additional")],
     });
     await multipleSourceConverter.run();
     const replaceDataListPath = resolve(
@@ -70,26 +79,24 @@ describe("App Data Converter", () => {
 
 // Folders used for error tests
 const errorPaths = {
-  inputFolders: [path.resolve(SCRIPTS_TEST_DATA_DIR, "input", "errorChecking")],
-  outputFolder: path.resolve(SCRIPTS_TEST_DATA_DIR, "output", "errorChecking"),
-  cacheFolder: path.resolve(SCRIPTS_TEST_DATA_DIR, "cache"),
+  inputFolders: [resolve(SHEETS_INPUT_FOLDER, "errorChecking")],
+  outputFolder: resolve(SHEETS_OUTPUT_FOLDER, "errorChecking"),
+  cacheFolder: resolve(SHEETS_CACHE_FOLDER),
 };
 describe("App Data Converter - Error Checking", () => {
   let errorConverter: AppDataConverter;
-  let errorLogger: jasmine.Spy;
   beforeAll(() => {
     if (existsSync(paths.outputFolder)) {
       emptyDirSync(paths.outputFolder);
     }
   });
   beforeEach(() => {
-    errorLogger = useMockErrorLogger();
     errorConverter = new AppDataConverter(errorPaths);
   });
   it("Tracks conversion errors", async () => {
-    await errorConverter.run();
-    const loggerErrors = getLogs("error");
-    const errorMessages = loggerErrors.map((err) => err.message);
+    clearLogs(true);
+    const { errors } = await errorConverter.run();
+    const errorMessages = errors.map((err) => err.message);
     expect(errorMessages).toEqual([
       "Duplicate flow name",
       "No parser available for flow_type: test_invalid_type",

packages/scripts/src/commands/app-data/convert/index.ts

@@ -69,7 +69,10 @@ export class AppDataConverter {
 
   cache: JsonFileCache;
 
-  constructor(private options: IConverterOptions, testOverrides: Partial<AppDataConverter> = {}) {
+  constructor(
+    private options: IConverterOptions,
+    testOverrides: Partial<AppDataConverter> = {}
+  ) {
     console.log(chalk.yellow("App Data Convert"));
     // optional overrides, used for tests
     if (testOverrides.version) this.version = testOverrides.version;

packages/scripts/src/commands/app-data/convert/processors/base.spec.ts

@@ -1,14 +1,7 @@
-import path from "path";
 import BaseProcessor from "./base";
 
-import { SCRIPTS_WORKSPACE_PATH } from "../../../../paths";
 import { clearLogs, getLogs } from "../utils";
-const testDataDir = path.resolve(SCRIPTS_WORKSPACE_PATH, "test", "data");
-const paths = {
-  SHEETS_CACHE_FOLDER: path.resolve(testDataDir, "cache"),
-  SHEETS_INPUT_FOLDER: path.resolve(testDataDir, "input"),
-  SHEETS_OUTPUT_FOLDER: path.resolve(testDataDir, "output"),
-};
+import { TEST_DATA_PATHS } from "../../../../../test/helpers/utils";
 
 const testData = [
   {
@@ -23,9 +16,15 @@ class TestProcessor extends BaseProcessor<string, any> {
   }
 }
 let processor: TestProcessor;
+
+/** yarn workspace scripts test -t base.spec.ts */
 describe("Base Processor", () => {
   beforeAll(() => {
-    processor = new TestProcessor({ namespace: "BaseProcessor", paths });
+    processor = new TestProcessor({
+      namespace: "BaseProcessor",
+      paths: TEST_DATA_PATHS,
+      cacheVersion: new Date().getTime(),
+    });
     processor.cache.clear();
   });
   afterAll(() => {
@@ -58,7 +57,11 @@ describe("Deferred Processor", () => {
   }
   let deferredProcessor: DeferredProcessor;
   beforeEach(() => {
-    deferredProcessor = new DeferredProcessor({ namespace: "BaseProcessor", paths });
+    deferredProcessor = new DeferredProcessor({
+      namespace: "BaseProcessor",
+      paths: TEST_DATA_PATHS,
+      cacheVersion: new Date().getTime(),
+    });
     deferredProcessor.cache.clear();
   });
   afterAll(() => {

packages/scripts/src/commands/app-data/convert/processors/flowParser/flowParser.spec.ts

@@ -1,19 +1,7 @@
-import path from "path";
 import { FlowTypes } from "data-models";
-import { SCRIPTS_WORKSPACE_PATH } from "../../../../../paths";
 import { clearLogs, getLogs } from "../../utils";
 import { FlowParserProcessor } from "./flowParser";
-
-const testDataDir = path.resolve(SCRIPTS_WORKSPACE_PATH, "test", "data");
-const paths = {
-  SHEETS_CACHE_FOLDER: path.resolve(testDataDir, "cache"),
-  SHEETS_INPUT_FOLDER: path.resolve(testDataDir, "input"),
-  SHEETS_OUTPUT_FOLDER: path.resolve(testDataDir, "output"),
-};
-// Export method to allow use in parser-specific tests (to test on multiple instances of a flow type)
-export function getTestFlowParserProcessor() {
-  return new FlowParserProcessor(paths);
-}
+import { TEST_DATA_PATHS } from "../../../../../../test/helpers/utils";
 
 // NOTE - inputs are just to test general structure and not run actual parser code
 const testInputs: FlowTypes.FlowTypeWithData[] = [
@@ -48,7 +36,7 @@ const testInputs: FlowTypes.FlowTypeWithData[] = [
 let processor: FlowParserProcessor;
 describe("FlowParser Processor", () => {
   beforeAll(() => {
-    processor = getTestFlowParserProcessor();
+    processor = new FlowParserProcessor(TEST_DATA_PATHS);
     processor.cache.clear();
   });
   beforeEach(() => {
@@ -77,7 +65,7 @@ describe("FlowParser Processor", () => {
       rows: null,
     };
     await processor.process([brokenFlow]);
-    const errorLogs = getLogs("error", "Template parse error");
+    const errorLogs = getLogs("error").filter(({ message }) => message === "Template parse error");
     expect(errorLogs).toEqual([
       {
         source: "flowParser",
@@ -113,7 +101,7 @@ describe("FlowParser Processor", () => {
 /** Additional tests for data pipe integration */
 describe("FlowParser Processor - Data Pipes", () => {
   beforeAll(() => {
-    processor = getTestFlowParserProcessor();
+    processor = new FlowParserProcessor(TEST_DATA_PATHS);
     processor.cache.clear();
   });
   beforeEach(() => {

packages/scripts/src/commands/app-data/convert/processors/flowParser/parsers/data_list.parser.spec.ts

@@ -1,5 +1,6 @@
 import { DataListParser } from ".";
-import { getTestFlowParserProcessor } from "../flowParser.spec";
+import { TEST_DATA_PATHS } from "../../../../../../../test/helpers/utils";
+import { FlowParserProcessor } from "../flowParser";
 
 const testFlow = {
   flow_type: "data_list",
@@ -15,6 +16,7 @@ const testFlow = {
   ],
 };
 
+/** yarn workspace scripts test -t  data_list.parser.spec.ts **/
 describe("data_list Parser (single)", () => {
   let outputRows: any[];
   beforeAll(() => {
@@ -44,7 +46,7 @@ describe("data_list Parser (single)", () => {
 });
 
 describe("data_list Parser (multiple)", () => {
-  const parser = getTestFlowParserProcessor();
+  const parser = new FlowParserProcessor(TEST_DATA_PATHS);
   beforeAll(() => {
     parser.cache.clear();
   });

packages/scripts/src/commands/app-data/convert/processors/xlsxWorkbook.spec.ts

@@ -32,7 +32,7 @@ describe("XLSX Workbook Processor", () => {
   // an array of arrays
   it("Converts XLSXs to array of JSON sheet arrays", async () => {
     const outputs = await processor.process(testInputs);
-    expect(Array.isArray(outputs)).toBeTrue();
+    expect(Array.isArray(outputs)).toEqual(true);
     expect(outputs.length).toEqual(testInputs.length);
     // each entry may contain multiple sheets from workbook
     const testInputsheets = outputs[0];

packages/scripts/src/commands/app-data/convert/utils/app-data-override.utils.spec.ts

@@ -1,5 +1,5 @@
 import { FlowTypes } from "data-models";
-import { useMockWarningLogger } from "../../../../../test/helpers/utils";
+import { useMockLogger } from "../../../../../test/helpers/utils";
 import { assignFlowOverrides } from "./app-data-override.utils";
 
 const TEST_INPUTS: FlowTypes.FlowTypeWithData[] = [
@@ -24,6 +24,7 @@ const TEST_INPUTS: FlowTypes.FlowTypeWithData[] = [
   },
 ];
 
+/** yarn workspace scripts test -t app-data-override.utils.spec.ts */
 describe("App Data Override", () => {
   it("Assigns flow override mapping", () => {
     const output = assignFlowOverrides(TEST_INPUTS);
@@ -36,7 +37,7 @@ describe("App Data Override", () => {
   });
 
   it("Logs warning on missing override target", () => {
-    const warningLogger = useMockWarningLogger();
+    const loggerSpy = useMockLogger(true);
     assignFlowOverrides([
       ...TEST_INPUTS,
       {
@@ -47,7 +48,8 @@ describe("App Data Override", () => {
         override_condition: "test_condition_3",
       },
     ]);
-    expect(warningLogger).toHaveBeenCalledOnceWith({
+    expect(loggerSpy.warning).toHaveBeenCalledTimes(1);
+    expect(loggerSpy.warning).toHaveBeenCalledWith({
       msg1: "Override target does not exist: missing_list",
       msg2: "override_invalid",
     });