Merge branch 'master' into core/capacitor-5

.github/workflows/web-build.yml

@@ -75,7 +75,7 @@ jobs:
       # Set deployment from input, replacing branch slash name with suffix (`prod/my_deployment` -> `my_deployment`)
       # https://stackoverflow.com/a/40524879
       - name: Set deployment
-        if: ${{inputs.deployment && inputs.deployment != 'master'}}
+        if: ${{inputs.deployment && inputs.deployment != 'master' && inputs.deployment!='refs/heads/master'}}
         run: |
           echo "DEPLOYMENT_NAME=$(echo ${{inputs.deployment}} | sed 's/.*\///')" >> $GITHUB_ENV;  
         shell: bash
@@ -154,11 +154,6 @@ jobs:
         if: ${{env.GIT_SHA}}
         run: echo "export const GIT_SHA = \"$GIT_SHA\";" > src/environments/sha.ts
 
-      # Ensure required packages built to enable deployment set
-      # TODO - should add implicit dep to scripts when NX added
-      - name: Precompile
-        run: yarn scripts compile types
-
       - name: Set deployment
         run: yarn workflow deployment set ${{env.DEPLOYMENT_NAME}}
 

android/app/build.gradle

@@ -7,8 +7,8 @@ android {
         applicationId "international.idems.plh_teens"
         minSdkVersion rootProject.ext.minSdkVersion
         targetSdkVersion rootProject.ext.targetSdkVersion
-        versionCode 16019
-        versionName "0.16.19"
+        versionCode 16020
+        versionName "0.16.20"
         testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
     }
     buildTypes {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "frontend",
-  "version": "0.16.19",
+  "version": "0.16.20",
   "author": "IDEMS International",
   "license": "See LICENSE",
   "homepage": "https://idems.international/",
@@ -11,11 +11,10 @@
   ],
   "scripts": {
     "ng": "ng",
-    "start": "yarn prepare && concurrently --kill-others --raw \"yarn data-models:serve\" \"ng serve --open\"",
+    "start": "yarn prepare && ng serve --open",
     "start:local": "yarn prepare && concurrently --kill-others --raw  \"ng serve --open\" \"yarn workflow sync_local\"",
-    "prepare": "node .husky/prepare.js && yarn scripts compile types && yarn workflow populate_src_assets",
+    "prepare": "node .husky/prepare.js && yarn workflow populate_src_assets",
     "build": "yarn prepare && ng build --configuration=production",
-    "data-models:serve": "yarn workspace data-models serve",
     "compodoc:generate": "compodoc --tsconfig tsconfig.docs.json --theme material --assetsFolder 'documentation/docs/assets' --output documentation/docs/generated --disablePrivate --disableLifeCycleHooks",
     "compodoc:serve": "yarn compodoc:generate --serve --watch --open",
     "scripts": "yarn workspace scripts start",

packages/data-models/README.md

@@ -0,0 +1,40 @@
+## Data Models
+
+### How to import
+
+**Node**
+The workspace is designed for direct import as `.ts` files, to be compiled within parent project. Most imports are made available from the top-level `index.ts`
+```ts
+import { getDefaultAppConfig } from "data-models"
+```
+If any imports are not available simply update the workspace `index.ts` to include
+
+**Angular**
+The angular compiler uses local `tsconfig.app.json` to include external dependencies, therefore if using an entry should be included
+```json
+{
+  "extends": "./tsconfig.json",
+  "include": [
+    "packages/data-models/**/*.ts",
+  ]
+}
+
+```
+Once included, data-models can be imported in the same way
+```ts
+import { getDefaultAppConfig } from "data-models"
+```
+
+*Legacy Implementation*
+If it is not possible to import via tsconfig, it is still possible to include as a workspace dependency (defined in package.json), however the data models package.json requires update to build and import from dist folder. Additionally any angular serve script should concurrently serve the data-models workspace to ensure changes are recompiled, e.g.
+
+```ts
+"main": "dist/index.js",
+"types": "dist/index.d.ts",
+"scripts": {
+    "build": "tsc -b",
+    "serve": "tsc -w"
+},
+```
+
+

packages/data-models/index.ts

@@ -1,12 +1,6 @@
+export * from "./appConfig";
+export * from "./db.model";
+export * from "./deployment.model";
 export * from "./flowTypes";
 export * from "./functions";
-// NOTE - avoid exporting workflows as node-based can't be consumed by frontend src
-// export { IDeploymentWorkflows, IWorkflow, IWorkflowContext, WORKFLOW_DEFAULTS } from "./workflows";
-export {
-  IDeploymentConfig,
-  IDeploymentConfigJson,
-  DEPLOYMENT_CONFIG_EXAMPLE_DEFAULTS,
-  DEPLOYMENT_CONFIG_VERSION,
-} from "./deployment.model";
-export { IAppSkin } from "./skin.model";
-export { IAppConfig, IAppConfigOverride, getDefaultAppConfig } from "./appConfig";
+export * from "./skin.model";

packages/data-models/package.json

@@ -2,19 +2,18 @@
   "name": "data-models",
   "version": "1.0.0",
   "description": "",
-  "types": "index.d.ts",
-  "scripts": {
-    "build": "tsc -b",
-    "serve": "tsc -w"
+  "files": [
+    "dist"
+  ],
+  "main": "./index.ts",
+  "types": "./index.ts",
+  "dependencies": {
+    "clone": "^2.1.2"
   },
   "devDependencies": {
     "@types/clone": "^2.1.1",
     "@types/node": "^16.18.9",
-    "scripts": "workspace:*",
     "shared": "workspace:*",
     "typescript": "~4.2.4"
-  },
-  "dependencies": {
-    "clone": "^2.1.2"
   }
 }

packages/data-models/tsconfig.json

@@ -9,6 +9,6 @@
     "target": "es5",
     "esModuleInterop": true
   },
-  "include": ["index.ts"],
+  "include": ["**/*.ts"],
   "exclude": ["**/*.d.ts", "dist", "node_modules"]
 }

packages/scripts/package.json

@@ -30,10 +30,8 @@
     "commander": "^8.3.0",
     "data-models": "workspace:*",
     "fs-extra": "^9.0.1",
-    "glob": "^7.2.0",
     "inquirer": "^7.3.3",
     "node-rsa": "^1.1.1",
-    "open": "^9.1.0",
     "p-queue": "^6.6.2",
     "semver": "^7.5.2",
     "shared": "workspace:*",
@@ -42,6 +40,7 @@
     "ts-morph": "^15.0.0",
     "typescript": "^4.1.2",
     "winston": "^3.8.1",
+    "workflows": "workspace:*",
     "xlsx": "^0.18.5"
   },
   "devDependencies": {
@@ -59,7 +58,6 @@
     "nodemon": "^2.0.19",
     "pkg": "^5.5.1",
     "ts-node": "^10.8.0",
-    "ts-node-dev": "^1.0.0",
-    "tsimportlib": "^0.0.5"
+    "ts-node-dev": "^1.0.0"
   }
 }

packages/scripts/src/commands/app-data/convert/convert.spec.ts

@@ -59,7 +59,7 @@ describe("App Data Converter", () => {
   });
   it("Throws on duplicate flows", async () => {
     await converter.run().catch((err) => {
-      expect(err.message.includes("Duplicate flows found"));
+      expect(err.message.includes("Duplicate flows found")).toEqual(true);
     });
   });
 });
@@ -70,20 +70,20 @@ describe("App Data Converter - Error Checking", () => {
     outputFolder: path.resolve(SCRIPTS_TEST_DATA_DIR, "output", "errorChecking"),
     cacheFolder: path.resolve(SCRIPTS_TEST_DATA_DIR, "cache"),
   };
-  const errorConverter = new AppDataConverter(errorPaths);
+  let errorConverter = new AppDataConverter(errorPaths);
   beforeAll(() => {
     if (existsSync(paths.outputFolder)) {
       emptyDirSync(paths.outputFolder);
     }
   });
   it("Tracks number of conversion errors", async () => {
-    const errorConverter = new AppDataConverter(errorPaths);
+    errorConverter = new AppDataConverter(errorPaths);
     const { errors } = await errorConverter.run();
     expect(errors.length).toBeGreaterThan(0);
   });
-  it("Throws on duplicate flows", async () => {
+  it("Throws on duplicate flows (2)", async () => {
     await errorConverter.run().catch((err) => {
-      expect(err.message.includes("Duplicate flows found"));
+      expect(err.message.includes("Duplicate flows found")).toBe(true);
     });
   });
 });

packages/scripts/src/commands/app-data/convert/processors/flowParser/parsers/data_pipe.parser.spec.ts

@@ -1,5 +1,5 @@
 import { FlowTypes } from "data-models";
-import { IDataPipeOperation } from "shared/src/models/dataPipe/types";
+import type { IDataPipeOperation } from "shared/src/models/dataPipe/types";
 import { DataPipeParser } from "./data_pipe.parser";
 
 const testInputSources = {

packages/scripts/src/commands/app-data/postProcess/assets.spec.ts

@@ -9,7 +9,7 @@ import mockFs from "mock-fs";
 
 // Use default imports to allow spying on functions and replacing with mock methods
 import { ActiveDeployment } from "../../deployment/get";
-import { Logger } from "../../../utils/logging.utils";
+import { Logger } from "../../../utils";
 import path from "path";
 import { IAssetEntryHashmap } from "data-models/deployment.model";
 

packages/scripts/src/commands/app-data/postProcess/sheets.spec.ts

@@ -6,7 +6,6 @@ import mockFs from "mock-fs";
 
 // Use default imports to allow spying on functions and replacing with mock methods
 import { ActiveDeployment } from "../../deployment/get";
-import { Logger } from "../../../utils/logging.utils";
 import path from "path";
 
 /** Mock file system folders for use in tests */

packages/scripts/src/commands/compile/types.ts

@@ -13,19 +13,27 @@ import { DATA_MODELS_WORKSPACE_PATH } from "../../paths";
 
 const program = new Command("types");
 export default program.description("Compile types").action(() => {
-  compileTypes();
+  _DEPRECATED_compileTypes();
 });
 
 /***************************************************************************************
  * Main Methods
  *************************************************************************************/
 /**
+ * DEPRECATED - CC 2023-08-18
+ * Previously frontend app would import from data-models compiled dist, so extra step
+ * required to ensure this had been built before running
+ * However this has been updated to import .ts directly and compile with angular compiler
+ * so not required.
+ * Method should be retained for short term in case pattern required
+ * (recommend to remove post 2023-11-18)
+ * ===
  *
  * Data models import types from scripts but scripts also import types from data models
  * Whilst the types are not in conflict themselves, they do lead to compilation issues
  * Remove any outstanding types before compiling noew
  */
-function compileTypes() {
+function _DEPRECATED_compileTypes() {
   const dataModelsBuildPath = path.resolve(DATA_MODELS_WORKSPACE_PATH, "dist");
   console.log(chalk.yellow("Compiling type definitions"));
   if (fs.existsSync(dataModelsBuildPath)) {

packages/scripts/src/commands/deployment/create.ts

@@ -5,7 +5,6 @@ import path from "path";
 import { DEPLOYMENTS_PATH } from "../../paths";
 import { Logger, logOutput, logWarning, promptInput, promptOptions } from "../../utils";
 import type { IDeploymentConfigJson } from "./common";
-import { DeploymentSet } from "./set";
 import generateDefaultConfig from "./templates/config.default";
 import generateExtendedConfig from "./templates/config.extended";
 import { loadDeploymentJson } from "./utils";

packages/scripts/src/commands/index.ts

@@ -9,7 +9,7 @@ import configCmd from "./config";
 import deploymentCmd from "./deployment";
 import versionCmd from "./version";
 import workflowCmd from "./workflow";
-import { logWarning, logProgramHelp } from "../utils/logging.utils";
+import { logWarning, logProgramHelp } from "../utils";
 
 const program = new Command();
 

packages/scripts/src/commands/workflow/run.ts

@@ -4,7 +4,7 @@ import boxen from "boxen";
 import chalk from "chalk";
 import path from "path";
 import { Command } from "commander";
-import { IDeploymentWorkflows, IWorkflow, WORKFLOW_DEFAULTS } from "data-models/workflows";
+import { IDeploymentWorkflows, IWorkflow, WORKFLOW_DEFAULTS } from "workflows";
 import ALL_TASKS from "../../tasks";
 import { Logger, logProgramHelp, pad, promptOptions } from "../../utils";
 import { ActiveDeployment } from "../deployment/get";

packages/scripts/src/paths.ts

@@ -1,24 +1,2 @@
-import * as path from "path";
-
-export const ROOT_DIR = path.resolve(__dirname, "../../../");
-export const MAIN_PACKAGE_PATH = path.join(ROOT_DIR, "package.json");
-
-export const DEPLOYMENTS_PATH = path.resolve(ROOT_DIR, ".idems_app", "deployments");
-
-export const SCRIPTS_WORKSPACE_PATH = path.join(ROOT_DIR, "packages/scripts");
-export const SCRIPTS_TEST_DATA_DIR = path.join(SCRIPTS_WORKSPACE_PATH, "test", "data");
-export const SCRIPTS_LOGS_DIR = path.join(SCRIPTS_WORKSPACE_PATH, "logs");
-export const DATA_MODELS_WORKSPACE_PATH = path.join(ROOT_DIR, "packages/data-models");
-
-export const SRC_ASSETS_PATH = path.join(ROOT_DIR, "src", "assets");
-
-export const CONFIG_FOLDER_PATH = path.join(SCRIPTS_WORKSPACE_PATH, "config");
-export const PRIVATE_KEY_PATH = path.join(CONFIG_FOLDER_PATH, "private.key");
-export const PUBLIC_KEY_PATH = path.join(CONFIG_FOLDER_PATH, "public.key");
-export const CREDENTIALS_PATH = path.join(CONFIG_FOLDER_PATH, "credentials.json");
-export const AUTH_TOKEN_PATH = path.join(CONFIG_FOLDER_PATH, "token.json");
-
-export const RESOURCE_FOLDER_PATH = path.join(ROOT_DIR, "resources");
-
-export const ANDROID_RES_PATH = path.join(ROOT_DIR, "android/app/src/main/res");
-export const APP_BUILD_GRADLE_PATH = path.join(ROOT_DIR, "android/app/build.gradle");
+// re-export shared utils for ease of import
+export * from "shared/src/paths";

packages/scripts/src/tasks/providers/workflow.ts

@@ -1,4 +1,4 @@
-import { IWorkflowContext } from "data-models/workflows";
+import { IWorkflowContext } from "workflows";
 import { parseCommand } from "../../commands";
 
 /**

packages/scripts/src/utils.ts

@@ -0,0 +1,2 @@
+// re-export shared utils for ease of import
+export * from "shared/src/utils";

packages/shared/README.md

@@ -0,0 +1,44 @@
+## Shared
+
+Common platform methods and utility functions.
+
+There are particularly a number of utility methods used by both scripts and workflow packages
+
+Note, some methods will only work in node-based environments (non-browser)
+
+### How to import
+
+**Node**
+The workspace is designed for direct import as `.ts` files, to be compiled within parent project. Most imports are made available from the top-level `index.ts`
+```ts
+import { TemplatedData } from "shared"
+```
+If any imports are not available simply update the workspace `index.ts` to include
+
+**Angular**
+The angular compiler uses local `tsconfig.app.json` to include external dependencies, therefore if using an entry should be included
+
+```json
+{
+  "extends": "./tsconfig.json",
+  "include": [
+    "packages/shared/**/*.ts",
+  ]
+}
+```
+
+**NOTE** - the `shared` package workspace name conflicts with local `shared` folder
+As such when importing methods a package-based path will be required, i.e
+
+```ts
+import { TemplatedData } from "packages/shared"
+```
+
+In addition, to avoid compiler errors thrown by non-browser shared methods, explicit paths should be included to import only the supported files as required
+
+```ts
+import { AppStringEvaluator } from "packages/shared/src/models/appStringEvaluator/appStringEvaluator";
+import { TemplatedData } from "packages/shared/src/models/templatedData/templatedData";
+```
+
+In future it may be better to move any non-compatible shared methods to own package, or polyfill missing functions (e.g. `os`)