Merge branch 'master' into fix-cname-part-3

package.json

@@ -1,6 +1,6 @@
 {
   "name": "frontend",
-  "version": "0.16.35",
+  "version": "0.16.36",
   "author": "IDEMS International",
   "license": "See LICENSE",
   "homepage": "https://idems.international/",

packages/data-models/flowTypes.ts

@@ -82,6 +82,8 @@ export namespace FlowTypes {
   export interface DataPipeFlow extends FlowTypeWithData {
     flow_type: "data_pipe";
     rows: IDataPipeOperation[];
+    /** Generated list of output flows created by generator */
+    _output_flows?: FlowTypeBase[];
   }
   export interface GeneratorFlow extends FlowTypeWithData {
     flow_type: "generator";
@@ -91,6 +93,8 @@ export namespace FlowTypes {
       output_flow_subtype?: string;
       output_flow_type?: FlowType;
     };
+    /** Generated list of output flows created by generator */
+    _output_flows?: FlowTypeBase[];
   }
   export interface Translation_strings {
     [sourceText: string]: string;

packages/scripts/src/commands/app-data/convert/convert.spec.ts

@@ -65,7 +65,7 @@ describe("App Data Converter", () => {
   it("Populates output to folder by data type", async () => {
     await converter.run();
     const outputFolders = readdirSync(paths.outputFolder);
-    expect(outputFolders).toEqual(["data_list", "template"]);
+    expect(outputFolders).toEqual(["data_list", "data_pipe", "template"]);
   });
   it("Supports input from multiple source folders", async () => {
     const multipleSourceConverter = new AppDataConverter({

packages/scripts/src/commands/app-data/convert/processors/flowParser/flowParser.spec.ts

@@ -81,7 +81,7 @@ describe("FlowParser Processor", () => {
   it("Outputs flows by type", async () => {
     const output = await processor.process(testInputs);
     // NOTE - data_pipe and generator flows will not populate self but instead generated outputs
-    expect(Object.keys(output)).toEqual(["data_list", "template"]);
+    expect(Object.keys(output)).toEqual(["data_list", "template", "data_pipe"]);
     const errorLogs = getLogs("error");
     if (errorLogs.length > 0) {
       console.log("Unexpected Errors:\n", errorLogs);

packages/scripts/src/commands/app-data/convert/processors/flowParser/flowParser.ts

@@ -4,7 +4,7 @@ import { IConverterPaths, IFlowHashmapByType, IParsedWorkbookData } from "../../
 import { arrayToHashmap, groupJsonByKey, IContentsEntry } from "../../utils";
 import BaseProcessor from "../base";
 
-const cacheVersion = 20240924.4;
+const cacheVersion = 20241001.2;
 
 export class FlowParserProcessor extends BaseProcessor<FlowTypes.FlowTypeWithData> {
   public parsers: { [flowType in FlowTypes.FlowType]: Parsers.DefaultParser } = {

packages/scripts/src/commands/app-data/convert/processors/flowParser/parsers/data_pipe.parser.spec.ts

@@ -18,7 +18,7 @@ describe("data_pipe Parser", () => {
     parser.flowProcessor.processedFlowHashmap = getTestData();
   });
   it("Populates generated data lists", async () => {
-    parser.run({
+    const res = parser.run({
       flow_name: "test_pipe_parse",
       flow_type: "data_pipe",
       rows: [
@@ -45,6 +45,20 @@ describe("data_pipe Parser", () => {
       test_output_1: [{ id: 2 }, { id: 3 }],
       test_output_2: [{ id: 3 }],
     });
+
+    // Also check output references stored
+    expect(res._output_flows).toEqual([
+      {
+        flow_type: "data_list",
+        flow_subtype: "generated",
+        flow_name: "test_output_1",
+      },
+      {
+        flow_type: "data_list",
+        flow_subtype: "generated",
+        flow_name: "test_output_2",
+      },
+    ]);
   });
 
   it("Allows outputs from one pipe to be used in another", async () => {

packages/scripts/src/commands/app-data/convert/processors/flowParser/parsers/data_pipe.parser.ts

@@ -7,7 +7,6 @@ export class DataPipeParser extends DefaultParser<FlowTypes.DataPipeFlow> {
   private outputHashmap: { [flow_name: string]: { [output_name: string]: any } } = {};
 
   /** If extending the class add additional postprocess pipeline here */
-
   public postProcessFlow(flow: FlowTypes.DataPipeFlow): FlowTypes.DataPipeFlow {
     const inputSources = this.loadInputSources();
     const pipe = new DataPipe(flow.rows, inputSources);
@@ -24,30 +23,31 @@ export class DataPipeParser extends DefaultParser<FlowTypes.DataPipeFlow> {
       // HACK - populate to output hashmap for use in tests. Clone output due to deep nest issues
       this.outputHashmap[flow.flow_name] = JSON.parse(JSON.stringify(outputs));
 
-      this.populateGeneratedFlows(outputs);
-      // As the populated flows will be passed directly to the processor queue
-      // can just return undefined so that the data pipe will not be stored in outputs
-      return undefined;
+      const generated = this.generateFlows(outputs);
+
+      // Pass all generated flows to the back of the current processing queue so that they can be
+      // populated to processed hashmap and referenced from other processes as required
+      for (const generatedFlow of generated) {
+        const deferId = `${generatedFlow.flow_type}.${generatedFlow.flow_subtype}.${generatedFlow.flow_name}`;
+        this.flowProcessor.deferInputProcess(generatedFlow, deferId);
+      }
+
+      // Return the parsed flow along with a summary of output flows to store within outputs
+      flow._output_flows = generated.map(({ rows, ...keptFields }) => keptFields);
+      return flow;
     } catch (error) {
       console.trace(error);
       throw error;
     }
   }
 
-  private populateGeneratedFlows(outputs: { [output_name: string]: any[] }) {
-    for (const [flow_name, rows] of Object.entries(outputs)) {
-      const flow: FlowTypes.FlowTypeWithData = {
-        flow_name,
-        flow_subtype: "generated",
-        flow_type: "data_list",
-        rows,
-      };
-      const deferId = `${flow.flow_type}.${flow.flow_subtype}.${flow.flow_name}`;
-
-      // Pass all generated flows to the back of the current processing queue so that they can be
-      // populated to processed hashmap and referenced from other processes as required
-      this.flowProcessor.deferInputProcess(flow, deferId);
-    }
+  private generateFlows(outputs: { [output_name: string]: any[] }) {
+    const generatedFlows: FlowTypes.Data_list[] = Object.entries(outputs).map(
+      ([flow_name, rows]) => {
+        return { flow_name, flow_subtype: "generated", flow_type: "data_list", rows };
+      }
+    );
+    return generatedFlows;
   }
 
   private loadInputSources() {

packages/scripts/src/commands/app-data/convert/processors/flowParser/parsers/generator.parser.spec.ts

@@ -99,6 +99,21 @@ describe("generator Parser", () => {
       },
     ]);
   });
+  it("populates list of outputs", async () => {
+    const res = parser.run(generatorInput()) as FlowTypes.GeneratorFlow;
+    expect(res._output_flows).toEqual([
+      {
+        flow_type: "template",
+        flow_subtype: "generated_type_1",
+        flow_name: "generated_template_1",
+      },
+      {
+        flow_type: "template",
+        flow_subtype: "generated_type_2",
+        flow_name: "generated_template_2",
+      },
+    ]);
+  });
   it("parses generated flows using type parser", async () => {
     parser.run(generatorInput());
     await parser.flowProcessor.queue.onIdle();

packages/scripts/src/commands/app-data/convert/processors/flowParser/parsers/generator.parser.ts

@@ -30,9 +30,9 @@ export class GeneratorParser extends DefaultParser<FlowTypes.GeneratorFlow> {
         const deferId = `${generatedFlow.flow_type}.${generatedFlow.flow_subtype}.${generatedFlow.flow_name}`;
         this.flowProcessor.deferInputProcess(generatedFlow, deferId);
       }
-      // As the populated flows will be passed directly to the processor queue
-      // can just return undefined so that the data pipe will not be stored in outputs
-      return undefined;
+      // Return the parsed generator along with a summary of output flows to store within outputs
+      flow._output_flows = generated.map(({ rows, ...keptFields }) => keptFields);
+      return flow;
     } catch (error) {
       console.trace(error);
       throw error;

packages/scripts/src/commands/app-data/report/reporters/asset-summary.spec.ts

@@ -71,6 +71,19 @@ describe("Asset Summary Report", () => {
     ]);
   });
 
+  it("Allows authors to specify assets with preceding forward slash", async () => {
+    const { asset_summary } = await reporter.process({
+      template: [
+        {
+          flow_type: "template",
+          flow_name: "test_forward_slash",
+          rows: [{ type: "audio", value: "/mock_audio.mp3" }],
+        },
+      ],
+    });
+    expect(asset_summary.data).toEqual([{ path: "mock_audio.mp3", size_kb: 2048, count: 1 }]);
+  });
+
   it("Reports missing assets", async () => {
     const { assets_missing } = await reporter.process(MOCK_WORKBOOK_DATA);
     expect(assets_missing.data).toEqual([{ path: "missing_audio.mp3", count: 1, missing: true }]);

packages/scripts/src/commands/app-data/report/reporters/asset-summary.ts

@@ -1,6 +1,6 @@
-import { FlowTypes, IAssetEntryHashmap, IDeploymentConfigJson } from "data-models";
+import { FlowTypes, IAssetEntryHashmap } from "data-models";
 import { IReportTable } from "../report.types";
-import { isObjectLiteral, kbToMB, sortJsonKeys } from "shared";
+import { cleanAssetName, isObjectLiteral, kbToMB, sortJsonKeys } from "shared";
 import { IParsedWorkbookData } from "../../convert/types";
 
 interface IReportData {
@@ -147,6 +147,8 @@ export class AssetsSummaryReport {
   }
 
   private markAsset(name: string) {
+    // NOTE - use same method as frontend asset service to allow assets with preceding `/`
+    name = cleanAssetName(name);
     this.reportSummary[name] ??= 0;
     this.reportSummary[name]++;
   }

packages/shared/src/utils/string-utils.ts

@@ -43,3 +43,9 @@ export function parseStringValue(v: string): any {
   if (v.match(/^"[a-z0-9.]*"$/gi)) return v.replace(/"/g, "");
   return v;
 }
+
+/** Remove preceding `/` from any named asset paths */
+export function cleanAssetName(value: string) {
+  if (value.startsWith("/")) value = value.substring(1);
+  return value;
+}

src/app/shared/components/template/services/template-asset.service.ts

@@ -6,6 +6,7 @@ import { TemplateTranslateService } from "./template-translate.service";
 import { IAssetEntry, IAssetContentsEntryMinimal } from "data-models";
 import { HttpClient } from "@angular/common/http";
 import { BehaviorSubject, lastValueFrom } from "rxjs";
+import { cleanAssetName } from "packages/shared/src/utils/string-utils";
 
 /** Synced assets are automatically copied during build to asset subfolder */
 const ASSETS_BASE = `assets/app_data/assets`;
@@ -55,7 +56,7 @@ export class TemplateAssetService extends AsyncServiceBase {
    * 4. default theme, default language
    */
   getTranslatedAssetPath(value: string) {
-    let assetName = this.cleanAssetName(value);
+    let assetName = cleanAssetName(value);
     const assetEntry = this.assetsContentsList$.value[assetName];
     if (!assetEntry) {
       console.error("Asset missing", value, assetName);
@@ -86,12 +87,6 @@ export class TemplateAssetService extends AsyncServiceBase {
     return this.getAssetPath(assetName, assetEntry);
   }
 
-  private cleanAssetName(value: string) {
-    // remove prefix slash
-    if (value.startsWith("/")) value = value.substring(1);
-    return value;
-  }
-
   private getAssetPath(
     assetName: string,
     contentsEntry: IAssetContentsEntryMinimal | Partial<IAssetEntry>