fix: metadata row removal

packages/scripts/src/commands/app-data/convert/processors/flowParser/flowParser.ts

@@ -4,7 +4,7 @@ import { IConverterPaths, IFlowHashmapByType, IParsedWorkbookData } from "../../
 import { arrayToHashmap, groupJsonByKey, IContentsEntry } from "../../utils";
 import BaseProcessor from "../base";
 
-const cacheVersion = 20241114.2;
+const cacheVersion = 20241121.0;
 
 export class FlowParserProcessor extends BaseProcessor<FlowTypes.FlowTypeWithData> {
   public parsers: { [flowType in FlowTypes.FlowType]: Parsers.DefaultParser } = {

packages/scripts/src/commands/app-data/convert/processors/flowParser/parsers/data_list.parser.spec.ts

@@ -1,6 +1,6 @@
 import { FlowTypes } from "data-models";
 import { DataListParser } from ".";
-import { TEST_DATA_PATHS, useMockLogger } from "../../../../../../../test/helpers/utils";
+import { TEST_DATA_PATHS } from "../../../../../../../test/helpers/utils";
 import { FlowParserProcessor } from "../flowParser";
 
 const MOCK_DATA_LIST = (): FlowTypes.Data_list => ({
@@ -47,18 +47,19 @@ describe("data_list Parser Metadata", () => {
       additional: { type: "boolean" },
     });
   });
-  it("Assigns column metadata from metadata row", () => {
+  it("Assigns column metadata from metadata row (and removes it)", () => {
     const flowWithMetadataRow = {
       ...MOCK_DATA_LIST(),
       rows: [
         { id: "@metadata", number: "type: number", text: "type: string" },
         { id: "id_1", number: undefined, text: undefined },
       ],
     };
-    const { _metadata } = parser.postProcessFlow(flowWithMetadataRow);
+    const { _metadata, rows } = parser.postProcessFlow(flowWithMetadataRow);
     expect(_metadata).toEqual({
       number: { type: "number" },
     });
+    expect(rows.length).toEqual(1);
   });
   it("Omits metadata when all columns are strings", () => {
     const flowWithStringColumns = {

packages/scripts/src/commands/app-data/convert/processors/flowParser/parsers/data_list.parser.ts

@@ -35,27 +35,32 @@ export class DataListParser extends DefaultParser {
   }
 
   public override postProcessFlow(flow: FlowTypes.Data_list) {
-    const metadata = this.getFlowMetadata(flow);
-    if (!isEmptyObjectDeep(metadata)) {
-      flow._metadata = metadata;
-    }
-    return flow;
+    const flowWithMetadata = this.getFlowMetadata(flow);
+    return flowWithMetadata;
   }
 
   public postProcessFlows(flows: FlowTypes.Data_list[]) {
     const flowsWithOverrides = assignFlowOverrides(flows);
     return flowsWithOverrides;
   }
 
-  /** Assign column metadata from @metadata row if provided, or infer from data if not*/
-  private getFlowMetadata(flow: FlowTypes.FlowTypeWithData) {
+  /** Assign column metadata from @metadata row if provided, or infer from data if not */
+  private getFlowMetadata(flow: FlowTypes.Data_list) {
     const [firstRow] = flow.rows;
-    const initialMetadata =
-      firstRow?.id === "@metadata"
-        ? this.assignMetadataFromRow(firstRow)
-        : this.inferMetadataFromData(flow.rows);
+    let metadataInitial: FlowTypes.Data_list["_metadata"];
+    // assign from metadata row and remove
+    if (firstRow?.id === "@metadata") {
+      metadataInitial = this.assignMetadataFromRow(firstRow);
+      flow.rows = flow.rows.slice(0, 1);
+    }
+    // infer from data when metadata row not available
+    else {
+      metadataInitial = this.inferMetadataFromData(flow.rows);
+    }
+    // quality check metadata and assign cleaned meta to flow
+    const { warnings, metadata } = this.qualityCheckMetadata(metadataInitial);
+    flow._metadata = metadata;
 
-    const { metadata, warnings } = this.qualityCheckMetadata(initialMetadata);
     if (warnings.length > 0) {
       for (const warning of warnings) {
         console.warn(warning);
@@ -65,7 +70,7 @@ export class DataListParser extends DefaultParser {
         msg2: flow._xlsxPath,
       });
     }
-    return metadata;
+    return flow;
   }
 
   /** Assign data_list metadata using a first `@metadata` row */