diff --git a/packages/scripts/src/commands/app-data/convert/processors/flowParser/flowParser.ts b/packages/scripts/src/commands/app-data/convert/processors/flowParser/flowParser.ts index c3bf1bf2cd..6e33b50a67 100644 --- a/packages/scripts/src/commands/app-data/convert/processors/flowParser/flowParser.ts +++ b/packages/scripts/src/commands/app-data/convert/processors/flowParser/flowParser.ts @@ -4,7 +4,7 @@ import { IConverterPaths, IFlowHashmapByType, IParsedWorkbookData } from "../../ import { arrayToHashmap, groupJsonByKey, IContentsEntry } from "../../utils"; import BaseProcessor from "../base"; -const cacheVersion = 20241114.2; +const cacheVersion = 20241121.0; export class FlowParserProcessor extends BaseProcessor { public parsers: { [flowType in FlowTypes.FlowType]: Parsers.DefaultParser } = { diff --git a/packages/scripts/src/commands/app-data/convert/processors/flowParser/parsers/data_list.parser.spec.ts b/packages/scripts/src/commands/app-data/convert/processors/flowParser/parsers/data_list.parser.spec.ts index d810303a40..23b4a1b3cf 100644 --- a/packages/scripts/src/commands/app-data/convert/processors/flowParser/parsers/data_list.parser.spec.ts +++ b/packages/scripts/src/commands/app-data/convert/processors/flowParser/parsers/data_list.parser.spec.ts @@ -1,6 +1,6 @@ import { FlowTypes } from "data-models"; import { DataListParser } from "."; -import { TEST_DATA_PATHS, useMockLogger } from "../../../../../../../test/helpers/utils"; +import { TEST_DATA_PATHS } from "../../../../../../../test/helpers/utils"; import { FlowParserProcessor } from "../flowParser"; const MOCK_DATA_LIST = (): FlowTypes.Data_list => ({ @@ -47,7 +47,7 @@ describe("data_list Parser Metadata", () => { additional: { type: "boolean" }, }); }); - it("Assigns column metadata from metadata row", () => { + it("Assigns column metadata from metadata row (and removes it)", () => { const flowWithMetadataRow = { ...MOCK_DATA_LIST(), rows: [ @@ -55,10 +55,11 @@ describe("data_list Parser Metadata", () => { { id: "id_1", number: undefined, text: undefined }, ], }; - const { _metadata } = parser.postProcessFlow(flowWithMetadataRow); + const { _metadata, rows } = parser.postProcessFlow(flowWithMetadataRow); expect(_metadata).toEqual({ number: { type: "number" }, }); + expect(rows.length).toEqual(1); }); it("Omits metadata when all columns are strings", () => { const flowWithStringColumns = { diff --git a/packages/scripts/src/commands/app-data/convert/processors/flowParser/parsers/data_list.parser.ts b/packages/scripts/src/commands/app-data/convert/processors/flowParser/parsers/data_list.parser.ts index 8bdc2f4bd4..dd38d3f0ee 100644 --- a/packages/scripts/src/commands/app-data/convert/processors/flowParser/parsers/data_list.parser.ts +++ b/packages/scripts/src/commands/app-data/convert/processors/flowParser/parsers/data_list.parser.ts @@ -35,11 +35,8 @@ export class DataListParser extends DefaultParser { } public override postProcessFlow(flow: FlowTypes.Data_list) { - const metadata = this.getFlowMetadata(flow); - if (!isEmptyObjectDeep(metadata)) { - flow._metadata = metadata; - } - return flow; + const flowWithMetadata = this.getFlowMetadata(flow); + return flowWithMetadata; } public postProcessFlows(flows: FlowTypes.Data_list[]) { @@ -47,15 +44,23 @@ export class DataListParser extends DefaultParser { return flowsWithOverrides; } - /** Assign column metadata from @metadata row if provided, or infer from data if not*/ - private getFlowMetadata(flow: FlowTypes.FlowTypeWithData) { + /** Assign column metadata from @metadata row if provided, or infer from data if not */ + private getFlowMetadata(flow: FlowTypes.Data_list) { const [firstRow] = flow.rows; - const initialMetadata = - firstRow?.id === "@metadata" - ? this.assignMetadataFromRow(firstRow) - : this.inferMetadataFromData(flow.rows); + let metadataInitial: FlowTypes.Data_list["_metadata"]; + // assign from metadata row and remove + if (firstRow?.id === "@metadata") { + metadataInitial = this.assignMetadataFromRow(firstRow); + flow.rows = flow.rows.slice(0, 1); + } + // infer from data when metadata row not available + else { + metadataInitial = this.inferMetadataFromData(flow.rows); + } + // quality check metadata and assign cleaned meta to flow + const { warnings, metadata } = this.qualityCheckMetadata(metadataInitial); + flow._metadata = metadata; - const { metadata, warnings } = this.qualityCheckMetadata(initialMetadata); if (warnings.length > 0) { for (const warning of warnings) { console.warn(warning); @@ -65,7 +70,7 @@ export class DataListParser extends DefaultParser { msg2: flow._xlsxPath, }); } - return metadata; + return flow; } /** Assign data_list metadata using a first `@metadata` row */