diff --git a/web-common/src/features/dashboards/pivot/pivot-data-store.ts b/web-common/src/features/dashboards/pivot/pivot-data-store.ts index 45610807756..912cf0179e7 100644 --- a/web-common/src/features/dashboards/pivot/pivot-data-store.ts +++ b/web-common/src/features/dashboards/pivot/pivot-data-store.ts @@ -36,6 +36,7 @@ import { getTimeGrainFromDimension, getTotalColumnCount, isTimeDimension, + reconcileMissingDimensionValues, } from "./pivot-utils"; import { PivotChipType, @@ -150,7 +151,6 @@ export function createTableCellQuery( }; } else return { name: dimension }; }); - const measureBody = config.measureNames.map((m) => ({ name: m })); const { filters: filterForInitialTable, timeFilters } = getFilterForPivotTable( @@ -174,7 +174,7 @@ export function createTableCellQuery( ]; return createPivotAggregationRowQuery( ctx, - measureBody, + config.measureNames, dimensionBody, mergedFilter, config.measureFilter, @@ -235,6 +235,7 @@ function createPivotDataStore(ctx: StateManagers): PivotDataStore { */ return derived(getPivotConfig(ctx), (config, configSet) => { const { rowDimensionNames, colDimensionNames, measureNames } = config; + if ( (!rowDimensionNames.length && !measureNames.length) || (colDimensionNames.length && !measureNames.length) @@ -251,13 +252,10 @@ function createPivotDataStore(ctx: StateManagers): PivotDataStore { totalColumns: 0, }); } - const measureBody = measureNames.map((m) => ({ name: m })); - const columnDimensionAxesQuery = getAxisForDimensions( ctx, config, colDimensionNames, - measureBody, config.whereFilter, ); @@ -275,35 +273,32 @@ function createPivotDataStore(ctx: StateManagers): PivotDataStore { } const anchorDimension = rowDimensionNames[0]; - const { - where: measureWhere, - sortPivotBy, - timeRange, - } = getSortForAccessor( + const { where, sortPivotBy, timeRange } = getSortForAccessor( anchorDimension, config, columnDimensionAxes?.data, ); - let sortFilteredMeasureBody = measureBody; - if (sortPivotBy.length && measureWhere) { - const accessor = sortPivotBy[0]?.name; - sortFilteredMeasureBody = measureBody.map((m) => { - if (m.name === accessor) return { ...m, filter: measureWhere }; - return m; - }); - } - const rowDimensionAxisQuery = getAxisForDimensions( ctx, config, rowDimensionNames.slice(0, 1), - sortFilteredMeasureBody, - config.whereFilter, + where, sortPivotBy, timeRange, ); + /** + * We need to query the unsorted row dimension values because the sorted + * row dimension values may not have all the dimensions values + */ + const rowDimensionUnsortedAxisQuery = getAxisForDimensions( + ctx, + config, + rowDimensionNames.slice(0, 1), + config.whereFilter, + ); + let globalTotalsQuery: | Readable | CreateQueryResult = @@ -315,7 +310,7 @@ function createPivotDataStore(ctx: StateManagers): PivotDataStore { if (rowDimensionNames.length && measureNames.length) { globalTotalsQuery = createPivotAggregationRowQuery( ctx, - config.measureNames.map((m) => ({ name: m })), + config.measureNames, [], config.whereFilter, config.measureFilter, @@ -338,16 +333,27 @@ function createPivotDataStore(ctx: StateManagers): PivotDataStore { * Derive a store from axes queries */ return derived( - [rowDimensionAxisQuery, globalTotalsQuery, totalsRowQuery], + [ + rowDimensionAxisQuery, + rowDimensionUnsortedAxisQuery, + globalTotalsQuery, + totalsRowQuery, + ], ( - [rowDimensionAxes, globalTotalsResponse, totalsRowResponse], + [ + rowDimensionAxes, + rowDimensionUnsortedAxis, + globalTotalsResponse, + totalsRowResponse, + ], axesSet, ) => { if ( (globalTotalsResponse !== null && globalTotalsResponse?.isFetching) || (totalsRowResponse !== null && totalsRowResponse?.isFetching) || - rowDimensionAxes?.isFetching + rowDimensionAxes?.isFetching || + rowDimensionUnsortedAxis?.isFetching ) { return axesSet({ isFetching: true, @@ -358,9 +364,6 @@ function createPivotDataStore(ctx: StateManagers): PivotDataStore { }); } - const rowDimensionValues = - rowDimensionAxes?.data?.[anchorDimension] || []; - const rowTotals = rowDimensionAxes?.totals?.[anchorDimension] || []; const totalsRow = getTotalsRow( config, columnDimensionAxes?.data, @@ -370,6 +373,13 @@ function createPivotDataStore(ctx: StateManagers): PivotDataStore { const totalColumns = getTotalColumnCount(totalsRow); + const { rows: rowDimensionValues, totals: rowTotals } = + reconcileMissingDimensionValues( + anchorDimension, + rowDimensionAxes, + rowDimensionUnsortedAxis, + ); + let initialTableCellQuery: | Readable | CreateQueryResult = diff --git a/web-common/src/features/dashboards/pivot/pivot-expansion.ts b/web-common/src/features/dashboards/pivot/pivot-expansion.ts index d7ae1b480e0..2c997941a91 100644 --- a/web-common/src/features/dashboards/pivot/pivot-expansion.ts +++ b/web-common/src/features/dashboards/pivot/pivot-expansion.ts @@ -86,7 +86,6 @@ export function createSubTableCellQuery( }; } else return { name: dimension }; }); - const measureBody = config.measureNames.map((m) => ({ name: m })); const { filters: filterForSubTable, timeFilters: colTimeFilters } = getFilterForPivotTable(config, columnDimensionAxesData, totalsRow); @@ -106,7 +105,7 @@ export function createSubTableCellQuery( ]; return createPivotAggregationRowQuery( ctx, - measureBody, + config.measureNames, dimensionBody, filterForSubTable, config.measureFilter, @@ -141,7 +140,6 @@ export function queryExpandedRowMeasureValues( const expanded = config.pivot.expanded; if (!tableData || Object.keys(expanded).length == 0) return writable(null); - const measureBody = config.measureNames.map((m) => ({ name: m })); return derived( Object.keys(expanded)?.map((expandIndex) => { const nestLevel = expandIndex?.split(".")?.length; @@ -227,7 +225,6 @@ export function queryExpandedRowMeasureValues( ctx, config, [anchorDimension], - measureBody, allMergedFilters, sortPivotBy, timeRange, diff --git a/web-common/src/features/dashboards/pivot/pivot-queries.ts b/web-common/src/features/dashboards/pivot/pivot-queries.ts index 024caa4931f..a504b990205 100644 --- a/web-common/src/features/dashboards/pivot/pivot-queries.ts +++ b/web-common/src/features/dashboards/pivot/pivot-queries.ts @@ -18,7 +18,6 @@ import type { TimeRangeString } from "@rilldata/web-common/lib/time/types"; import { V1Expression, V1MetricsViewAggregationDimension, - V1MetricsViewAggregationMeasure, V1MetricsViewAggregationResponseDataItem, V1MetricsViewAggregationSort, createQueryServiceMetricsViewAggregation, @@ -33,7 +32,7 @@ import { mergeFilters } from "./pivot-merge-filters"; */ export function createPivotAggregationRowQuery( ctx: StateManagers, - measures: V1MetricsViewAggregationMeasure[], + measures: string[], dimensions: V1MetricsViewAggregationDimension[], whereFilter: V1Expression, measureFilter: ResolvedMeasureFilter | undefined, @@ -46,7 +45,7 @@ export function createPivotAggregationRowQuery( sort = [ { desc: false, - name: measures[0]?.name || dimensions?.[0]?.name, + name: measures[0] || dimensions?.[0]?.name, }, ]; } @@ -58,7 +57,7 @@ export function createPivotAggregationRowQuery( runtime.instanceId, metricViewName, { - measures, + measures: measures.map((measure) => ({ name: measure })), dimensions, where: sanitiseExpression(whereFilter, measureFilter?.filter), timeRange: { @@ -87,13 +86,13 @@ export function getAxisForDimensions( ctx: StateManagers, config: PivotDataStoreConfig, dimensions: string[], - measures: V1MetricsViewAggregationMeasure[], whereFilter: V1Expression, sortBy: V1MetricsViewAggregationSort[] = [], timeRange: TimeRangeString | undefined = undefined, ): Readable { if (!dimensions.length) return readable(null); + const measures = config.measureNames; const { time } = config; let sortProvided = true; @@ -101,7 +100,7 @@ export function getAxisForDimensions( sortBy = [ { desc: true, - name: measures[0]?.name || dimensions?.[0], + name: measures[0] || dimensions?.[0], }, ]; sortProvided = false; @@ -181,7 +180,6 @@ export function getTotalsRowQuery( const { colDimensionNames } = config; const { time } = config; - const measureBody = config.measureNames.map((m) => ({ name: m })); const dimensionBody = colDimensionNames.map((dimension) => { if (isTimeDimension(dimension, time.timeDimension)) { return { @@ -212,7 +210,7 @@ export function getTotalsRowQuery( ]; return createPivotAggregationRowQuery( ctx, - measureBody, + config.measureNames, dimensionBody, mergedFilter, config.measureFilter, diff --git a/web-common/src/features/dashboards/pivot/pivot-utils.ts b/web-common/src/features/dashboards/pivot/pivot-utils.ts index 442ab657440..0f31caaa079 100644 --- a/web-common/src/features/dashboards/pivot/pivot-utils.ts +++ b/web-common/src/features/dashboards/pivot/pivot-utils.ts @@ -1,3 +1,4 @@ +import { mergeFilters } from "@rilldata/web-common/features/dashboards/pivot/pivot-merge-filters"; import { createAndExpression, createInExpression, @@ -16,12 +17,72 @@ import type { } from "@rilldata/web-common/runtime-client"; import { getColumnFiltersForPage } from "./pivot-infinite-scroll"; import type { + PivotAxesData, PivotDataRow, PivotDataStoreConfig, PivotTimeConfig, TimeFilters, } from "./types"; +/** + * Returns a sorted data array by appending the missing values in + * sorted row axes data + */ +export function reconcileMissingDimensionValues( + anchorDimension: string, + sortedRowAxes: PivotAxesData | null, + unsortedRowAxes: PivotAxesData | null, +) { + // Return empty data if either sortedRowAxes or unsortedRowAxes is null + if (!sortedRowAxes || !unsortedRowAxes) { + return { rows: [], totals: [] }; + } + + // Extract data and totals from sortedRowAxes + const sortedRowAxisValues = sortedRowAxes.data?.[anchorDimension] || []; + const sortedTotals = sortedRowAxes.totals?.[anchorDimension] || []; + + // Return early if there are too many values + if (sortedRowAxisValues.length >= 100) { + return { + rows: sortedRowAxisValues.slice(0, 100), + totals: sortedTotals.slice(0, 100), + }; + } + + // Extract data and totals from unsortedRowAxes + const unsortedRowAxisValues = unsortedRowAxes.data?.[anchorDimension] || []; + const unsortedTotals = unsortedRowAxes.totals?.[anchorDimension] || []; + + // Find missing values that are in unsortedRowAxes but not in sortedRowAxes + const missingValues = unsortedRowAxisValues.filter( + (value) => !sortedRowAxisValues.includes(value), + ); + + // Combine and limit the rows to 100 + const combinedRows = [...sortedRowAxisValues, ...missingValues].slice(0, 100); + + // Reorder the totals to match the order of combinedRows + const reorderedTotals = combinedRows.map((rowValue) => { + const sortedTotal = sortedTotals.find( + (total) => total[anchorDimension] === rowValue, + ); + if (sortedTotal) { + return sortedTotal; + } + // Use the total from unsortedRowAxes if not found in sortedTotals + const unsortedTotal = unsortedTotals.find( + (total) => total[anchorDimension] === rowValue, + ); + return unsortedTotal || { [anchorDimension]: rowValue }; + }); + + return { + rows: combinedRows, + totals: reorderedTotals, + }; +} + /** * Construct a key for a pivot config to store expanded table data * in the cache @@ -272,7 +333,7 @@ export function getSortForAccessor( config: PivotDataStoreConfig, columnDimensionAxes: Record = {}, ): { - where?: V1Expression; + where: V1Expression; sortPivotBy: V1MetricsViewAggregationSort[]; timeRange: TimeRangeString; } { @@ -286,6 +347,7 @@ export function getSortForAccessor( // Return un-changed filter if no sorting is applied if (config.pivot?.sorting?.length === 0) { return { + where: config.whereFilter, sortPivotBy, timeRange: defaultTimeRange, }; @@ -304,6 +366,7 @@ export function getSortForAccessor( }, ]; return { + where: config.whereFilter, sortPivotBy, timeRange: defaultTimeRange, }; @@ -318,6 +381,7 @@ export function getSortForAccessor( }, ]; return { + where: config.whereFilter, sortPivotBy, timeRange: defaultTimeRange, }; @@ -359,10 +423,9 @@ export function getSortForAccessor( }); } - let filterForSort: V1Expression | undefined; - if (colDimensionFilters.length) { - filterForSort = createAndExpression(colDimensionFilters); - } + const filterForSort: V1Expression = createAndExpression(colDimensionFilters); + const mergedFilter = mergeFilters(config.whereFilter, filterForSort); + const timeRange: TimeRangeString = getTimeForQuery(config.time, timeFilters); sortPivotBy = [ @@ -373,7 +436,7 @@ export function getSortForAccessor( ]; return { - where: filterForSort, + where: mergedFilter, sortPivotBy, timeRange, };