From 326963c8e1420e5af025acc651c10b2ab5f515f3 Mon Sep 17 00:00:00 2001 From: Dominik Haentsch Date: Mon, 9 Oct 2023 11:16:57 +0200 Subject: [PATCH 1/4] sort h5 column names by order --- renumics/spotlight_plugins/core/hdf5_data_source.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/renumics/spotlight_plugins/core/hdf5_data_source.py b/renumics/spotlight_plugins/core/hdf5_data_source.py index 2acc2678..8dc20687 100644 --- a/renumics/spotlight_plugins/core/hdf5_data_source.py +++ b/renumics/spotlight_plugins/core/hdf5_data_source.py @@ -103,7 +103,13 @@ def __del__(self) -> None: @property def column_names(self) -> List[str]: - return self._table.keys() + column_names = self._table.keys() + orders = { + name: self._table.get_column_attributes(name).get("order") or -1 + for name in column_names + } + column_names.sort(key=lambda name: orders[name], reverse=True) + return column_names @property def intermediate_dtypes(self) -> DTypeMap: From 758854db0b781715a35934bef6ba09b5ac1885e8 Mon Sep 17 00:00:00 2001 From: Dominik Haentsch Date: Mon, 9 Oct 2023 11:17:27 +0200 Subject: [PATCH 2/4] dont sort cols alphanumerically after fetch --- src/lib.ts | 2 +- src/stores/dataset/dataset.ts | 13 +------------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/src/lib.ts b/src/lib.ts index aa734dea..5684c279 100644 --- a/src/lib.ts +++ b/src/lib.ts @@ -57,7 +57,7 @@ export { export { default as api } from './api'; export type { App } from './stores/pluginStore'; -export { useDataset, convertValue, compareColumnOrder } from './stores/dataset'; +export { useDataset, convertValue } from './stores/dataset'; export { useColors } from './stores/colors'; export { isAudio, isBoolean, getNullValue, isCategorical } from './datatypes'; export type { diff --git a/src/stores/dataset/dataset.ts b/src/stores/dataset/dataset.ts index 2124706f..9c820676 100644 --- a/src/stores/dataset/dataset.ts +++ b/src/stores/dataset/dataset.ts @@ -106,15 +106,6 @@ export function convertValue(value: any, type: DataType) { return value; } -export function compareColumnOrder(a: DataColumn, b: DataColumn) { - if (a.isInternal && !b.isInternal) { - return 1; - } else if (b.isInternal && !a.isInternal) { - return -1; - } - return a.name.localeCompare(b.name); -} - const fetchTable = async (): Promise<{ uid: string; generationID: number; @@ -164,10 +155,8 @@ const fetchTable = async (): Promise<{ const length = _.max(table.columns.map((col) => col.values?.length ?? 0)) ?? 0; - const sortedColumns = columns.sort(compareColumnOrder); - const dataframe: DataFrame = { - columns: sortedColumns, + columns, length, data: columnData, }; From b799e2566386c409ea7e5f2f8dcd186b72645342 Mon Sep 17 00:00:00 2001 From: Dominik Haentsch Date: Mon, 9 Oct 2023 11:26:27 +0200 Subject: [PATCH 3/4] remove isInternal from frontend code --- .../dataset/colorTransferFunctionFactory.tsx | 6 +--- src/stores/dataset/columnFactory.ts | 5 --- src/stores/dataset/dataset.ts | 5 +-- src/types/dataset.ts | 1 - src/widgets/Histogram/Histogram.tsx | 2 +- .../ScatterplotView/ScatterplotView.tsx | 24 ++++---------- src/widgets/SimilarityMap/MenuBar.tsx | 32 ++++++------------- src/widgets/SimilarityMap/SimilarityMap.tsx | 15 +++------ 8 files changed, 23 insertions(+), 67 deletions(-) diff --git a/src/stores/dataset/colorTransferFunctionFactory.tsx b/src/stores/dataset/colorTransferFunctionFactory.tsx index 65ed493b..f918ec43 100644 --- a/src/stores/dataset/colorTransferFunctionFactory.tsx +++ b/src/stores/dataset/colorTransferFunctionFactory.tsx @@ -64,11 +64,7 @@ export const makeColumnsColorTransferFunctions = ( filteredMask: boolean[] ): ColumnsTransferFunctions => { return columns - .filter( - (column) => - !column.isInternal && - (isScalarColumn(column) || isCategoricalColumn(column)) - ) + .filter((column) => isScalarColumn(column) || isCategoricalColumn(column)) .reduce((a, column) => { a[column.key] = { full: makeApplicableColorTransferFunctions( diff --git a/src/stores/dataset/columnFactory.ts b/src/stores/dataset/columnFactory.ts index 26018d58..31f44039 100644 --- a/src/stores/dataset/columnFactory.ts +++ b/src/stores/dataset/columnFactory.ts @@ -66,11 +66,6 @@ export function makeColumn(column: Column, index: number): DataColumn { optional: column.optional, description: column.description ?? '', tags: _.uniq(column.tags), - - // we access some internal columns like __id__ by their name - // therefore, if we set the key to something different than column.name - // we have to check for isInternal and use column.name for it - isInternal: column.name.startsWith('__'), }; return col; diff --git a/src/stores/dataset/dataset.ts b/src/stores/dataset/dataset.ts index 9c820676..0cd3e3c0 100644 --- a/src/stores/dataset/dataset.ts +++ b/src/stores/dataset/dataset.ts @@ -423,10 +423,7 @@ export const useDataset = create( }, recomputeColorTransferFunctions: async () => { const columnsToCompute = get() - .columns.filter( - (c) => - !c.isInternal && (isScalar(c.type) || isCategorical(c.type)) - ) + .columns.filter((c) => isScalar(c.type) || isCategorical(c.type)) .map((c) => c.key); const newTransferFunctions = makeColumnsColorTransferFunctions( diff --git a/src/types/dataset.ts b/src/types/dataset.ts index 3947d99a..5c0cd41c 100644 --- a/src/types/dataset.ts +++ b/src/types/dataset.ts @@ -8,7 +8,6 @@ export interface DataColumn { type: datatypes.DataType; editable: boolean; optional: boolean; - isInternal: boolean; description: string; tags: string[]; } diff --git a/src/widgets/Histogram/Histogram.tsx b/src/widgets/Histogram/Histogram.tsx index 2f69e881..2344033d 100644 --- a/src/widgets/Histogram/Histogram.tsx +++ b/src/widgets/Histogram/Histogram.tsx @@ -35,7 +35,7 @@ const Histogram: Widget = () => { const columnKeys = useMemo( () => columns - .filter((col) => !col.isInternal && validTypes.includes(col.type.kind)) + .filter((col) => validTypes.includes(col.type.kind)) .map((col) => col.key), [columns] ); diff --git a/src/widgets/ScatterplotView/ScatterplotView.tsx b/src/widgets/ScatterplotView/ScatterplotView.tsx index 1c6a54c0..3e9dd600 100644 --- a/src/widgets/ScatterplotView/ScatterplotView.tsx +++ b/src/widgets/ScatterplotView/ScatterplotView.tsx @@ -287,10 +287,7 @@ const ScatterplotView: Widget = () => { // compute z-scores for all number columns and order them descending const remainingColumns = Object.values(allColumns).filter( - (col) => - isNumberColumn(col) && - !col.isInternal && - !defaultColumns.includes(col.key) + (col) => isNumberColumn(col) && !defaultColumns.includes(col.key) ) as NumberColumn[]; const interestingColumns = sortColumnsByZScore( rowIndex, @@ -327,22 +324,15 @@ const ScatterplotView: Widget = () => { const placeableColumns = useMemo( () => Object.values(allColumns) - .filter( - (col) => - ['int', 'float', 'bool'].includes(col.type.kind) && - !col.isInternal - ) + .filter((col) => ['int', 'float', 'bool'].includes(col.type.kind)) .map((col) => col.key), [allColumns] ); const colorableColumns = useMemo( () => Object.values(allColumns) - .filter( - (col) => - ['int', 'float', 'str', 'bool', 'Category'].includes( - col.type.kind - ) && !col.isInternal + .filter((col) => + ['int', 'float', 'str', 'bool', 'Category'].includes(col.type.kind) ) .map((col) => col.key), [allColumns] @@ -350,10 +340,8 @@ const ScatterplotView: Widget = () => { const scaleableColumns = useMemo( () => Object.values(allColumns) - .filter( - (col: DataColumn) => - ['int', 'float', 'bool'].includes(col.type.kind) && - !col.isInternal + .filter((col: DataColumn) => + ['int', 'float', 'bool'].includes(col.type.kind) ) .map((col) => col.key), [allColumns] diff --git a/src/widgets/SimilarityMap/MenuBar.tsx b/src/widgets/SimilarityMap/MenuBar.tsx index b430d7d0..e0767116 100644 --- a/src/widgets/SimilarityMap/MenuBar.tsx +++ b/src/widgets/SimilarityMap/MenuBar.tsx @@ -50,12 +50,6 @@ const columnTypeSelector = (d: Dataset): { [key: string]: DataType } => return a; }, {}); -const columnIsInternalSelector = (d: Dataset): { [key: string]: boolean } => - d.columns.reduce((a: { [key: string]: boolean }, c: DataColumn) => { - a[c.key] = c.isInternal; - return a; - }, {}); - const SettingsMenu = ({ colorBy, sizeBy, @@ -77,7 +71,6 @@ const SettingsMenu = ({ onChangePCANormalization, }: Props) => { const columnType = useDataset(columnTypeSelector, shallow); - const columnIsInternal = useDataset(columnIsInternalSelector, shallow); const changePlaceBy = useCallback( (keys: string[]): void => { @@ -88,27 +81,20 @@ const SettingsMenu = ({ const colorableColumns = useMemo( () => - Object.entries(columnType) - .filter( - ([key, type]) => - ['int', 'float', 'str', 'bool', 'Category'].includes( - type.kind - ) && !columnIsInternal[key] + Object.values(columnType) + .filter((type) => + ['int', 'float', 'str', 'bool', 'Category'].includes(type.kind) ) - .map(([key]) => key), - [columnIsInternal, columnType] + .map((type) => type.kind), + [columnType] ); const scaleableColumns = useMemo( () => - Object.entries(columnType) - .filter( - ([key, type]) => - ['int', 'float', 'bool'].includes(type.kind) && - !columnIsInternal[key] - ) - .map(([key]) => key), - [columnIsInternal, columnType] + Object.values(columnType) + .filter((type) => ['int', 'float', 'bool'].includes(type.kind)) + .map((type) => type.kind), + [columnType] ); const reductionParameterMenu = diff --git a/src/widgets/SimilarityMap/SimilarityMap.tsx b/src/widgets/SimilarityMap/SimilarityMap.tsx index b9d72695..cd76ccd1 100644 --- a/src/widgets/SimilarityMap/SimilarityMap.tsx +++ b/src/widgets/SimilarityMap/SimilarityMap.tsx @@ -124,7 +124,6 @@ const SimilarityMap: Widget = () => { return undefined; } const availableColumns = fullColumns - .filter((col) => !col.isInternal) .filter((col) => ['int', 'float', 'str', 'bool', 'Category'].includes(col.type.kind) ) @@ -165,11 +164,10 @@ const SimilarityMap: Widget = () => { const embeddableColumnKeys = useMemo(() => { return fullColumns - .filter( - (col) => - ['int', 'bool', 'float', 'Category', 'Embedding'].includes( - col.type.kind - ) && !col.isInternal + .filter((col) => + ['int', 'bool', 'float', 'Category', 'Embedding'].includes( + col.type.kind + ) ) .map((c) => c.key); }, [fullColumns]); @@ -355,10 +353,7 @@ const SimilarityMap: Widget = () => { // compute z-scores for all number columns and order them descending // const remainingColumns = fullColumns.filter( - (col) => - isNumberColumn(col) && - !col.isInternal && - !defaultColumns.includes(col) + (col) => isNumberColumn(col) && !defaultColumns.includes(col) ) as NumberColumn[]; const interestingColumns = sortColumnsByZScore( rowIndex, From 0c530022b295b337665a004ae4097f4fd271faea Mon Sep 17 00:00:00 2001 From: Dominik Haentsch Date: Mon, 9 Oct 2023 11:34:29 +0200 Subject: [PATCH 4/4] remove isInternal from histogram --- src/widgets/Histogram/Histogram.tsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/widgets/Histogram/Histogram.tsx b/src/widgets/Histogram/Histogram.tsx index ba50f6d4..99feb64e 100644 --- a/src/widgets/Histogram/Histogram.tsx +++ b/src/widgets/Histogram/Histogram.tsx @@ -43,9 +43,7 @@ const Histogram: Widget = () => { const defaultColumnKey = useMemo( () => columns - .filter( - (col) => !col.isInternal && defaultTypes.includes(col.type.kind) - ) + .filter((col) => defaultTypes.includes(col.type.kind)) .map((col) => col.key)[0], [columns] );