From 3facbf1a464c29c18ea611682b2d794d90c751c0 Mon Sep 17 00:00:00 2001 From: Cedric van Putten Date: Fri, 5 Apr 2024 15:04:56 +0200 Subject: [PATCH] refactor: simplify module filters and api endpoints --- .../api/stats/[entry]/modules/graph+api.ts | 46 ++------ .../api/stats/[entry]/modules/index+api.ts | 49 +++----- .../src/app/stats/[entry]/folders/[path].tsx | 12 +- webui/src/app/stats/[entry]/index.tsx | 12 +- .../components/forms/StatsModuleFilter.tsx | 50 ++------- webui/src/utils/__tests__/search.test.ts | 94 ---------------- webui/src/utils/filters.ts | 106 ++++++++++++++++++ webui/src/utils/search.ts | 44 -------- 8 files changed, 149 insertions(+), 264 deletions(-) delete mode 100644 webui/src/utils/__tests__/search.test.ts create mode 100644 webui/src/utils/filters.ts delete mode 100644 webui/src/utils/search.ts diff --git a/webui/src/app/api/stats/[entry]/modules/graph+api.ts b/webui/src/app/api/stats/[entry]/modules/graph+api.ts index 33b3e2e..9feaf39 100644 --- a/webui/src/app/api/stats/[entry]/modules/graph+api.ts +++ b/webui/src/app/api/stats/[entry]/modules/graph+api.ts @@ -1,8 +1,7 @@ -import { statsModuleFiltersFromUrlParams } from '~/components/forms/StatsModuleFilter'; import { getSource } from '~/utils/atlas'; -import { globFilterModules } from '~/utils/search'; +import { filterModules, moduleFiltersFromParams } from '~/utils/filters'; import { type TreemapNode, createModuleTree, finalizeModuleTree } from '~/utils/treemap'; -import type { StatsEntry, StatsModule } from '~core/data/types'; +import type { StatsEntry } from '~core/data/types'; export type ModuleGraphResponse = { data: TreemapNode; @@ -26,9 +25,15 @@ export async function GET(request: Request, params: Record<'entry', string>) { return Response.json({ error: error.message }, { status: 406 }); } + const query = new URL(request.url).searchParams; const allModules = Array.from(entry.modules.values()); - const modules = modulesMatchingFilters(request, entry, allModules); - const tree = createModuleTree(modules); + const filteredModules = filterModules(allModules, { + projectRoot: entry.projectRoot, + filters: moduleFiltersFromParams(query), + rootPath: query.get('path') || undefined, + }); + + const tree = createModuleTree(filteredModules); const response: ModuleGraphResponse = { data: finalizeModuleTree(tree), @@ -38,37 +43,10 @@ export async function GET(request: Request, params: Record<'entry', string>) { moduleFiles: entry.modules.size, }, filtered: { - moduleSize: modules.reduce((size, module) => size + module.size, 0), - moduleFiles: modules.length, + moduleSize: filteredModules.reduce((size, module) => size + module.size, 0), + moduleFiles: filteredModules.length, }, }; return Response.json(response); } - -/** - * Get and filter the modules from the stats entry based on query parameters. - * - `modules=project,node_modules` to show only project code and/or node_modules - * - `include=` to only include specific glob patterns - * - `exclude=` to only exclude specific glob patterns - * - `path=` to only show modules in a specific folder - */ -function modulesMatchingFilters( - request: Request, - entry: StatsEntry, - modules: StatsModule[] -): StatsModule[] { - const searchParams = new URL(request.url).searchParams; - - const folderRef = searchParams.get('path'); - if (folderRef) { - modules = modules.filter((module) => module.path.startsWith(folderRef)); - } - - const filters = statsModuleFiltersFromUrlParams(searchParams); - if (!filters.modules.includes('node_modules')) { - modules = modules.filter((module) => !module.package); - } - - return globFilterModules(modules, entry.projectRoot, filters); -} diff --git a/webui/src/app/api/stats/[entry]/modules/index+api.ts b/webui/src/app/api/stats/[entry]/modules/index+api.ts index 567793e..ee72aa5 100644 --- a/webui/src/app/api/stats/[entry]/modules/index+api.ts +++ b/webui/src/app/api/stats/[entry]/modules/index+api.ts @@ -1,6 +1,5 @@ -import { statsModuleFiltersFromUrlParams } from '~/components/forms/StatsModuleFilter'; import { getSource } from '~/utils/atlas'; -import { globFilterModules } from '~/utils/search'; +import { filterModules, moduleFiltersFromParams } from '~/utils/filters'; import { type StatsEntry, type StatsModule } from '~core/data/types'; /** The partial module data, when listing all available modules from a stats entry */ @@ -19,6 +18,7 @@ export type ModuleListResponse = { }; }; +/** Get all modules as simple list */ export async function GET(request: Request, params: Record<'entry', string>) { let entry: StatsEntry; @@ -28,54 +28,37 @@ export async function GET(request: Request, params: Record<'entry', string>) { return Response.json({ error: error.message }, { status: 406 }); } + const query = new URL(request.url).searchParams; const allModules = Array.from(entry.modules.values()); - const modules = modulesMatchingFilters(request, entry, allModules); + const filteredModules = filterModules(allModules, { + projectRoot: entry.projectRoot, + filters: moduleFiltersFromParams(query), + rootPath: query.get('path') || undefined, + }); const response: ModuleListResponse = { - data: modules, + data: filteredModules.map((module) => ({ + ...module, + source: undefined, + output: undefined, + })), entry: { platform: entry.platform as any, moduleSize: allModules.reduce((size, module) => size + module.size, 0), moduleFiles: entry.modules.size, }, filtered: { - moduleSize: modules.reduce((size, module) => size + module.size, 0), - moduleFiles: modules.length, + moduleSize: filteredModules.reduce((size, module) => size + module.size, 0), + moduleFiles: filteredModules.length, }, }; return Response.json(response); } -/** - * Get and filter the modules from the stats entry based on query parameters. - * - `modules=project,node_modules` to show only project code and/or node_modules - * - `include=` to only include specific glob patterns - * - `exclude=` to only exclude specific glob patterns - * - `path=` to only show modules in a specific folder - */ -function modulesMatchingFilters( - request: Request, - entry: StatsEntry, - modules: StatsModule[] -): StatsModule[] { - const searchParams = new URL(request.url).searchParams; - - const folderRef = searchParams.get('path'); - if (folderRef) { - modules = modules.filter((module) => module.path.startsWith(folderRef)); - } - - const filters = statsModuleFiltersFromUrlParams(searchParams); - if (!filters.modules.includes('node_modules')) { - modules = modules.filter((module) => !module.package); - } - - return globFilterModules(modules, entry.projectRoot, filters); -} - /** * Get the full module information through a post request. + * This requires a `path` property in the request body. */ export async function POST(request: Request, params: Record<'entry', string>) { const moduleRef: string | undefined = (await request.json()).path; diff --git a/webui/src/app/stats/[entry]/folders/[path].tsx b/webui/src/app/stats/[entry]/folders/[path].tsx index 3d845c5..4dcc0aa 100644 --- a/webui/src/app/stats/[entry]/folders/[path].tsx +++ b/webui/src/app/stats/[entry]/folders/[path].tsx @@ -4,22 +4,18 @@ import { useLocalSearchParams } from 'expo-router'; import type { ModuleGraphResponse } from '~/app/api/stats/[entry]/modules/graph+api'; import { BundleGraph } from '~/components/BundleGraph'; import { Page, PageHeader, PageTitle } from '~/components/Page'; -import { - ModuleFilters, - StatsModuleFilter, - statsModuleFiltersToUrlParams, - useStatsModuleFilters, -} from '~/components/forms/StatsModuleFilter'; +import { StatsModuleFilter } from '~/components/forms/StatsModuleFilter'; import { useStatsEntry } from '~/providers/stats'; import { Tag } from '~/ui/Tag'; import { fetchApi } from '~/utils/api'; +import { type ModuleFilters, useModuleFilters, moduleFiltersToParams } from '~/utils/filters'; import { formatFileSize } from '~/utils/formatString'; import { relativeEntryPath } from '~/utils/stats'; export default function FolderPage() { const { path: absolutePath } = useLocalSearchParams<{ path: string }>(); const { entry } = useStatsEntry(); - const { filters, filtersEnabled } = useStatsModuleFilters(); + const { filters, filtersEnabled } = useModuleFilters(); const modules = useModuleGraphDataInFolder(entry.id, absolutePath!, filters); const treeHasData = !!modules.data?.data?.children?.length; @@ -91,7 +87,7 @@ function useModuleGraphDataInFolder(entryId: string, path: string, filters: Modu ModuleFilters | undefined, ]; const url = filters - ? `/api/stats/${entry}/modules/graph?path=${encodeURIComponent(path)}&${statsModuleFiltersToUrlParams(filters)}` + ? `/api/stats/${entry}/modules/graph?path=${encodeURIComponent(path)}&${moduleFiltersToParams(filters)}` : `/api/stats/${entry}/modules/graph?path=${encodeURIComponent(path)}`; return fetchApi(url) diff --git a/webui/src/app/stats/[entry]/index.tsx b/webui/src/app/stats/[entry]/index.tsx index 841705d..7f2a2f7 100644 --- a/webui/src/app/stats/[entry]/index.tsx +++ b/webui/src/app/stats/[entry]/index.tsx @@ -3,21 +3,17 @@ import { keepPreviousData, useQuery } from '@tanstack/react-query'; import type { ModuleGraphResponse } from '~/app/api/stats/[entry]/modules/graph+api'; import { BundleGraph } from '~/components/BundleGraph'; import { Page, PageHeader, PageTitle } from '~/components/Page'; -import { - type ModuleFilters, - StatsModuleFilter, - statsModuleFiltersToUrlParams, - useStatsModuleFilters, -} from '~/components/forms/StatsModuleFilter'; +import { StatsModuleFilter } from '~/components/forms/StatsModuleFilter'; import { useStatsEntry } from '~/providers/stats'; import { Spinner } from '~/ui/Spinner'; import { Tag } from '~/ui/Tag'; import { fetchApi } from '~/utils/api'; +import { type ModuleFilters, moduleFiltersToParams, useModuleFilters } from '~/utils/filters'; import { formatFileSize } from '~/utils/formatString'; export default function StatsPage() { const { entry } = useStatsEntry(); - const { filters, filtersEnabled } = useStatsModuleFilters(); + const { filters, filtersEnabled } = useModuleFilters(); const modules = useModuleGraphData(entry.id, filters); const treeHasData = !!modules.data?.data?.children?.length; @@ -77,7 +73,7 @@ function useModuleGraphData(entryId: string, filters: ModuleFilters) { queryFn: ({ queryKey }) => { const [_key, entry, filters] = queryKey as [string, string, ModuleFilters | undefined]; const url = filters - ? `/api/stats/${entry}/modules/graph?${statsModuleFiltersToUrlParams(filters)}` + ? `/api/stats/${entry}/modules/graph?${moduleFiltersToParams(filters)}` : `/api/stats/${entry}/modules/graph`; return fetchApi(url) diff --git a/webui/src/components/forms/StatsModuleFilter.tsx b/webui/src/components/forms/StatsModuleFilter.tsx index 9a90380..60ce905 100644 --- a/webui/src/components/forms/StatsModuleFilter.tsx +++ b/webui/src/components/forms/StatsModuleFilter.tsx @@ -1,4 +1,4 @@ -import { useGlobalSearchParams, useRouter } from 'expo-router'; +import { useRouter } from 'expo-router'; import { type FormEvent, type KeyboardEvent, useState, useCallback } from 'react'; import { Button } from '~/ui/Button'; @@ -14,26 +14,7 @@ import { SheetTrigger, } from '~/ui/Sheet'; import { debounce } from '~/utils/debounce'; - -export type ModuleFilters = typeof DEFAULT_FILTERS; - -const DEFAULT_FILTERS = { - modules: 'project,node_modules', - include: '', - exclude: '', -}; - -export function useStatsModuleFilters(): { filters: ModuleFilters; filtersEnabled: boolean } { - const filters = useGlobalSearchParams>(); - return { - filtersEnabled: !!filters.modules || !!filters.include || !!filters.exclude, - filters: { - modules: filters.modules || DEFAULT_FILTERS.modules, - include: filters.include || DEFAULT_FILTERS.include, - exclude: filters.exclude || DEFAULT_FILTERS.exclude, - }, - }; -} +import { useModuleFilters } from '~/utils/filters'; type StatsModuleFilterProps = { disableNodeModules?: boolean; @@ -41,7 +22,7 @@ type StatsModuleFilterProps = { export function StatsModuleFilter(props: StatsModuleFilterProps) { const router = useRouter(); - const { filters, filtersEnabled } = useStatsModuleFilters(); + const { filters, filtersEnabled } = useModuleFilters(); // NOTE(cedric): we want to programmatically close the dialog when the form is submitted, so make it controlled const [dialogOpen, setDialogOpen] = useState(false); @@ -59,9 +40,9 @@ export function StatsModuleFilter(props: StatsModuleFilterProps) { } } - const onModuleChange = useCallback((includeNodeModules: boolean) => { + const onModuleChange = useCallback((withNodeModules: boolean) => { router.setParams({ - modules: includeNodeModules ? undefined : 'project', + scope: withNodeModules ? undefined : 'project', }); }, []); @@ -82,7 +63,7 @@ export function StatsModuleFilter(props: StatsModuleFilterProps) { const onClearFilters = useCallback(() => { setDialogOpen(false); router.setParams({ - modules: undefined, + scope: undefined, include: undefined, exclude: undefined, }); @@ -108,7 +89,7 @@ export function StatsModuleFilter(props: StatsModuleFilterProps) { ); } - -export function statsModuleFiltersToUrlParams(filters: ModuleFilters) { - const params = new URLSearchParams({ modules: filters.modules }); - - if (filters.include) params.set('include', filters.include); - if (filters.exclude) params.set('exclude', filters.exclude); - - return params.toString(); -} - -export function statsModuleFiltersFromUrlParams(params: URLSearchParams): ModuleFilters { - return { - modules: params.get('modules') || DEFAULT_FILTERS.modules, - include: params.get('include') || DEFAULT_FILTERS.include, - exclude: params.get('exclude') || DEFAULT_FILTERS.exclude, - }; -} diff --git a/webui/src/utils/__tests__/search.test.ts b/webui/src/utils/__tests__/search.test.ts deleted file mode 100644 index 88c8ca2..0000000 --- a/webui/src/utils/__tests__/search.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -// Note(cedric): this file was copied from core, and isn't currently used as test. -import { describe, expect, it } from 'bun:test'; - -import { globFilterModules } from '../search'; - -import { type StatsModule } from '~core/data/types'; - -const projectRoot = '/user/expo'; -const modules = [ - asModule({ path: '/user/expo/node_modules/lodash/lodash.js' }), - asModule({ path: '/user/expo/node_modules/expo/package.json' }), - asModule({ path: '/user/expo/src/index.ts' }), - asModule({ path: '/user/expo/src/app/index.ts' }), -]; - -function asModule(module: Pick) { - return module as StatsModule; -} - -describe.skip('globFilterModules', () => { - describe('include', () => { - it('filters by exact file name', () => { - expect(globFilterModules(modules, projectRoot, { include: 'index.ts' })).toEqual([ - asModule({ path: '/user/expo/src/index.ts' }), - asModule({ path: '/user/expo/src/app/index.ts' }), - ]); - }); - - it('filters by exact directory name', () => { - expect(globFilterModules(modules, projectRoot, { include: 'node_modules' })).toEqual([ - asModule({ path: '/user/expo/node_modules/lodash/lodash.js' }), - asModule({ path: '/user/expo/node_modules/expo/package.json' }), - ]); - }); - - it('filters by multiple exact file or directory names', () => { - expect(globFilterModules(modules, projectRoot, { include: 'index.ts, lodash' })).toEqual([ - asModule({ path: '/user/expo/src/index.ts' }), - asModule({ path: '/user/expo/src/app/index.ts' }), - asModule({ path: '/user/expo/node_modules/lodash/lodash.js' }), - ]); - }); - - it('filters using star pattern on directory', () => { - expect(globFilterModules(modules, projectRoot, { include: 'src/*' })).toEqual([ - asModule({ path: '/user/expo/src/index.ts' }), - asModule({ path: '/user/expo/src/app/index.ts' }), - ]); - }); - - it('filters using star pattern on nested directory', () => { - expect(globFilterModules(modules, projectRoot, { include: 'expo/src/**' })).toEqual([ - asModule({ path: '/user/expo/src/index.ts' }), - asModule({ path: '/user/expo/src/app/index.ts' }), - ]); - }); - }); - - describe('exclude', () => { - it('filters by exact file name', () => { - expect(globFilterModules(modules, projectRoot, { exclude: 'index.ts' })).toEqual([ - asModule({ path: '/user/expo/node_modules/lodash/lodash.js' }), - asModule({ path: '/user/expo/node_modules/expo/package.json' }), - ]); - }); - - it('filters by exact directory name', () => { - expect(globFilterModules(modules, projectRoot, { exclude: 'node_modules' })).toEqual([ - asModule({ path: '/user/expo/src/index.ts' }), - asModule({ path: '/user/expo/src/app/index.ts' }), - ]); - }); - - it('filters by multiple exact file or directory names', () => { - expect(globFilterModules(modules, projectRoot, { exclude: 'index.ts, lodash' })).toEqual([ - asModule({ path: '/user/expo/node_modules/expo/package.json' }), - ]); - }); - - it('filters using star pattern on directory', () => { - expect(globFilterModules(modules, projectRoot, { exclude: 'src/*' })).toEqual([ - asModule({ path: '/user/expo/node_modules/lodash/lodash.js' }), - asModule({ path: '/user/expo/node_modules/expo/package.json' }), - ]); - }); - - it('filters using star pattern on nested directory', () => { - expect(globFilterModules(modules, projectRoot, { exclude: 'expo/src/**' })).toEqual([ - asModule({ path: '/user/expo/node_modules/lodash/lodash.js' }), - asModule({ path: '/user/expo/node_modules/expo/package.json' }), - ]); - }); - }); -}); diff --git a/webui/src/utils/filters.ts b/webui/src/utils/filters.ts new file mode 100644 index 0000000..048580a --- /dev/null +++ b/webui/src/utils/filters.ts @@ -0,0 +1,106 @@ +import { useGlobalSearchParams } from 'expo-router'; +import path from 'path'; +import picomatch from 'picomatch'; + +import { type StatsModule } from '~core/data/types'; + +export type ModuleFilters = { + /** Only match the project code, or all code including (external) packages */ + scope?: 'project'; + /** Include results based on comma separated glob patterns */ + include?: string; + /** Exclude results based on comma separated glob patterns */ + exclude?: string; +}; + +/** The default filters to use */ +export const DEFAULT_FILTERS: ModuleFilters = { + scope: undefined, + include: undefined, + exclude: undefined, +}; + +/** + * Get the module filters based on query parameters. + * - `modules=project,node_modules` to show only project code and/or node_modules + * - `include=` to only include specific glob patterns + * - `exclude=` to only exclude specific glob patterns + */ +export function moduleFiltersFromParams(params: URLSearchParams): ModuleFilters { + const scope = params.get('scope') || undefined; + return { + scope: scope === 'project' ? scope : undefined, + include: params.get('include') || undefined, + exclude: params.get('exclude') || undefined, + }; +} + +/** + * Get the query parameters string for the module filters. + * This only applies the filters that are set. + */ +export function moduleFiltersToParams(filters: ModuleFilters) { + const params = new URLSearchParams(); + + if (filters.scope) params.set('scope', filters.scope); + if (filters.include) params.set('include', filters.include); + if (filters.exclude) params.set('exclude', filters.exclude); + + return params; +} + +/** + * Get the current module filters from URL search params, using Expo Router. + * This returns the filters, with default values, and if any of the filters has been defined. + */ +export function useModuleFilters() { + const filters = useGlobalSearchParams(); + return { + filtersEnabled: !!filters.scope || !!filters.include || !!filters.exclude, + filters, + }; +} + +/** Filter the modules based on the filters, and an optional (root) path. */ +export function filterModules( + modules: StatsModule[], + options: { + projectRoot: string; + filters: ModuleFilters; + rootPath?: string; + } +) { + const { filters, projectRoot, rootPath } = options; + + if (rootPath || filters.scope === 'project') { + modules = modules.filter( + (module) => + (!rootPath || module.path.startsWith(rootPath)) && + (filters.scope !== 'project' || !module.package) + ); + } + + if (filters.include || filters.exclude) { + const matcher = picomatch(splitPattern(options.filters.include) || '**', { + cwd: '', + dot: true, + nocase: true, + contains: true, + ignore: !options.filters.exclude ? undefined : splitPattern(options.filters.exclude), + }); + + modules = modules.filter((module) => matcher(path.relative(projectRoot, module.path))); + } + + return modules; +} + +/** + * Split the comma separated string into an array of separate patterns. + * This splits on any combination of `,` and whitespaces. + */ +function splitPattern(pattern = '') { + if (!pattern) return undefined; + const split = pattern.split(/\s*,\s*/).filter(Boolean); + return split.length ? split : undefined; +} diff --git a/webui/src/utils/search.ts b/webui/src/utils/search.ts deleted file mode 100644 index 9eab2ff..0000000 --- a/webui/src/utils/search.ts +++ /dev/null @@ -1,44 +0,0 @@ -import path from 'path'; -import picomatch from 'picomatch'; - -import { type StatsModule } from '~core/data/types'; - -type ModuleFilters = { - include?: string; - exclude?: string; -}; - -/** - * Filter the modules based on the include and exclude glob patterns. - * Note, you can provide multiple patterns using the comma separator. - * This also only searches the relative module path from project root, avoiding false positives. - */ -export function globFilterModules( - items: StatsModule[], - projectRoot: string, - options: ModuleFilters -) { - if (!options.include && !options.exclude) { - return items; - } - - const matcher = picomatch(splitPattern(options.include) || '**', { - cwd: '', - dot: true, - nocase: true, - contains: true, - ignore: !options.exclude ? undefined : splitPattern(options.exclude), - }); - - return items.filter((item) => matcher(path.relative(projectRoot, item.path))); -} - -/** - * Split the comma separated string into an array of separate patterns. - * This splits on any combination of `,` and whitespaces. - */ -function splitPattern(pattern = '') { - if (!pattern) return undefined; - const split = pattern.split(/\s*,\s*/).filter(Boolean); - return split.length ? split : undefined; -}