From d9fa611122312d63f49c24bba49c46d497e2f63d Mon Sep 17 00:00:00 2001 From: Cedric van Putten Date: Wed, 13 Mar 2024 17:52:31 +0100 Subject: [PATCH] refactor: add data layer with direct metro serialization access --- README.md | 6 +- metro.d.ts | 2 +- metro.js | 2 +- src/cli/resolveOptions.ts | 9 +- src/cli/server.ts | 4 +- src/data/MetroGraphSource.ts | 123 ++++++++++++++++++++++++++ src/data/StatsFileSource.ts | 84 ++++++++++++++++++ src/data/types.ts | 46 ++++++++++ src/index.ts | 13 ++- src/metro/convertGraphToStats.ts | 124 -------------------------- src/metro/serializeStatsFile.ts | 137 ----------------------------- src/metro/withMetroBundleConfig.ts | 27 ------ src/useMetroBundlePlugin.ts | 23 ----- src/utils/file.ts | 58 ------------ src/utils/ndjson.ts | 93 ++++++++++++++++++++ src/utils/package.ts | 18 ++++ src/utils/stats.ts | 56 ++++++++++++ src/withExpoAtlas.ts | 53 +++++++++++ 18 files changed, 488 insertions(+), 390 deletions(-) create mode 100644 src/data/MetroGraphSource.ts create mode 100644 src/data/StatsFileSource.ts create mode 100644 src/data/types.ts delete mode 100644 src/metro/convertGraphToStats.ts delete mode 100644 src/metro/serializeStatsFile.ts delete mode 100644 src/metro/withMetroBundleConfig.ts delete mode 100644 src/useMetroBundlePlugin.ts delete mode 100644 src/utils/file.ts create mode 100644 src/utils/ndjson.ts create mode 100644 src/utils/package.ts create mode 100644 src/utils/stats.ts create mode 100644 src/withExpoAtlas.ts diff --git a/README.md b/README.md index 1d29e65..4d08542 100644 --- a/README.md +++ b/README.md @@ -17,12 +17,12 @@ Configure your Metro config to emit a `.expo/stats.json` file containing informa ```js metro.config.js const { getDefaultConfig } = require('expo/metro-config'); -const { withMetroBundleConfig } = require('expo-atlas/metro'); +const { withExpoAtlas } = require('expo-atlas/metro'); const config = getDefaultConfig(__dirname); -// Add the `withMetroBundleConfig` from `expo-atlas/metro` as last change -module.exports = withMetroBundleConfig(config); +// Add the `withExpoAtlas` from `expo-atlas/metro` as last change +module.exports = withExpoAtlas(config); ``` After that, you can generate a new bundle and inspect these through the CLI diff --git a/metro.d.ts b/metro.d.ts index 2a3061b..cb70a9c 100644 --- a/metro.d.ts +++ b/metro.d.ts @@ -1 +1 @@ -export * from './build/src/metro/withMetroBundleConfig'; +export * from './build/src/withExpoAtlas'; diff --git a/metro.js b/metro.js index ce8df68..9c7bc8e 100644 --- a/metro.js +++ b/metro.js @@ -1 +1 @@ -module.exports = require('./build/src/metro/withMetroBundleConfig'); +module.exports = require('./build/src/withExpoAtlas'); diff --git a/src/cli/resolveOptions.ts b/src/cli/resolveOptions.ts index 2d09c0f..ba48f3f 100644 --- a/src/cli/resolveOptions.ts +++ b/src/cli/resolveOptions.ts @@ -1,9 +1,8 @@ -import fs from 'fs'; import path from 'path'; import { type Input } from './bin'; -import { getStatsPath, validateStatsFile } from '../metro/serializeStatsFile'; import { getFreePort } from '../utils/port'; +import { getStatsPath, validateStatsFile } from '../utils/stats'; export type Options = Awaited>; @@ -15,13 +14,7 @@ export async function resolveOptions(input: Input) { async function resolveStatsFile(input: Input) { const statsFile = input._[0] ?? getStatsPath(process.cwd()); - - if (!fs.existsSync(statsFile)) { - throw new Error(`Could not find stats file "${statsFile}".`); - } - await validateStatsFile(statsFile); - return path.resolve(statsFile); } diff --git a/src/cli/server.ts b/src/cli/server.ts index 591f324..fb4b2f7 100644 --- a/src/cli/server.ts +++ b/src/cli/server.ts @@ -5,6 +5,7 @@ import morgan from 'morgan'; import path from 'path'; import { type Options } from './resolveOptions'; +import { StatsFileSource } from '../data/StatsFileSource'; import { env } from '../utils/env'; const WEBUI_ROOT = path.resolve(__dirname, '../../../webui'); @@ -12,8 +13,9 @@ const CLIENT_BUILD_DIR = path.join(WEBUI_ROOT, 'dist/client'); const SERVER_BUILD_DIR = path.join(WEBUI_ROOT, 'dist/server'); export function createServer(options: Options) { + // Instantiate the global variable for the server + global['EXPO_ATLAS_SOURCE'] = new StatsFileSource(options.statsFile); process.env.NODE_ENV = 'production'; - process.env.EXPO_METRO_BUNDLE_STATS_FILE = options.statsFile; const app = express(); diff --git a/src/data/MetroGraphSource.ts b/src/data/MetroGraphSource.ts new file mode 100644 index 0000000..cca156d --- /dev/null +++ b/src/data/MetroGraphSource.ts @@ -0,0 +1,123 @@ +import type metro from 'metro'; + +import type { StatsEntry, StatsModule, StatsSource } from './types'; +import { getNonBinaryContents } from '../utils/buffer'; +import { getPackageNameFromPath } from '../utils/package'; + +type MetroGraph = metro.Graph | metro.ReadOnlyGraph; +type MetroModule = metro.Module; + +type ConvertGraphToStatsOptions = { + projectRoot: string; + entryPoint: string; + preModules: Readonly; + graph: MetroGraph; + options: Readonly; +}; + +export class MetroGraphSource implements StatsSource { + /** All known stats entries, stored by ID */ + protected entries: Map = new Map(); + + listEntries() { + return Array.from(this.entries.values()).map((entry) => ({ + id: entry.id, + platform: entry.platform, + projectRoot: entry.projectRoot, + entryPoint: entry.entryPoint, + })); + } + + getEntry(id: string) { + const entry = this.entries.get(id); + if (!entry) { + throw new Error(`Stats entry "${id}" not found.`); + } + return entry; + } + + /** + * Event handler when a new graph instance is ready to serialize. + * This converts all relevant data stored in the graph to stats objects. + */ + onSerializeGraph(options: ConvertGraphToStatsOptions) { + const entry = convertGraph(options); + this.entries.set(entry.id, entry); + return entry; + } +} + +/** Convert a Metro graph instance to a JSON-serializable stats entry */ +export function convertGraph(options: ConvertGraphToStatsOptions): StatsEntry { + const serializeOptions = convertSerializeOptions(options.options); + const transformOptions = convertTransformOptions(options.graph.transformOptions); + const platform = transformOptions?.platform ?? 'unknown'; + + return { + id: `${options.entryPoint}+${platform}`, + platform, + projectRoot: options.projectRoot, + entryPoint: options.entryPoint, + runtimeModules: options.preModules.map((module) => convertModule(options.graph, module)), + modules: collectEntryPointModules(options.graph, options.entryPoint), + serializeOptions, + transformOptions, + }; +} + +/** Find and collect all dependnecies related to the entrypoint within the graph */ +export function collectEntryPointModules(graph: MetroGraph, entryPoint: string) { + const modules = new Map(); + + function discover(modulePath: string) { + const module = graph.dependencies.get(modulePath); + + if (module && !modules.has(modulePath)) { + modules.set(modulePath, convertModule(graph, module)); + module.dependencies.forEach((modulePath) => discover(modulePath.absolutePath)); + } + } + + discover(entryPoint); + return modules; +} + +/** Convert a Metro module to a JSON-serializable stats module */ +export function convertModule(graph: MetroGraph, module: MetroModule): StatsModule { + return { + path: module.path, + package: getPackageNameFromPath(module.path), + size: module.output.reduce((bytes, output) => bytes + Buffer.byteLength(output.data.code), 0), + imports: Array.from(module.dependencies.values()).map((module) => module.absolutePath), + importedBy: Array.from(module.inverseDependencies).filter((dependecy) => + graph.dependencies.has(dependecy) + ), + source: getNonBinaryContents(module.getSource()) ?? '[binary file]', + output: module.output.map((output) => ({ + type: output.type, + data: { code: output.data.code }, + })), + }; +} + +/** Convert Metro transform options to a JSON-serializable object */ +export function convertTransformOptions( + transformer: metro.TransformInputOptions +): StatsEntry['transformOptions'] { + return transformer; +} + +/** Convert Metro serialize options to a JSON-serializable object */ +export function convertSerializeOptions( + serializer: metro.SerializerOptions +): StatsEntry['serializeOptions'] { + const options: StatsEntry['serializeOptions'] = { ...serializer }; + + // Delete all filters + delete options['processModuleFilter']; + delete options['createModuleId']; + delete options['getRunModuleStatement']; + delete options['shouldAddToIgnoreList']; + + return options; +} diff --git a/src/data/StatsFileSource.ts b/src/data/StatsFileSource.ts new file mode 100644 index 0000000..d666b37 --- /dev/null +++ b/src/data/StatsFileSource.ts @@ -0,0 +1,84 @@ +import assert from 'assert'; + +import type { PartialStatsEntry, StatsEntry, StatsSource } from './types'; +import { appendNDJsonToFile, mapNDJson, parseNDJsonAtLine } from '../utils/ndjson'; + +export class StatsFileSource implements StatsSource { + constructor(public readonly statsPath: string) { + // + } + + listEntries() { + return listStatsEntries(this.statsPath); + } + + getEntry(id: string) { + const numeric = parseInt(id, 10); + assert(!Number.isNaN(numeric) && numeric > 1, `Invalid stats entry ID: ${id}`); + return readStatsEntry(this.statsPath, Number(id)); + } +} + +/** + * List all stats entries without parsing the data. + * This only reads the bundle name, and adds a line number as ID. + */ +export async function listStatsEntries(statsPath: string) { + const bundlePattern = /^\["([^"]+)","([^"]+)","([^"]+)/; + const entries: PartialStatsEntry[] = []; + + await mapNDJson(statsPath, (index, line) => { + // Skip the stats metadata line + if (index === 1) return; + + const [_, platform, projectRoot, entryPoint] = line.match(bundlePattern) ?? []; + if (platform && projectRoot && entryPoint) { + entries.push({ + id: String(index), + platform: platform as any, + projectRoot, + entryPoint, + }); + } + }); + + return entries; +} + +/** + * Get the stats entry by id or line number, and parse the data. + */ +export async function readStatsEntry(statsPath: string, id: number): Promise { + const statsEntry = await parseNDJsonAtLine(statsPath, id); + return { + id: String(id), + platform: statsEntry[0], + projectRoot: statsEntry[1], + entryPoint: statsEntry[2], + runtimeModules: statsEntry[3], + modules: new Map(statsEntry[4]), + transformOptions: statsEntry[5], + serializeOptions: statsEntry[6], + }; +} + +/** Simple promise to avoid mixing appended data */ +let writeStatsQueue: Promise = Promise.resolve(); + +/** + * Add a new stats entry to the stats file. + * This is appended on a new line, so we can load the stats selectively. + */ +export function writeStatsEntry(statsPath: string, stats: StatsEntry) { + const entry = [ + stats.platform, + stats.projectRoot, + stats.entryPoint, + stats.runtimeModules, + stats.modules, + stats.transformOptions, + stats.serializeOptions, + ]; + + return (writeStatsQueue = writeStatsQueue.then(() => appendNDJsonToFile(statsPath, entry))); +} diff --git a/src/data/types.ts b/src/data/types.ts new file mode 100644 index 0000000..1d63bc7 --- /dev/null +++ b/src/data/types.ts @@ -0,0 +1,46 @@ +import type { MixedOutput } from 'metro'; + +export interface StatsSource { + /** List all available stats entries */ + listEntries(): PartialStatsEntry[] | Promise; + /** Load the full stats entry, by reference */ + getEntry(ref: string): StatsEntry | Promise; +} + +export type PartialStatsEntry = Pick; + +export type StatsEntry = { + /** The unique reference or ID to this stats entry */ + id: string; + /** The platform for which the bundle was created */ + platform: 'android' | 'ios' | 'web'; + /** The absolute path to the root of the project */ + projectRoot: string; + /** The absolute path to the entry point used when creating the bundle */ + entryPoint: string; + /** All known modules that are prepended for the runtime itself */ + runtimeModules: StatsModule[]; + /** All known modules imported within the bundle, stored by absolute path */ + modules: Map; + /** The sarialization options used for this bundle */ + serializeOptions?: Record; + /** The transformation options used for this bundle */ + transformOptions?: Record; +}; + +export type StatsModule = { + /** The absoluate path of this module */ + path: string; + /** The name of the package this module belongs to, if from an external package */ + package?: string; + /** The original module size, in bytes */ + size: number; + /** Absolute file paths of modules imported inside this module */ + imports: string[]; + /** Absolute file paths of modules importing this module */ + importedBy: string[]; + /** The original source code, as a buffer or string */ + source?: string; + /** The transformed output source code */ + output?: MixedOutput[]; +}; diff --git a/src/index.ts b/src/index.ts index 62f67f6..aa8fc95 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,8 +1,7 @@ -export let useMetroBundlePlugin: typeof import('./useMetroBundlePlugin').useMetroBundlePlugin; +export type * from './data/types'; -// @ts-ignore process.env.NODE_ENV is defined by metro transform plugins -if (process.env.NODE_ENV !== 'production') { - useMetroBundlePlugin = require('./useMetroBundlePlugin').useMetroBundlePlugin; -} else { - useMetroBundlePlugin = () => {}; -} +export { MetroGraphSource } from './data/MetroGraphSource'; +export { StatsFileSource } from './data/StatsFileSource'; + +export { AtlasError, AtlasValidationError } from './utils/errors'; +export { createStatsFile, validateStatsFile, getStatsMetdata, getStatsPath } from './utils/stats'; diff --git a/src/metro/convertGraphToStats.ts b/src/metro/convertGraphToStats.ts deleted file mode 100644 index 2c2dc57..0000000 --- a/src/metro/convertGraphToStats.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { type SerializerConfigT } from 'metro-config'; -import path from 'path'; -import { getNonBinaryContents } from '../utils/buffer'; - -type CustomSerializerParameters = Parameters>; -type ConvertOptions = { - projectRoot: string; - entryPoint: CustomSerializerParameters[0]; - preModules: CustomSerializerParameters[1]; - graph: CustomSerializerParameters[2]; - options: CustomSerializerParameters[3]; -}; - -export type MetroStatsEntry = ReturnType; -export type MetroStatsModule = ReturnType; - -export function convertGraphToStats({ - projectRoot, - entryPoint, - preModules, - graph, - options, -}: ConvertOptions) { - return { - projectRoot, - entryPoint, - platform: graph.transformOptions.platform ?? 'unknown', - preModules: preModules.map((module) => convertModule(projectRoot, graph, module)), - graph: convertGraph(projectRoot, entryPoint, graph), - options: convertOptions(options), - }; -} - -function convertOptions(options: ConvertOptions['options']) { - return { - ...options, - processModuleFilter: undefined, - createModuleId: undefined, - getRunModuleStatement: undefined, - shouldAddToIgnoreList: undefined, - }; -} - -function convertGraph(projectRoot: string, entryPoint: string, graph: ConvertOptions['graph']) { - const dependencies = new Map(); - - function walk(modulePath: string) { - const module = graph.dependencies.get(modulePath); - if (module && !dependencies.has(modulePath)) { - dependencies.set(modulePath, convertModule(projectRoot, graph, module)); - module.dependencies.forEach((modulePath) => walk(modulePath.absolutePath)); - } - } - - walk(entryPoint); - - return { - ...graph, - entryPoints: Array.from(graph.entryPoints.values()), - dependencies: Array.from(dependencies.values()), - }; -} - -function convertModule( - projectRoot: string, - graph: ConvertOptions['graph'], - module: ConvertOptions['preModules'][0] -) { - const nodeModuleName = getNodeModuleNameFromPath(module.path); - - return { - nodeModuleName: nodeModuleName || '[unknown]', - isNodeModule: !!nodeModuleName, - relativePath: path.relative(projectRoot, module.path), - absolutePath: module.path, - size: getModuleOutputInBytes(module), - dependencies: Array.from(module.dependencies.values()).map((dependency) => - path.relative(projectRoot, dependency.absolutePath) - ), - inverseDependencies: Array.from(module.inverseDependencies) - .filter((dependencyPath) => graph.dependencies.has(dependencyPath)) - .map((dependencyPath) => ({ - relativePath: path.relative(projectRoot, dependencyPath), - absolutePath: dependencyPath, - })), - - source: getNonBinaryContents(module.getSource()) ?? '[binary file]', - output: module.output.map((output) => ({ - type: output.type, - data: { code: output.data.code }, // Avoid adding source maps, this is too big for json - })), - }; -} - -function getModuleOutputInBytes(module: ConvertOptions['preModules'][0]) { - return module.output.reduce( - (bytes, module) => bytes + Buffer.byteLength(module.data.code, 'utf-8'), - 0 - ); -} - -const nodeModuleNameCache = new Map(); -function getNodeModuleNameFromPath(path: string) { - if (nodeModuleNameCache.has(path)) { - return nodeModuleNameCache.get(path) ?? null; - } - - const segments = path.split('/'); - - for (let i = segments.length - 1; i >= 0; i--) { - if (segments[i] === 'node_modules') { - let name = segments[i + 1]; - - if (name.startsWith('@') && i + 2 < segments.length) { - name += '/' + segments[i + 2]; - } - - nodeModuleNameCache.set(path, name); - return name; - } - } - - return null; -} diff --git a/src/metro/serializeStatsFile.ts b/src/metro/serializeStatsFile.ts deleted file mode 100644 index 6075c50..0000000 --- a/src/metro/serializeStatsFile.ts +++ /dev/null @@ -1,137 +0,0 @@ -import fs from 'fs'; -import path from 'path'; - -import { type MetroStatsEntry } from './convertGraphToStats'; -import { name, version } from '../../package.json'; -import { env } from '../utils/env'; -import { AtlasValidationError } from '../utils/errors'; -import { mapLines, readFirstLine, readLine } from '../utils/file'; - -export type StatsMetadata = { name: string; version: string }; - -/** The default location of the metro stats file */ -export function getStatsPath(projectRoot: string) { - return path.join(projectRoot, '.expo/stats.json'); -} - -/** The information to validate if a stats file is compatible with this library version */ -export function getStatsMetdata(): StatsMetadata { - return { name, version }; -} - -/** Validate if the stats file is compatible with this library version */ -export async function validateStatsFile(statsFile: string, metadata = getStatsMetdata()) { - if (!fs.existsSync(statsFile)) { - throw new AtlasValidationError('STATS_FILE_NOT_FOUND', statsFile); - } - - if (env.EXPO_NO_STATS_VALIDATION) { - return; - } - - const line = await readFirstLine(statsFile); - const data = line ? JSON.parse(line) : {}; - - if (data.name !== metadata.name || data.version !== metadata.version) { - throw new AtlasValidationError('STATS_FILE_INCOMPATIBLE', statsFile, data.version); - } -} - -/** - * Create or overwrite the stats file with basic metadata. - * This metdata is used by the API to determine version compatibility. - */ -export async function createStatsFile(projectRoot: string) { - const filePath = getStatsPath(projectRoot); - - if (fs.existsSync(filePath)) { - try { - await validateStatsFile(filePath); - } catch { - await fs.promises.writeFile(filePath, JSON.stringify(getStatsMetdata()) + '\n'); - } - - return; - } - - await fs.promises.mkdir(path.dirname(filePath), { recursive: true }); - await fs.promises.writeFile(filePath, JSON.stringify(getStatsMetdata()) + '\n'); -} - -/** Simple promise to avoid mixing appended data */ -let writeQueue: Promise = Promise.resolve(); - -/** - * Add a new stats entry to the stats file. - * This is appended on a new line, so we can load the stats selectively. - */ -export async function addStatsEntry(projectRoot: string, stats: MetroStatsEntry) { - // NOTE(cedric): inline bfj to avoid loading it in the webui - const bfj = require('bfj'); - const statsFile = getStatsPath(projectRoot); - const entry = [ - stats.platform, - stats.projectRoot, - stats.entryPoint, - stats.preModules, - stats.options, - stats.graph, - ]; - - writeQueue = writeQueue.then(async () => { - await bfj.write(statsFile, entry, { flags: 'a' }); - await fs.promises.appendFile(statsFile, '\n'); - }); -} - -/** - * List all stats entries without parsing the data. - * This only reads the bundle name, and adds a line number as ID. - */ -export async function listStatsEntries(statsFile: string) { - const bundlePattern = /^\["([^"]+)","([^"]+)","([^"]+)/; - const entries: { - id: number; - absolutePath: string; - relativePath: string; - projectRoot: string; - platform: 'android' | 'ios' | 'web'; - }[] = []; - - await mapLines(statsFile, (index, line) => { - if (index === 1) return; - - const [_, platform, projectRoot, entryPoint] = line.match(bundlePattern) ?? []; - if (platform && projectRoot && entryPoint) { - entries.push({ - id: index, - platform: platform as any, - projectRoot, - absolutePath: entryPoint, - relativePath: path.relative(projectRoot, entryPoint), - }); - } - }); - - return entries; -} - -/** - * Get the stats entry by id or line number, and parse the data. - */ -export async function getStatsEntry(statsFile: string, id: number): Promise { - const line = await readLine(statsFile, id); - if (!line) { - throw new Error(`Stats entry "${id}" not found.`); - } - - const list = JSON.parse(line); - return { - platform: list[0], - projectRoot: list[1], - entryPoint: list[2], - preModules: list[3], - options: list[4], - graph: list[5], - }; -} diff --git a/src/metro/withMetroBundleConfig.ts b/src/metro/withMetroBundleConfig.ts deleted file mode 100644 index 5baf5fe..0000000 --- a/src/metro/withMetroBundleConfig.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { type MetroConfig } from 'metro-config'; - -import { convertGraphToStats } from './convertGraphToStats'; -import { addStatsEntry, createStatsFile } from './serializeStatsFile'; - -export function withMetroBundleConfig(config: MetroConfig) { - if (!config.projectRoot) { - throw new Error('No "projectRoot" configured in Metro config.'); - } - - const originalSerializer = config.serializer?.customSerializer ?? (() => {}); - const projectRoot = config.projectRoot; - - // Note(cedric): we don't have to await this, Metro would never bundle before this is finisheds - createStatsFile(projectRoot); - - // @ts-expect-error - config.serializer.customSerializer = (entryPoint, preModules, graph, options) => { - addStatsEntry( - projectRoot, - convertGraphToStats({ projectRoot, entryPoint, preModules, graph, options }) - ); - return originalSerializer(entryPoint, preModules, graph, options); - }; - - return config; -} diff --git a/src/useMetroBundlePlugin.ts b/src/useMetroBundlePlugin.ts deleted file mode 100644 index bc8f48b..0000000 --- a/src/useMetroBundlePlugin.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { useDevToolsPluginClient, type EventSubscription } from 'expo/devtools'; -import { useEffect } from 'react'; - -export function useMetroBundlePlugin() { - const client = useDevToolsPluginClient('metro-bundle-plugin'); - - useEffect(() => { - const subscriptions: EventSubscription[] = []; - - subscriptions.push( - client?.addMessageListener('ping', (data) => { - alert(`Received ping from ${data.from}`); - }) - ); - client?.sendMessage('ping', { from: 'app' }); - - return () => { - for (const subscription of subscriptions) { - subscription?.remove(); - } - }; - }, [client]); -} diff --git a/src/utils/file.ts b/src/utils/file.ts deleted file mode 100644 index 0300805..0000000 --- a/src/utils/file.ts +++ /dev/null @@ -1,58 +0,0 @@ -import events from 'events'; -import fs from 'fs'; -import readline from 'readline'; - -export async function readFirstLine(filePath: string) { - const stream = fs.createReadStream(filePath); - const reader = readline.createInterface({ input: stream }); - const contents = new Promise((resolve, reject) => { - reader.on('error', reject); - reader.on('line', (line) => { - reader.close(); - resolve(line); - }); - }); - - contents.finally(() => stream.close()); - - return await contents; -} - -export async function mapLines( - filePath: string, - callback: (line: number, contents: string) => any -) { - const stream = fs.createReadStream(filePath); - const reader = readline.createInterface({ input: stream }); - let lineNumber = 1; - - reader.on('error', (error) => { - throw error; - }); - - reader.on('line', (contents) => { - callback(lineNumber++, contents); - }); - - await events.once(reader, 'close'); -} - -export async function readLine(filePath: string, line: number) { - const stream = fs.createReadStream(filePath); - const reader = readline.createInterface({ input: stream }); - let lineNumber = 1; - - const contents = new Promise((resolve, reject) => { - reader.on('error', reject); - reader.on('line', (contents) => { - if (lineNumber++ === line) { - reader.close(); - resolve(contents); - } - }); - }); - - contents.finally(() => stream.close()); - - return await contents; -} diff --git a/src/utils/ndjson.ts b/src/utils/ndjson.ts new file mode 100644 index 0000000..b812b5a --- /dev/null +++ b/src/utils/ndjson.ts @@ -0,0 +1,93 @@ +import events from 'events'; +import fs from 'fs'; +import readline from 'readline'; + +// @ts-expect-error +Symbol.dispose ??= Symbol('Symbol.dispose'); +// @ts-expect-error +Symbol.asyncDispose ??= Symbol('Symbol.asyncDispose'); + +/** Create a self-disposing stream using explicit resource management */ +function disposableStream(stream: T): T & Disposable { + const disposableStream = stream as T & Disposable; + disposableStream[Symbol.dispose] = () => { + if (!stream.closed) stream.close(); + }; + return disposableStream; +} + +/** + * Efficiently map through all lines within the Newline-Delimited JSON (ndjson) file, using streams. + * This won't parse the actual JSON but returns the partial string instead. + * Note, line numbers starts at `1`. + */ +export async function mapNDJson( + filePath: string, + callback: (line: number, contents: string) => any +) { + using stream = disposableStream(fs.createReadStream(filePath)); + const reader = readline.createInterface({ input: stream }); + let lineNumber = 1; + + reader.on('error', (error) => { + throw error; + }); + + reader.on('line', (contents) => { + callback(lineNumber++, contents); + }); + + await events.once(reader, 'close'); +} + +/** + * Efficiently parse a single line from a Newline-Delimited JSON (ndjson) file, using streams. + * Note, line numbers starts at `1`. + */ +export async function parseNDJsonAtLine(filePath: string, line: number): Promise { + // Note(cedric): keep this dependency inlined to avoid loading it in the WebUI + const bfj = require('bfj'); + let lineCursor = 0; + + using stream = disposableStream(fs.createReadStream(filePath)); + const reader = readline.createInterface({ input: stream }); + + await new Promise((resolve, reject) => { + stream.once('error', reject); + reader.once('error', reject); + + reader.on('line', () => { + if (++lineCursor === line) { + reader.close(); + resolve(undefined); + } + }); + + reader.once('close', () => { + if (lineCursor !== line) { + reject(new Error(`Line "${line}" not found in file "${filePath}"`)); + } + }); + }); + + return await bfj.parse(stream, { ndjson: true }); +} + +/** Efficiently append a new line to a Newline-Delimited JSON (ndjson) file, using streams. */ +export async function appendNDJsonToFile(filePath: string, data: unknown): Promise { + // Note(cedric): keep this dependency inlined to avoid loading it in the WebUI + const bfj = require('bfj'); + await bfj.write(filePath, data, { + // Force stream to append to file + flags: 'a', + // Ignore all complex data types, which shouldn't exist in the data + buffers: 'ignore', + circular: 'ignore', + iterables: 'ignore', + promises: 'ignore', + // Only enable maps, as the graph dependencies are stored as a map + maps: 'object', + }); + + await fs.promises.appendFile(filePath, '\n', 'utf-8'); +} diff --git a/src/utils/package.ts b/src/utils/package.ts new file mode 100644 index 0000000..536a13c --- /dev/null +++ b/src/utils/package.ts @@ -0,0 +1,18 @@ +/** Pattern to match the last `node_modules/` occurance in a string */ +const NODE_MODULES_NAME_PATTERN = /(?:.*\/)?node_modules\/([^/]+)/i; +/** A simple map to return previously resolved package names from paths */ +const packageNameCache = new Map(); + +/** Get the package name from absolute path, if the file belongs to a package. */ +export function getPackageNameFromPath(absolutePath: string) { + const packageName = packageNameCache.get(absolutePath); + if (packageName) return packageName; + + const [_match, name] = absolutePath.match(NODE_MODULES_NAME_PATTERN) ?? []; + if (name) { + packageNameCache.set(absolutePath, name); + return name; + } + + return undefined; +} diff --git a/src/utils/stats.ts b/src/utils/stats.ts new file mode 100644 index 0000000..2a87b4c --- /dev/null +++ b/src/utils/stats.ts @@ -0,0 +1,56 @@ +import fs from 'fs'; +import path from 'path'; + +import { name, version } from '../../package.json'; +import { env } from '../utils/env'; +import { AtlasValidationError } from '../utils/errors'; +import { parseNDJsonAtLine } from '../utils/ndjson'; + +export type StatsMetadata = { name: string; version: string }; + +/** The default location of the metro stats file */ +export function getStatsPath(projectRoot: string) { + return path.join(projectRoot, '.expo/stats.json'); +} + +/** The information to validate if a stats file is compatible with this library version */ +export function getStatsMetdata(): StatsMetadata { + return { name, version }; +} + +/** Validate if the stats file is compatible with this library version */ +export async function validateStatsFile(statsFile: string, metadata = getStatsMetdata()) { + if (!fs.existsSync(statsFile)) { + throw new AtlasValidationError('STATS_FILE_NOT_FOUND', statsFile); + } + + if (env.EXPO_NO_STATS_VALIDATION) { + return; + } + + const line = await parseNDJsonAtLine(statsFile, 0); + const data = line ? JSON.parse(line) : {}; + + if (data.name !== metadata.name || data.version !== metadata.version) { + throw new AtlasValidationError('STATS_FILE_INCOMPATIBLE', statsFile, data.version); + } +} + +/** + * Create or overwrite the stats file with basic metadata. + * This metdata is used by the API to determine version compatibility. + */ +export async function createStatsFile(filePath: string) { + if (fs.existsSync(filePath)) { + try { + await validateStatsFile(filePath); + } catch { + await fs.promises.writeFile(filePath, JSON.stringify(getStatsMetdata()) + '\n'); + } + + return; + } + + await fs.promises.mkdir(path.dirname(filePath), { recursive: true }); + await fs.promises.writeFile(filePath, JSON.stringify(getStatsMetdata()) + '\n'); +} diff --git a/src/withExpoAtlas.ts b/src/withExpoAtlas.ts new file mode 100644 index 0000000..fd7ee17 --- /dev/null +++ b/src/withExpoAtlas.ts @@ -0,0 +1,53 @@ +import { type MetroConfig } from 'metro-config'; + +import { convertGraph } from './data/MetroGraphSource'; +import { writeStatsEntry } from './data/StatsFileSource'; +import { createStatsFile, getStatsPath } from './utils/stats'; + +export type ExpoAtlasOptions = Partial<{ + /** The output of the stats file, defaults to `.expo/stats.json` */ + statsFile: string; +}>; + +/** + * Enable Expo Atlas to gather statistics from Metro when exporting bundles. + * This function should be the last mutation of your Metro config. + * + * @example ```js + * // Learn more https://docs.expo.dev/guides/customizing-metro + * const { getDefaultConfig } = require('expo/metro-config'); + * const { withExpoAtlas } = require('expo-atlas/metro'); + * + * const config = getDefaultConfig(__dirname); + * + * // Make more changes + * + * module.exports = withExpoAtlas(config); + * ``` + */ +export function withExpoAtlas(config: MetroConfig, options: ExpoAtlasOptions = {}) { + const projectRoot = config.projectRoot; + const originalSerializer = config.serializer?.customSerializer ?? (() => {}); + + if (!projectRoot) { + throw new Error('No "projectRoot" configured in Metro config.'); + } + + const statsFile = options?.statsFile ?? getStatsPath(projectRoot); + + // Note(cedric): we don't have to await this, Metro would never bundle before this is finisheds + createStatsFile(statsFile); + + // @ts-expect-error + config.serializer.customSerializer = (entryPoint, preModules, graph, options) => { + // Note(cedric): we don't have to await this, it has a built-in write queue + writeStatsEntry( + statsFile, + convertGraph({ projectRoot, entryPoint, preModules, graph, options }) + ); + + return originalSerializer(entryPoint, preModules, graph, options); + }; + + return config; +}