refactor: add data layer with direct metro serialization access

README.md

@@ -17,12 +17,12 @@ Configure your Metro config to emit a `.expo/stats.json` file containing informa
 
 ```js metro.config.js
 const { getDefaultConfig } = require('expo/metro-config');
-const { withMetroBundleConfig } = require('expo-atlas/metro');
+const { withExpoAtlas } = require('expo-atlas/metro');
 
 const config = getDefaultConfig(__dirname);
 
-// Add the `withMetroBundleConfig` from `expo-atlas/metro` as last change
-module.exports = withMetroBundleConfig(config);
+// Add the `withExpoAtlas` from `expo-atlas/metro` as last change
+module.exports = withExpoAtlas(config);
 ```
 
 After that, you can generate a new bundle and inspect these through the CLI

metro.d.ts

@@ -1 +1 @@
-export * from './build/src/metro/withMetroBundleConfig';
+export * from './build/src/withExpoAtlas';

metro.js

@@ -1 +1 @@
-module.exports = require('./build/src/metro/withMetroBundleConfig');
+module.exports = require('./build/src/withExpoAtlas');

src/cli/resolveOptions.ts

@@ -1,9 +1,8 @@
-import fs from 'fs';
 import path from 'path';
 
 import { type Input } from './bin';
-import { getStatsPath, validateStatsFile } from '../metro/serializeStatsFile';
 import { getFreePort } from '../utils/port';
+import { getStatsPath, validateStatsFile } from '../utils/stats';
 
 export type Options = Awaited<ReturnType<typeof resolveOptions>>;
 
@@ -15,13 +14,7 @@ export async function resolveOptions(input: Input) {
 
 async function resolveStatsFile(input: Input) {
   const statsFile = input._[0] ?? getStatsPath(process.cwd());
-
-  if (!fs.existsSync(statsFile)) {
-    throw new Error(`Could not find stats file "${statsFile}".`);
-  }
-
   await validateStatsFile(statsFile);
-
   return path.resolve(statsFile);
 }
 

src/cli/server.ts

@@ -5,15 +5,17 @@ import morgan from 'morgan';
 import path from 'path';
 
 import { type Options } from './resolveOptions';
+import { StatsFileSource } from '../data/StatsFileSource';
 import { env } from '../utils/env';
 
 const WEBUI_ROOT = path.resolve(__dirname, '../../../webui');
 const CLIENT_BUILD_DIR = path.join(WEBUI_ROOT, 'dist/client');
 const SERVER_BUILD_DIR = path.join(WEBUI_ROOT, 'dist/server');
 
 export function createServer(options: Options) {
+  // Instantiate the global variable for the server
+  global['EXPO_ATLAS_SOURCE'] = new StatsFileSource(options.statsFile);
   process.env.NODE_ENV = 'production';
-  process.env.EXPO_METRO_BUNDLE_STATS_FILE = options.statsFile;
 
   const app = express();
 

src/data/MetroGraphSource.ts

@@ -0,0 +1,123 @@
+import type metro from 'metro';
+
+import type { StatsEntry, StatsModule, StatsSource } from './types';
+import { getNonBinaryContents } from '../utils/buffer';
+import { getPackageNameFromPath } from '../utils/package';
+
+type MetroGraph = metro.Graph | metro.ReadOnlyGraph;
+type MetroModule = metro.Module;
+
+type ConvertGraphToStatsOptions = {
+  projectRoot: string;
+  entryPoint: string;
+  preModules: Readonly<MetroModule[]>;
+  graph: MetroGraph;
+  options: Readonly<metro.SerializerOptions>;
+};
+
+export class MetroGraphSource implements StatsSource {
+  /** All known stats entries, stored by ID */
+  protected entries: Map<StatsEntry['id'], StatsEntry> = new Map();
+
+  listEntries() {
+    return Array.from(this.entries.values()).map((entry) => ({
+      id: entry.id,
+      platform: entry.platform,
+      projectRoot: entry.projectRoot,
+      entryPoint: entry.entryPoint,
+    }));
+  }
+
+  getEntry(id: string) {
+    const entry = this.entries.get(id);
+    if (!entry) {
+      throw new Error(`Stats entry "${id}" not found.`);
+    }
+    return entry;
+  }
+
+  /**
+   * Event handler when a new graph instance is ready to serialize.
+   * This converts all relevant data stored in the graph to stats objects.
+   */
+  onSerializeGraph(options: ConvertGraphToStatsOptions) {
+    const entry = convertGraph(options);
+    this.entries.set(entry.id, entry);
+    return entry;
+  }
+}
+
+/** Convert a Metro graph instance to a JSON-serializable stats entry */
+export function convertGraph(options: ConvertGraphToStatsOptions): StatsEntry {
+  const serializeOptions = convertSerializeOptions(options.options);
+  const transformOptions = convertTransformOptions(options.graph.transformOptions);
+  const platform = transformOptions?.platform ?? 'unknown';
+
+  return {
+    id: `${options.entryPoint}+${platform}`,
+    platform,
+    projectRoot: options.projectRoot,
+    entryPoint: options.entryPoint,
+    runtimeModules: options.preModules.map((module) => convertModule(options.graph, module)),
+    modules: collectEntryPointModules(options.graph, options.entryPoint),
+    serializeOptions,
+    transformOptions,
+  };
+}
+
+/** Find and collect all dependnecies related to the entrypoint within the graph */
+export function collectEntryPointModules(graph: MetroGraph, entryPoint: string) {
+  const modules = new Map<string, StatsModule>();
+
+  function discover(modulePath: string) {
+    const module = graph.dependencies.get(modulePath);
+
+    if (module && !modules.has(modulePath)) {
+      modules.set(modulePath, convertModule(graph, module));
+      module.dependencies.forEach((modulePath) => discover(modulePath.absolutePath));
+    }
+  }
+
+  discover(entryPoint);
+  return modules;
+}
+
+/** Convert a Metro module to a JSON-serializable stats module */
+export function convertModule(graph: MetroGraph, module: MetroModule): StatsModule {
+  return {
+    path: module.path,
+    package: getPackageNameFromPath(module.path),
+    size: module.output.reduce((bytes, output) => bytes + Buffer.byteLength(output.data.code), 0),
+    imports: Array.from(module.dependencies.values()).map((module) => module.absolutePath),
+    importedBy: Array.from(module.inverseDependencies).filter((dependecy) =>
+      graph.dependencies.has(dependecy)
+    ),
+    source: getNonBinaryContents(module.getSource()) ?? '[binary file]',
+    output: module.output.map((output) => ({
+      type: output.type,
+      data: { code: output.data.code },
+    })),
+  };
+}
+
+/** Convert Metro transform options to a JSON-serializable object */
+export function convertTransformOptions(
+  transformer: metro.TransformInputOptions
+): StatsEntry['transformOptions'] {
+  return transformer;
+}
+
+/** Convert Metro serialize options to a JSON-serializable object */
+export function convertSerializeOptions(
+  serializer: metro.SerializerOptions
+): StatsEntry['serializeOptions'] {
+  const options: StatsEntry['serializeOptions'] = { ...serializer };
+
+  // Delete all filters
+  delete options['processModuleFilter'];
+  delete options['createModuleId'];
+  delete options['getRunModuleStatement'];
+  delete options['shouldAddToIgnoreList'];
+
+  return options;
+}

src/data/StatsFileSource.ts

@@ -0,0 +1,84 @@
+import assert from 'assert';
+
+import type { PartialStatsEntry, StatsEntry, StatsSource } from './types';
+import { appendNDJsonToFile, mapNDJson, parseNDJsonAtLine } from '../utils/ndjson';
+
+export class StatsFileSource implements StatsSource {
+  constructor(public readonly statsPath: string) {
+    //
+  }
+
+  listEntries() {
+    return listStatsEntries(this.statsPath);
+  }
+
+  getEntry(id: string) {
+    const numeric = parseInt(id, 10);
+    assert(!Number.isNaN(numeric) && numeric > 1, `Invalid stats entry ID: ${id}`);
+    return readStatsEntry(this.statsPath, Number(id));
+  }
+}
+
+/**
+ * List all stats entries without parsing the data.
+ * This only reads the bundle name, and adds a line number as ID.
+ */
+export async function listStatsEntries(statsPath: string) {
+  const bundlePattern = /^\["([^"]+)","([^"]+)","([^"]+)/;
+  const entries: PartialStatsEntry[] = [];
+
+  await mapNDJson(statsPath, (index, line) => {
+    // Skip the stats metadata line
+    if (index === 1) return;
+
+    const [_, platform, projectRoot, entryPoint] = line.match(bundlePattern) ?? [];
+    if (platform && projectRoot && entryPoint) {
+      entries.push({
+        id: String(index),
+        platform: platform as any,
+        projectRoot,
+        entryPoint,
+      });
+    }
+  });
+
+  return entries;
+}
+
+/**
+ * Get the stats entry by id or line number, and parse the data.
+ */
+export async function readStatsEntry(statsPath: string, id: number): Promise<StatsEntry> {
+  const statsEntry = await parseNDJsonAtLine<any[]>(statsPath, id);
+  return {
+    id: String(id),
+    platform: statsEntry[0],
+    projectRoot: statsEntry[1],
+    entryPoint: statsEntry[2],
+    runtimeModules: statsEntry[3],
+    modules: new Map(statsEntry[4]),
+    transformOptions: statsEntry[5],
+    serializeOptions: statsEntry[6],
+  };
+}
+
+/** Simple promise to avoid mixing appended data */
+let writeStatsQueue: Promise<any> = Promise.resolve();
+
+/**
+ * Add a new stats entry to the stats file.
+ * This is appended on a new line, so we can load the stats selectively.
+ */
+export function writeStatsEntry(statsPath: string, stats: StatsEntry) {
+  const entry = [
+    stats.platform,
+    stats.projectRoot,
+    stats.entryPoint,
+    stats.runtimeModules,
+    stats.modules,
+    stats.transformOptions,
+    stats.serializeOptions,
+  ];
+
+  return (writeStatsQueue = writeStatsQueue.then(() => appendNDJsonToFile(statsPath, entry)));
+}

src/data/types.ts

@@ -0,0 +1,46 @@
+import type { MixedOutput } from 'metro';
+
+export interface StatsSource {
+  /** List all available stats entries */
+  listEntries(): PartialStatsEntry[] | Promise<PartialStatsEntry[]>;
+  /** Load the full stats entry, by reference */
+  getEntry(ref: string): StatsEntry | Promise<StatsEntry>;
+}
+
+export type PartialStatsEntry = Pick<StatsEntry, 'id' | 'platform' | 'projectRoot' | 'entryPoint'>;
+
+export type StatsEntry = {
+  /** The unique reference or ID to this stats entry */
+  id: string;
+  /** The platform for which the bundle was created */
+  platform: 'android' | 'ios' | 'web';
+  /** The absolute path to the root of the project */
+  projectRoot: string;
+  /** The absolute path to the entry point used when creating the bundle */
+  entryPoint: string;
+  /** All known modules that are prepended for the runtime itself */
+  runtimeModules: StatsModule[];
+  /** All known modules imported within the bundle, stored by absolute path */
+  modules: Map<string, StatsModule>;
+  /** The sarialization options used for this bundle */
+  serializeOptions?: Record<string, any>;
+  /** The transformation options used for this bundle */
+  transformOptions?: Record<string, any>;
+};
+
+export type StatsModule = {
+  /** The absoluate path of this module */
+  path: string;
+  /** The name of the package this module belongs to, if from an external package */
+  package?: string;
+  /** The original module size, in bytes */
+  size: number;
+  /** Absolute file paths of modules imported inside this module */
+  imports: string[];
+  /** Absolute file paths of modules importing this module */
+  importedBy: string[];
+  /** The original source code, as a buffer or string */
+  source?: string;
+  /** The transformed output source code */
+  output?: MixedOutput[];
+};

src/index.ts

@@ -1,8 +1,7 @@
-export let useMetroBundlePlugin: typeof import('./useMetroBundlePlugin').useMetroBundlePlugin;
+export type * from './data/types';
 
-// @ts-ignore process.env.NODE_ENV is defined by metro transform plugins
-if (process.env.NODE_ENV !== 'production') {
-  useMetroBundlePlugin = require('./useMetroBundlePlugin').useMetroBundlePlugin;
-} else {
-  useMetroBundlePlugin = () => {};
-}
+export { MetroGraphSource } from './data/MetroGraphSource';
+export { StatsFileSource } from './data/StatsFileSource';
+
+export { AtlasError, AtlasValidationError } from './utils/errors';
+export { createStatsFile, validateStatsFile, getStatsMetdata, getStatsPath } from './utils/stats';