How do you watch addon changes / rebuilds under embroider?

[NullVoxPopuli]

in the v2 addon blueprint, the default setup uses non-embroider, which relies on ember-auto-import's autoImport.watchDependencies configuration -- works great!

However, when switching to an embroider-only test-app, that config is no longer relevant, as embroider has taken over the full build.

I recall there was once an environment variable for telling embroider which addons to watch for development/rebuilding -- but I couldn't find it anywhere.

I asked our AI bot in the discord, and it told me EMBROIDER_REBUILD_ADDONS, from: https://dev.to/bendemboski/embroider-from-zero-to-route-splitting-in-3-5-weeks-5abo/

but, it seems like that doesn't work anymore? (or I gave it the wrong value, and maybe I need to PR some validation to the value of this environment variable?)

Thoughts?
Is something broken?

Thanks!

[ijlee2]

I believe it's not the environment variable EMBROIDER_REBUILD_ADDONS, but the package broccoli-side-watch, that you're looking for.

[NullVoxPopuli]

So something like:

const sideWatch = require('@embroider/broccoli-side-watch');

function watchLibraries(...libraries) {
  const paths = libraries.map(libraryName => {
    let entry = path.resolve(libraryName);
    let manifestPath = findPackageJsonUp(entry);
    let packagePath = path.dirname(manifestPath);
    let manifest = require(manifestPath);
    let toWatch = manifest.files.map(f => path.join(packagePath, f));
    return toWatch;
  });
  
  return sideWatch('app', { watching: paths.flat() });
}

const app = new EmberApp(defaults, {  
  trees: {
    app: watchLibraries('library-a', 'library-b', 'my-clone-of-whatever');
  },
});

[NullVoxPopuli]

Here is the code I actually ended up with (the above was just spit ballin'):

Deps

  "@embroider/broccoli-side-watch": "0.0.2-unstable.ba9fd29",
  "@pnpm/find-workspace-packages": "6.0.9",
  "package-up": "5.0.0"

some-file.js (as CJS)

'use strict';

const { findWorkspacePackagesNoCheck, arrayOfWorkspacePackagesToMap } = require('@pnpm/find-workspace-packages');
const path = require('path');
const fs = require('fs');

const monorepoRoot = path.join(__dirname, '../../../');

/**
 * For a given "currentPath", we determine what packages (specified in the package.json)
 * are from the monorepo.
 *
 * @param {string} currentPath directory of the package, containing the package.json
 */
async function addons(currentPath) {
	const thisPackage = require(path.join(currentPath, 'package.json'));
	const { dependencies, devDependencies } = thisPackage;

	const allDependencies = [...Object.keys(dependencies || {}), ...Object.keys(devDependencies || {})];

	const packageInfos = await findWorkspacePackagesNoCheck(monorepoRoot);
	const packageObjectMap = arrayOfWorkspacePackagesToMap(packageInfos);

	const relevantPackages = [];

	for (const [name, info] of Object.entries(packageObjectMap)) {
		if (!allDependencies.includes(name)) {
			continue;
		}

		// Info is an object of version => Object, but we only use one version throughout the monorepo
		// even if we didn't, for the purpose of discovering what is in the monorepo, we don't care about
		// what-would-be-extra-information.
		const actualInfo = Object.values(info)[0];
		relevantPackages.push(actualInfo);
	}

	const inMonorepoAddons = relevantPackages
		.map((packageInfo) => packageInfo.manifest)
		.filter((manifest) => manifest.keywords?.includes('ember-addon'));

	return inMonorepoAddons.map((manifest) => manifest.name);
}

const sideWatch = require('@embroider/broccoli-side-watch');

async function watchLibraries(projectDir) {
	const { packageUp } = await import('package-up');

	const libraries = await addons(projectDir);

	const promises = libraries.map(async (libraryName) => {
		const entry = require.resolve(libraryName, { paths: [projectDir] });
		const manifestPath = await packageUp({ cwd: entry });
		const packagePath = path.dirname(manifestPath);
		const manifest = require(manifestPath);

		if (!manifest.files) {
			return;
		}

		const toWatch = manifest.files.map((f) => path.join(packagePath, f));

		return toWatch;
	});

	const paths = (await Promise.all(promises)).flat().filter(Boolean);

	const relative = paths
		.filter((p) => {
			const repoRelative = p.replace(monorepoRoot, '');

			if (!fs.existsSync(p)) {
				// eslint-disable-next-line no-console
				console.warn(`Path ${repoRelative} doesn't exist. It will not be watched.`);
				return false;
			}

			if (!fs.lstatSync(p).isDirectory()) {
				// eslint-disable-next-line no-console
				console.warn(`Path ${repoRelative} is not a directory. It will not be watched.`);
				return false;
			}

			// NOTE: We don't have any libraries that don't need compilation today,
			//       but we might some day.
			return !p.endsWith('/src');
		})
		.map((p) => path.relative(projectDir, p));

	return sideWatch('app', { watching: relative });
}

module.exports = { addons, watchLibraries };

then in ember-cli-build.js

module.exports = async function (defaults) {
	const app = new EmberApp(defaults, {
		'trees': {
			app: await require('the-path-or-place-to-the-above').watchLibraries(__dirname),
		},

[simonihmig]

Is that not working in general, or some special cases?

Because it is definitely working for me in the common case: test-app on Embroider v3 having a dependency on v2 addon, that is rebuilding.

Not entirely sure when broccoli-side-watch needs to come into play. Been using it for watching a plain npm package (no addon). This comment is also interesting: https://github.com/embroider-build/embroider/blob/main/packages/broccoli-side-watch/index.js#L18-L20

[NullVoxPopuli]

Could be environment related, I suppose. I know watch behaviors are wildly different between machines that have watchman installed, vs using a native inotify -- though, I was testing this on MacOS as well, and embroider doesn't pick up changes from addons without the side-watcher.

[NullVoxPopuli]

Just set this up in another project, here is a full ember-cli-build.js

'use strict';

const EmberApp = require('ember-cli/lib/broccoli/ember-app');
const path = require('path');
const fs = require('fs');

module.exports = async function (defaults) {
  const { readPackageUpSync } = await import('read-package-up');

  let app = new EmberApp(defaults, {
    trees: {
      app: (() => {
        let sideWatch = require('@embroider/broccoli-side-watch');

        let paths = ['ember-primitives'].map((libraryName) => {
          let entry = require.resolve(libraryName);
          let { packageJson, path: packageJsonPath } = readPackageUpSync({ cwd: entry });
          let packagePath = path.dirname(packageJsonPath);

          console.debug(
            `Side-watching ${libraryName} from ${packagePath}, which started in ${entry}`
          );

          let toWatch = packageJson.files
            .map((f) => path.join(packagePath, f))
            .filter((p) => {
              if (!fs.existsSync(p)) return false;
              if (!fs.lstatSync(p).isDirectory()) return false;

              return !p.endsWith('/src');
            });

          return toWatch;
        });

        return sideWatch('app', { watching: paths.flat() });
      })(),
    },
    'ember-cli-babel': {
      enableTypeScriptTransform: true,
    },
  });

  const { Webpack } = require('@embroider/webpack');

  return require('@embroider/compat').compatBuild(app, Webpack, {
    extraPublicTrees: [],
    staticAddonTrees: true,
    staticAddonTestSupportTrees: true,
    staticHelpers: true,
    staticModifiers: true,
    staticComponents: true,
    staticEmberSource: true,
    packagerOptions: {
      webpackConfig: {
        // Slow, but very nice
        devtool: 'source-map',
      },
    },
  });
};

[simonihmig]

Because it is definitely working for me in the common case: test-app on Embroider v3 having a dependency on v2 addon, that is rebuilding.

It turned out this was a side-effect of some internal v1 addon we have that was integrating into the build, including processing v2 files as part of its broccoli magic. That apparently created implicit WatchDir's for v2 addons' dist folder. With that addon removed, I can confirm that Embroider apps don't react to any changes of v2 addons ootb. 😞

[bartocc]

So broccoli-side-watch and @NullVoxPopuli ember-cli-build example is the required setup ?

The env var EMBROIDER_REBUILD_ADDONS is stale?

[simonihmig]

Note that the latest version of broccoli-side-watch has become smarter by allowing you to pass package names (just like eai's watchDependencies) and will resolve the to-be-watched folders automatically. So you don't need much of the boilerplate shown above!

I have found a related issue though, posting here for visibility: #2194