Skip to content

Commit

Permalink
Merge pull request #330 from serverless-heaven/329-packager-interface
Browse files Browse the repository at this point in the history 
Abstract packager into universal interface.
  • Loading branch information
@HyperBrain
HyperBrain authored Mar 1, 2018
2 parents d38a385 + fee4947 commit 1b8a3be
Show file tree
Hide file tree
Showing 8 changed files with 663 additions and 409 deletions.
1 change: 1 addition & 0 deletions .npmignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@ node_modules
coverage
examples
tests
*.test.js
268 changes: 111 additions & 157 deletions lib/packExternalModules.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@
const BbPromise = require('bluebird');
const _ = require('lodash');
const path = require('path');
const childProcess = require('child_process');
const fse = require('fs-extra');
const isBuiltinModule = require('is-builtin-module');

const Packagers = require('./packagers');

function rebaseFileReferences(pathToPackageRoot, moduleVersion) {
if (/^file:[^/]{2}/.test(moduleVersion)) {
const filePath = _.replace(moduleVersion, /^file:/, '');
Expand All @@ -16,18 +17,6 @@ function rebaseFileReferences(pathToPackageRoot, moduleVersion) {
return moduleVersion;
}

function rebasePackageLock(pathToPackageRoot, module) {
if (module.version) {
module.version = rebaseFileReferences(pathToPackageRoot, module.version);
}

if (module.dependencies) {
_.forIn(module.dependencies, moduleDependency => {
rebasePackageLock(pathToPackageRoot, moduleDependency);
});
}
}

/**
* Add the given modules to a package json's dependencies.
*/
Expand Down Expand Up @@ -200,156 +189,121 @@ module.exports = {
const packagePath = includes.packagePath || './package.json';
const packageJsonPath = path.join(process.cwd(), packagePath);

this.options.verbose && this.serverless.cli.log(`Fetch dependency graph from ${packageJsonPath}`);
// Get first level dependency graph
const command = 'npm ls -prod -json -depth=1'; // Only prod dependencies

const ignoredNpmErrors = [
{ npmError: 'extraneous', log: false },
{ npmError: 'missing', log: false },
{ npmError: 'peer dep missing', log: true },
];

return BbPromise.fromCallback(cb => {
childProcess.exec(command, {
cwd: path.dirname(packageJsonPath),
maxBuffer: this.serverless.service.custom.packExternalModulesMaxBuffer || 200 * 1024,
encoding: 'utf8'
}, (err, stdout, stderr) => {
if (err) {
// Only exit with an error if we have critical npm errors for 2nd level inside
const errors = _.split(stderr, '\n');
const failed = _.reduce(errors, (failed, error) => {
if (failed) {
return true;
}
return !_.isEmpty(error) && !_.some(ignoredNpmErrors, ignoredError => _.startsWith(error, `npm ERR! ${ignoredError.npmError}`));
}, false);

if (failed) {
return cb(err);
}
}
return cb(null, stdout);
});
})
.then(depJson => BbPromise.try(() => JSON.parse(depJson)))
.then(dependencyGraph => {
const problems = _.get(dependencyGraph, 'problems', []);
if (this.options.verbose && !_.isEmpty(problems)) {
this.serverless.cli.log(`Ignoring ${_.size(problems)} NPM errors:`);
_.forEach(problems, problem => {
this.serverless.cli.log(`=> ${problem}`);
});
}

// (1) Generate dependency composition
const compositeModules = _.uniq(_.flatMap(stats.stats, compileStats => {
const externalModules = _.concat(
getExternalModules.call(this, compileStats),
_.map(packageForceIncludes, whitelistedPackage => ({ external: whitelistedPackage }))
);
return getProdModules.call(this, externalModules, packagePath, dependencyGraph);
}));
removeExcludedModules.call(this, compositeModules, packageForceExcludes, true);

if (_.isEmpty(compositeModules)) {
// The compiled code does not reference any external modules at all
this.serverless.cli.log('No external modules needed');
return BbPromise.resolve();
}

// (1.a) Install all needed modules
const compositeModulePath = path.join(this.webpackOutputPath, 'dependencies');
const compositePackageJson = path.join(compositeModulePath, 'package.json');

// (1.a.1) Create a package.json
const compositePackage = {
name: this.serverless.service.service,
version: '1.0.0',
description: `Packaged externals for ${this.serverless.service.service}`,
private: true
};
const relPath = path.relative(compositeModulePath, path.dirname(packageJsonPath));
addModulesToPackageJson(compositeModules, compositePackage, relPath);
this.serverless.utils.writeFileSync(compositePackageJson, JSON.stringify(compositePackage, null, 2));

// (1.a.2) Copy package-lock.json if it exists, to prevent unwanted upgrades
const packageLockPath = path.join(path.dirname(packageJsonPath), 'package-lock.json');
return BbPromise.fromCallback(cb => fse.pathExists(packageLockPath, cb))
.then(exists => {
if (exists) {
this.serverless.cli.log('Package lock found - Using locked versions');
try {
const packageLockJson = this.serverless.utils.readFileSync(packageLockPath);
/**
* We should not be modifying 'package-lock.json'
* because this file should be treat as internal to npm.
*
* Rebase package-lock is a temporary workaround and must be
* removed as soon as https://github.com/npm/npm/issues/19183 gets fixed.
*/
rebasePackageLock(relPath, packageLockJson);

this.serverless.utils.writeFileSync(path.join(compositeModulePath, 'package-lock.json'), JSON.stringify(packageLockJson, null, 2));
} catch(err) {
this.serverless.cli.log(`Warning: Could not read lock file: ${err.message}`);
}
// Determine and create packager
return BbPromise.try(() => Packagers.get.call(this, 'npm'))
.then(packager => {
// Get first level dependency graph
this.options.verbose && this.serverless.cli.log(`Fetch dependency graph from ${packageJsonPath}`);
const maxExecBufferSize = this.serverless.service.custom.packExternalModulesMaxBuffer || 200 * 1024;

return packager.getProdDependencies(path.dirname(packageJsonPath), 1, maxExecBufferSize)
.then(dependencyGraph => {
const problems = _.get(dependencyGraph, 'problems', []);
if (this.options.verbose && !_.isEmpty(problems)) {
this.serverless.cli.log(`Ignoring ${_.size(problems)} NPM errors:`);
_.forEach(problems, problem => {
this.serverless.cli.log(`=> ${problem}`);
});
}
return BbPromise.resolve();
})
.then(() => {
const start = _.now();
this.serverless.cli.log('Packing external modules: ' + compositeModules.join(', '));
return BbPromise.fromCallback(cb => {
childProcess.exec('npm install', {
cwd: compositeModulePath,
maxBuffer: this.serverless.service.custom.packExternalModulesMaxBuffer || 200 * 1024,
encoding: 'utf8'
}, cb);
})
.then(() => this.options.verbose && this.serverless.cli.log(`Package took [${_.now() - start} ms]`))
.return(stats.stats);
})
.mapSeries(compileStats => {
const modulePath = compileStats.compilation.compiler.outputPath;

// Create package.json
const modulePackageJson = path.join(modulePath, 'package.json');
const modulePackage = {
dependencies: {}
};
const prodModules = getProdModules.call(this,
_.concat(

// (1) Generate dependency composition
const compositeModules = _.uniq(_.flatMap(stats.stats, compileStats => {
const externalModules = _.concat(
getExternalModules.call(this, compileStats),
_.map(packageForceIncludes, whitelistedPackage => ({ external: whitelistedPackage }))
), packagePath, dependencyGraph);
removeExcludedModules.call(this, prodModules, packageForceExcludes);
const relPath = path.relative(modulePath, path.dirname(packageJsonPath));
addModulesToPackageJson(prodModules, modulePackage, relPath);
this.serverless.utils.writeFileSync(modulePackageJson, JSON.stringify(modulePackage, null, 2));

// GOOGLE: Copy modules only if not google-cloud-functions
// GCF Auto installs the package json
if (_.get(this.serverless, 'service.provider.name') === 'google') {
);
return getProdModules.call(this, externalModules, packagePath, dependencyGraph);
}));
removeExcludedModules.call(this, compositeModules, packageForceExcludes, true);

if (_.isEmpty(compositeModules)) {
// The compiled code does not reference any external modules at all
this.serverless.cli.log('No external modules needed');
return BbPromise.resolve();
}

const startCopy = _.now();
return BbPromise.fromCallback(callback => fse.copy(path.join(compositeModulePath, 'node_modules'), path.join(modulePath, 'node_modules'), callback))
.tap(() => this.options.verbose && this.serverless.cli.log(`Copy modules: ${modulePath} [${_.now() - startCopy} ms]`))

// (1.a) Install all needed modules
const compositeModulePath = path.join(this.webpackOutputPath, 'dependencies');
const compositePackageJson = path.join(compositeModulePath, 'package.json');

// (1.a.1) Create a package.json
const compositePackage = {
name: this.serverless.service.service,
version: '1.0.0',
description: `Packaged externals for ${this.serverless.service.service}`,
private: true
};
const relPath = path.relative(compositeModulePath, path.dirname(packageJsonPath));
addModulesToPackageJson(compositeModules, compositePackage, relPath);
this.serverless.utils.writeFileSync(compositePackageJson, JSON.stringify(compositePackage, null, 2));

// (1.a.2) Copy package-lock.json if it exists, to prevent unwanted upgrades
const packageLockPath = path.join(path.dirname(packageJsonPath), packager.lockfileName);
return BbPromise.fromCallback(cb => fse.pathExists(packageLockPath, cb))
.then(exists => {
if (exists) {
this.serverless.cli.log('Package lock found - Using locked versions');
try {
const packageLockJson = this.serverless.utils.readFileSync(packageLockPath);
/**
* We should not be modifying 'package-lock.json'
* because this file should be treat as internal to npm.
*
* Rebase package-lock is a temporary workaround and must be
* removed as soon as https://github.com/npm/npm/issues/19183 gets fixed.
*/
packager.rebaseLockfile(relPath, packageLockJson);

this.serverless.utils.writeFileSync(path.join(compositeModulePath, packager.lockfileName), JSON.stringify(packageLockJson, null, 2));
} catch(err) {
this.serverless.cli.log(`Warning: Could not read lock file: ${err.message}`);
}
}
return BbPromise.resolve();
})
.then(() => {
// Prune extraneous packages - removes not needed ones
const startPrune = _.now();
return BbPromise.fromCallback(callback => {
childProcess.exec('npm prune', {
cwd: modulePath
}, callback);
})
.tap(() => this.options.verbose && this.serverless.cli.log(`Prune: ${modulePath} [${_.now() - startPrune} ms]`));
});
})
.return();
const start = _.now();
this.serverless.cli.log('Packing external modules: ' + compositeModules.join(', '));
return packager.install(compositeModulePath, maxExecBufferSize)
.then(() => this.options.verbose && this.serverless.cli.log(`Package took [${_.now() - start} ms]`))
.return(stats.stats);
})
.mapSeries(compileStats => {
const modulePath = compileStats.compilation.compiler.outputPath;

// Create package.json
const modulePackageJson = path.join(modulePath, 'package.json');
const modulePackage = {
dependencies: {}
};
const prodModules = getProdModules.call(this,
_.concat(
getExternalModules.call(this, compileStats),
_.map(packageForceIncludes, whitelistedPackage => ({ external: whitelistedPackage }))
), packagePath, dependencyGraph);
removeExcludedModules.call(this, prodModules, packageForceExcludes);
const relPath = path.relative(modulePath, path.dirname(packageJsonPath));
addModulesToPackageJson(prodModules, modulePackage, relPath);
this.serverless.utils.writeFileSync(modulePackageJson, JSON.stringify(modulePackage, null, 2));

// GOOGLE: Copy modules only if not google-cloud-functions
// GCF Auto installs the package json
if (_.get(this.serverless, 'service.provider.name') === 'google') {
return BbPromise.resolve();
}

const startCopy = _.now();
return BbPromise.fromCallback(callback => fse.copy(path.join(compositeModulePath, 'node_modules'), path.join(modulePath, 'node_modules'), callback))
.tap(() => this.options.verbose && this.serverless.cli.log(`Copy modules: ${modulePath} [${_.now() - startCopy} ms]`))
.then(() => {
// Prune extraneous packages - removes not needed ones
const startPrune = _.now();
return packager.prune(modulePath, maxExecBufferSize)
.tap(() => this.options.verbose && this.serverless.cli.log(`Prune: ${modulePath} [${_.now() - startPrune} ms]`));
});
})
.return();
});
});
}
};
38 changes: 38 additions & 0 deletions lib/packagers/index.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
'use strict';
/**
* Factory for supported packagers.
*
* All packagers must implement the following interface:
*
* interface Packager {
*
* static get lockfileName(): string;
* static getProdDependencies(cwd: string, depth: number = 1, maxExecBufferSize = undefined): BbPromise<Object>;
* static rebaseLockfile(pathToPackageRoot: string, lockfile: Object): void;
* static install(cwd: string, maxExecBufferSize = undefined): BbPromise<void>;
* static prune(cwd: string): BbPromise<void>;
*
* }
*/

const _ = require('lodash');
const npm = require('./npm');

const registeredPackagers = {
npm: npm
};

/**
* Factory method.
* @this ServerlessWebpack - Active plugin instance
* @param {string} packagerId - Well known packager id.
* @returns {BbPromise<Packager>} - Promised packager to allow packagers be created asynchronously.
*/
module.exports.get = function(packagerId) {
if (!_.has(registeredPackagers, packagerId)) {
const message = `Could not find packager '${packagerId}'`;
this.serverless.cli.log(`ERROR: ${message}`);
throw new this.serverless.classes.Error(message);
}
return registeredPackagers[packagerId];
};
Loading

0 comments on commit 1b8a3be

Please sign in to comment.