From 6be08e78937183d816d13a350419d3eb273c12be Mon Sep 17 00:00:00 2001 From: Erlend Oftedal Date: Mon, 6 Feb 2023 20:31:48 +0100 Subject: [PATCH] Rewrite to TypeScript --- .eslintrc.js | 29 - .eslintrc.json | 13 + .gitignore | 3 +- .prettierrc.json | 8 + .vscode/settings.json | 7 + CHANGELOG.md | 26 +- appLayerCreator.js | 232 --- bin/doqr | 2 +- cli.js | 219 --- fileutil.js | 17 - logger.js | 24 - package-lock.json | 3131 ++++++++++++++++++++++++++++++++++++++-- package.json | 73 +- publish.sh | 3 + registry.js | 292 ---- src/appLayerCreator.ts | 256 ++++ src/cli.ts | 248 ++++ src/fileutil.ts | 17 + src/logger.ts | 34 + src/registry.ts | 304 ++++ src/tarExporter.ts | 61 + src/types.ts | 78 + src/utils.ts | 12 + tarExporter.js | 54 - tsconfig.json | 11 + 25 files changed, 4143 insertions(+), 1011 deletions(-) delete mode 100644 .eslintrc.js create mode 100644 .eslintrc.json create mode 100644 .prettierrc.json create mode 100644 .vscode/settings.json delete mode 100644 appLayerCreator.js delete mode 100755 cli.js delete mode 100644 fileutil.js delete mode 100644 logger.js delete mode 100644 registry.js create mode 100644 src/appLayerCreator.ts create mode 100755 src/cli.ts create mode 100644 src/fileutil.ts create mode 100644 src/logger.ts create mode 100644 src/registry.ts create mode 100644 src/tarExporter.ts create mode 100644 src/types.ts create mode 100644 src/utils.ts delete mode 100644 tarExporter.js create mode 100644 tsconfig.json diff --git a/.eslintrc.js b/.eslintrc.js deleted file mode 100644 index 2dc7527..0000000 --- a/.eslintrc.js +++ /dev/null @@ -1,29 +0,0 @@ -module.exports = { - "env": { - "es6": true, - "node": true - }, - "extends": "eslint:recommended", - "parserOptions": { - "ecmaVersion": 2018, - "sourceType": "module" - }, - "rules": { - "indent": [ - "error", - 2 - ], - "linebreak-style": [ - "error", - "unix" - ], - "quotes": [ - "error", - "single" - ], - "semi": [ - "error", - "always" - ] - } -}; \ No newline at end of file diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 0000000..c755ca6 --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,13 @@ +{ + "env": { + "browser": false, + "es2021": true + }, + "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended", "prettier"], + "parser": "@typescript-eslint/parser", + "parserOptions": { + "ecmaVersion": "latest", + "sourceType": "module" + }, + "plugins": ["@typescript-eslint"] +} diff --git a/.gitignore b/.gitignore index 1d97fe2..bc5cb6b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules/ tmp/ -.idea \ No newline at end of file +.idea +lib/ diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 0000000..bafd016 --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1,8 @@ +{ + "semi": true, + "trailingComma": "all", + "singleQuote": false, + "tabWidth": 2, + "useTabs": true, + "printWidth": 120 +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..12a8a85 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "editor.codeActionsOnSave": { + "source.fixAll.eslint": true + }, + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.formatOnSave": true +} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ccf70c..b94aa63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,79 +1,95 @@ # Changelog +## [1.0.0] - 2022-02-06 + +### Rewrite + +- Rewritten to TypeScript + ## [0.6.0] - 2022-02-03 ### Added + - Support for config as .json-file ## [0.5.0] - 2022-01-19 ### Added + - Support for adding environment variables similar to labels ### Fixed + - Don't try to add labels if none are specified ## [0.4.2] - 2023-01-19 ### Fixed -- Initialize `config.container_config` in case it's `undefined` +- Initialize `config.container_config` in case it's `undefined` ## [0.4.1] - 2022-03-24 ### Dependency update / security + - Update minimist due to vuln in 1.2.5: https://github.com/advisories/GHSA-xvch-5gv4-984h ## [0.4.0] - 2021-04-12 ### Added + - Allow specifying multiple labels by using `--label` multiple times ## [0.3.2] - 2021-04-12 ### Fixed -- Update help and readme +- Update help and readme ## [0.3.1] - 2021-02-10 ### Fixed -- Unwanted debug logging +- Unwanted debug logging ## [0.3.0] - 2021-02-10 ### Added -- Possible to add additional files/folders to specified destinations in the image with `--extraContent` +- Possible to add additional files/folders to specified destinations in the image with `--extraContent` ## [0.2.0] - 2020-08-05 ### Added -- Support for setting the owner of the work folder (gid:uid) +- Support for setting the owner of the work folder (gid:uid) ## [0.1.0] - 2020-03-30 ### Added + - Support custom layer (drops default entrypoint, user, workdir, node_modules layer and app layer). Only adds the specified files ## [0.0.11] - 2020-03-30 ### Modified + - Don't include .git and .gitignore if in same folder ## [0.0.10] - 2020-03-08 ### Modified + - Updated the README with improved example and missing option for the timestamp ## [0.0.9] - 2020-03-08 ### Removed + - Removed files from npm package and simplified package.json to use defaults ## [0.0.8] - 2020-03-08 ### Added + - Ability to set a specific timestamp on all files/tars/configs to support hermetic builds. Typically one would use the git commit time (`--setTimeStamp=$(git show -s --format="%aI" HEAD)`). If omitted, the timestamp is set to epoch 0. [`420e248`](https://github.com/eoftedal/doqr/commit/420e248e4daf5470e91834f11a52633a566f5783) diff --git a/appLayerCreator.js b/appLayerCreator.js deleted file mode 100644 index 135332b..0000000 --- a/appLayerCreator.js +++ /dev/null @@ -1,232 +0,0 @@ -const tar = require('tar'); -const fs = require('fs').promises; -const fse = require('fs-extra'); -const fss = require('fs'); -const path = require('path'); -const crypto = require('crypto'); -const Gunzip = require('minizlib').Gunzip; - -const fileutil = require('./fileutil'); -const logger = require('./logger'); - -const depLayerPossibles = ['package.json', 'package-lock.json', 'node_modules']; - -const ignore = ['.git', '.gitignore', '.npmrc', '.DS_Store', 'npm-debug.log', '.svn', '.hg', 'CVS'] - - - -function statCache(layerOwner) { - if (!layerOwner) return null; - // We use the stat cache to overwrite uid and gid in image. - // A bit hacky - const statCacheMap = new Map(); - const a = layerOwner.split(":"); - const gid = parseInt(a[0]); - const uid = parseInt(a[1]); - return { - get: function(name) { - if (statCacheMap.has(name)) return statCacheMap.get(name); - let stat = fss.statSync(name); - stat.uid = uid; - stat.gid = gid; - stat.atime = new Date(0); - stat.mtime = new Date(0); - stat.ctime = new Date(0); - stat.birthtime = new Date(0); - stat.atimeMs = 0; - stat.mtimeMs = 0; - stat.ctimeMs = 0; - stat.birthtimeMs = 0; - statCacheMap.set(name, stat); - return stat; - }, - set: function(name, stat) { - statCacheMap.set(name, stat); - }, - has: function(name) { - return true; - } - }; -} - - -const tarDefaultConfig = { - preservePaths: false, - follow: true -}; - -function calculateHashOfBuffer(buf) { - let hash = crypto.createHash('sha256'); - hash.update(buf); - return hash.digest('hex'); -} - -function calculateHash(path) { - return new Promise((resolve, reject) => { - let hash = crypto.createHash('sha256'); - let stream = require('fs').createReadStream(path); - stream.on('error', err => reject(err)); - stream.on('data', chunk => hash.update(chunk)); - stream.on('end', () => resolve(hash.digest('hex'))); - }); -} - -function copySync(src, dest) { - const copyOptions = { overwrite: true, dereference: true }; - let destFolder = dest.substring(0, dest.lastIndexOf('/')); - logger.debug('Copying ' + src + ' to ' + dest); - fse.ensureDirSync(destFolder) - fse.copySync(src, dest, copyOptions); -} - -function addEmptyLayer(config, options, operation, action) { - logger.info(`Applying ${operation}`); - config.history.push({ - created: options.setTimeStamp || new Date().toISOString(), - created_by: '/bin/sh -c #(nop) ' + operation, - empty_layer: true - }); - action(config); -} - -async function getHashOfUncompressed(file) { - return new Promise((resolve, reject) => { - let hash = crypto.createHash('sha256'); - let gunzip = new Gunzip(); - gunzip.on('data', chunk => hash.update(chunk)); - gunzip.on('end', () => resolve(hash.digest('hex'))); - gunzip.on('error', err => reject(err)); - require('fs').createReadStream(file).pipe(gunzip).on('error', err => reject(err)); - }); -} - -async function addDataLayer(tmpdir, todir, options, config, layers, files, comment) { - logger.info('Adding layer for ' + comment + ' ...'); - let buildDir = await fileutil.ensureEmptyDir(path.join(tmpdir, 'build')); - files.map(f => { - if (Array.isArray(f)) { - copySync(path.join(options.folder, f[0]), path.join(buildDir, f[1])); - } else { - copySync(path.join(options.folder, f), path.join(buildDir, options.workdir, f)); - } - }); - let layerFile = path.join(todir, 'layer.tar.gz'); - if (options.layerOwner) logger.info("Setting file ownership to: " + options.layerOwner) - let filesToTar = fss.readdirSync(buildDir); - await tar.c(Object.assign({}, tarDefaultConfig, { - statCache: statCache(options.layerOwner), - portable: !options.layerOwner, - prefix: "/", - cwd: buildDir, - file: layerFile, - gzip: true, - noMtime: (!options.setTimeStamp), - mtime: options.setTimeStamp - }), filesToTar); - let fhash = await calculateHash(layerFile); - let finalName = path.join(todir, fhash + '.tar.gz'); - await fse.move(layerFile, finalName); - layers.push({ - mediaType : 'application/vnd.docker.image.rootfs.diff.tar.gzip', - size: await fileutil.sizeOf(finalName), - digest: 'sha256:' + fhash - }); - let dhash = await getHashOfUncompressed(finalName); - config.rootfs.diff_ids.push('sha256:' + dhash); - config.history.push({ - created: options.setTimeStamp || new Date().toISOString(), - created_by: 'doqr', - comment: comment - }); -} - - -async function copyLayers(fromdir, todir, layers) { - await Promise.all(layers.map(async layer => { - let file = layer.digest.split(':')[1] + (layer.mediaType.includes('tar.gzip') ? '.tar.gz' : '.tar'); - await fse.copy(path.join(fromdir, file), path.join(todir, file)); - })); -} - -function parseCommandLineToParts(entrypoint) { - return entrypoint.split('"') - .map((p,i) => { - if (i % 2 == 1) return [p]; - return p.split(' '); - }) - .reduce((a, b) => a.concat(b), []) - .filter(a => a != ''); -} - -async function addAppLayers(options, config, todir, manifest, tmpdir) { - if (options.customContent) { - await addEnvsLayer(options, config); - await addLabelsLayer(options, config); - await addDataLayer(tmpdir, todir, options, config, manifest.layers, options.customContent, 'custom'); - } else { - addEmptyLayer(config, options, `WORKDIR ${options.workdir}`, config => config.config.WorkingDir = options.workdir); - let entrypoint = parseCommandLineToParts(options.entrypoint); - addEmptyLayer(config, options, `ENTRYPOINT ${JSON.stringify(entrypoint)}`, config => config.config.Entrypoint = entrypoint); - addEmptyLayer(config, options, `USER ${options.user}`, config => { - config.config.user = options.user; - config.container_config.user = options.user; - }); - await addEnvsLayer(options, config); - await addLabelsLayer(options, config); - let appFiles = (await fs.readdir(options.folder)).filter(l => !ignore.includes(l)); - let depLayerContent = appFiles.filter(l => depLayerPossibles.includes(l)); - let appLayerContent = appFiles.filter(l => !depLayerPossibles.includes(l)); - - await addDataLayer(tmpdir, todir, options, config, manifest.layers, depLayerContent, 'dependencies'); - await addDataLayer(tmpdir, todir, options, config, manifest.layers, appLayerContent, 'app'); - } - if (options.extraContent) { - for (let i in options.extraContent) { - await addDataLayer(tmpdir, todir, options, config, manifest.layers, [options.extraContent[i]], 'extra'); - } - } -} - -async function addLabelsLayer(options, config) { - if (Object.keys(options.labels).length > 0) { - addEmptyLayer(config, options, `LABELS ${JSON.stringify(options.labels)}`, config => { - config.config.labels = options.labels; - config.container_config.labels = options.labels; - }); - } -} - -async function addEnvsLayer(options, config) { - if (options.envs.length > 0) { - addEmptyLayer(config, options, `ENV ${JSON.stringify(options.envs)}`, config => { - // Keep old environment variables - config.config.env = [...config.config['Env'], ...options.envs]; - config.container_config.env = options.envs; - }); - } -} - -async function addLayers(tmpdir, fromdir, todir, options) { - logger.info('Parsing image ...'); - let manifest = await fse.readJson(path.join(fromdir, 'manifest.json')); - let config = await fse.readJson(path.join(fromdir, 'config.json')); - config.container_config = config.container_config || {}; - - logger.info('Adding new layers...'); - await copyLayers(fromdir, todir, manifest.layers); - await addAppLayers(options, config, todir, manifest, tmpdir); - - logger.info('Writing final image...'); - let configContent = Buffer.from(JSON.stringify(config)); - let configHash = calculateHashOfBuffer(configContent); - let configFile = path.join(todir, configHash + '.json'); - await fs.writeFile(configFile, configContent); - manifest.config.digest = 'sha256:' + configHash; - manifest.config.size = await fileutil.sizeOf(configFile); - await fs.writeFile(path.join(todir, 'manifest.json'), JSON.stringify(manifest)); -} - -module.exports = { - addLayers : addLayers -}; - diff --git a/bin/doqr b/bin/doqr index 9fbea32..e31b912 120000 --- a/bin/doqr +++ b/bin/doqr @@ -1 +1 @@ -../cli.js \ No newline at end of file +../lib/cli.js \ No newline at end of file diff --git a/cli.js b/cli.js deleted file mode 100755 index b6f2776..0000000 --- a/cli.js +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env node - -const os = require('os'); -const program = require('commander'); -const path = require('path'); -const fse = require('fs-extra'); -const fs = require('fs'); - -const Registry = require('./registry').Registry; -const DockerRegistry = require('./registry').DockerRegistry; -const appLayerCreator = require('./appLayerCreator'); -const fileutil = require('./fileutil'); -const tarExporter = require('./tarExporter'); - -const logger = require('./logger'); - - -const possibleArgs = { - '--fromImage ' : 'Required: Image name of base image - [path/]image:tag', - '--toImage ' : 'Required: Image name of target image - [path/]image:tag', - '--folder ' : 'Required: Base folder of node application (contains package.json)', - '--file ' : 'Optional: Name of configuration file (defaults to doqr.json if found on path)', - '--fromRegistry ' : 'Optional: URL of registry to pull base image from - Default: https://registry-1.docker.io/v2/', - '--fromToken ' : 'Optional: Authentication token for from registry', - '--toRegistry ' : 'Optional: URL of registry to push base image to - Default: https://registry-1.docker.io/v2/', - '--toToken ' : 'Optional: Authentication token for target registry', - '--toTar ' : 'Optional: Export to tar file', - '--registry ' : 'Optional: Convenience argument for setting both from and to registry', - '--token ' : 'Optional: Convenience argument for setting token for both from and to registry', - '--user ' : 'Optional: User account to run process in container - default: 1000', - '--workdir ' : 'Optional: Workdir where node app will be added and run from - default: /app', - '--entrypoint ' : 'Optional: Entrypoint when starting container - default: npm start', - '--labels ' : 'Optional: Comma-separated list of key value pairs to use as labels', - '--label