diff --git a/package.json b/package.json index c55b3215..d1aa96ca 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "devDependencies": { "@types/jest": "^26.0.12", "@types/node": "^14.6.2", + "@types/write": "^2.0.0", "@typescript-eslint/eslint-plugin": "^4.0.1", "@typescript-eslint/parser": "^4.0.1", "eslint": "^7.8.1", @@ -48,7 +49,8 @@ "ts-jest": "^26.3.0", "typescript": "^4.0.2", "yalc": "^1.0.0-pre.44", - "jsonschema": "^1.2.11" + "jsonschema": "^1.2.11", + "write": "^2.0.0" }, "dependencies": { "@types/lodash.omit": "^4.5.6", @@ -66,7 +68,6 @@ "queue": "^6.0.1", "@snyk/fast-glob": "^3.2.6-patch", "micromatch": "^4.0.2", - "flat-cache": "^3.0.4", "@types/uuid": "^8.3.0", "uuid": "^8.3.2" } diff --git a/src/cache.ts b/src/cache.ts new file mode 100644 index 00000000..f03be66e --- /dev/null +++ b/src/cache.ts @@ -0,0 +1,72 @@ +//This is our own implementation of flat-cache without the use of flattened as we do not need cicular JSON support +//and the executable for flattened was broken +import path from 'path'; +import fs from 'fs'; + +export class Cache { + public visited = {}; + public persisted = {}; + public pathToFile = ''; + constructor(docId: string, cacheDir?: any) { + this.pathToFile = cacheDir ? path.resolve(cacheDir, docId) : path.resolve(__dirname, '../.cache/', docId); + if (fs.existsSync(this.pathToFile)) { + this.persisted = tryParse(this.pathToFile, {}); + } + } + + public save(noPrune: boolean = false): void { + !noPrune && this.prune(); + writeJSON(this.pathToFile, this.persisted); + } + + public getKey(key: string): any { + this.visited[key] = true; + return this.persisted[key]; + } + + public setKey(key: string, value: any): void { + this.visited[key] = true; + this.persisted[key] = value; + } + private prune() { + let obj = {}; + + const keys = Object.keys(this.visited); + + // no keys visited for either get or set value + if (keys.length === 0) { + return; + } + + keys.forEach(key => { + obj[key] = this.persisted[key]; + }); + + this.visited = {}; + this.persisted = obj; + } +} + +function writeJSON(filePath: string, data: any): void { + fs.mkdirSync(path.dirname(filePath), { + recursive: true, + }); + fs.writeFileSync(filePath, JSON.stringify(data)); +} +function tryParse(filePath: string, defaultValue: any): JSON { + let result; + try { + result = readJSON(filePath); + } catch (ex) { + result = defaultValue; + } + return result; +} + +export function readJSON(filePath: string): JSON { + return JSON.parse( + fs.readFileSync(filePath, { + encoding: 'utf8', + }), + ); +} diff --git a/src/files.ts b/src/files.ts index 9b68a1f5..15dbd131 100644 --- a/src/files.ts +++ b/src/files.ts @@ -5,8 +5,7 @@ import micromatch from 'micromatch'; import crypto from 'crypto'; import union from 'lodash.union'; import util from 'util'; -import * as flatCache from 'flat-cache'; - +import { Cache } from './cache'; import { HASH_ALGORITHM, ENCODE_TYPE, MAX_PAYLOAD, IGNORES_DEFAULT, IGNORE_FILES_NAMES, CACHE_KEY } from './constants'; import { ISupportedFiles, IFileInfo } from './interfaces/files.interface'; @@ -165,8 +164,7 @@ export async function* collectBundleFiles( maxFileSize = MAX_PAYLOAD, symlinksEnabled = false, ): AsyncGenerator { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access - const cache = flatCache.load(CACHE_KEY, baseDir); + const cache = new Cache(CACHE_KEY, baseDir); const files = []; const dirs = []; @@ -208,7 +206,6 @@ export async function* collectBundleFiles( } } - // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access cache.save(); } @@ -222,7 +219,7 @@ export async function prepareExtendingBundle( ): Promise<{ files: IFileInfo[]; removedFiles: string[] }> { let removedFiles: string[] = []; let bundleFiles: IFileInfo[] = []; - const cache = flatCache.load(CACHE_KEY, baseDir); + const cache = new Cache(CACHE_KEY, baseDir); // Filter for supported extensions/files only let processingFiles: string[] = filterSupportedFiles(files, supportedFiles); @@ -283,7 +280,7 @@ export async function getFileInfo( filePath: string, baseDir: string, withContent = false, - cache: flatCache.Cache | null = null, + cache: Cache | null = null, ): Promise { const fileStats = await lStat(filePath); if (fileStats === null) { @@ -300,11 +297,9 @@ export async function getFileInfo( let fileHash = ''; if (!withContent && !!cache) { // Try to get hash from cache - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access const cachedData: CachedData | null = cache.getKey(filePath); if (cachedData) { if (cachedData[0] === fileStats.size && cachedData[1] === fileStats.mtimeMs) { - // eslint-disable-next-line prefer-destructuring fileHash = cachedData[2]; } else { // console.log(`did not match cache for: ${filePath} | ${cachedData} !== ${[fileStats.size, fileStats.mtime]}`); @@ -316,7 +311,6 @@ export async function getFileInfo( try { fileContent = fs.readFileSync(filePath, { encoding: 'utf8' }); fileHash = calcHash(fileContent); - // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access cache?.setKey(filePath, [fileStats.size, fileStats.mtimeMs, fileHash]); } catch (err) { // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access @@ -338,15 +332,13 @@ export async function getFileInfo( } export async function resolveBundleFiles(baseDir: string, bundleMissingFiles: string[]): Promise { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access - const cache = flatCache.load('.dccache', baseDir); + const cache = new Cache('.dccache', baseDir); const tasks = bundleMissingFiles.map(mf => { const filePath = resolveBundleFilePath(baseDir, mf); return getFileInfo(filePath, baseDir, true, cache); }); const res = (await Promise.all(tasks)).filter(notEmpty); - // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access cache.save(true); return res; } diff --git a/tests/__snapshots__/api.spec.ts.snap b/tests/__snapshots__/api.spec.ts.snap new file mode 100644 index 00000000..823d71f8 --- /dev/null +++ b/tests/__snapshots__/api.spec.ts.snap @@ -0,0 +1,87 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Requests to public API test successful workflow with and without linters 1`] = ` +Object { + "2": Array [ + Object { + "cols": Array [ + 8, + 27, + ], + "fingerprints": Array [ + Object { + "fingerprint": "f8e3391465a47f6586489cffd1f44ae47a1c4885c722de596d6eb931fe43bb16", + "version": 0, + }, + ], + "markers": Array [], + "rows": Array [ + 5, + 5, + ], + }, + ], + "3": Array [ + Object { + "cols": Array [ + 6, + 25, + ], + "fingerprints": Array [ + Object { + "fingerprint": "3e40a81739245db8fff4903a7e28e08bffa03486a677e7c91594cfdf15fb5a1d", + "version": 0, + }, + Object { + "fingerprint": "57664a44.2c254dac.98501263.9e345555.da547a36.9509b717.a713c1c8.45d76bdf.57664a44.2c254dac.98501263.9e345555.da547a36.9509b717.a713c1c8.45d76bdf", + "version": 1, + }, + ], + "markers": Array [ + Object { + "msg": Array [ + 25, + 36, + ], + "pos": Array [ + Object { + "cols": Array [ + 7, + 14, + ], + "file": "/AnnotatorTest.cpp", + "rows": Array [ + 8, + 8, + ], + }, + ], + }, + Object { + "msg": Array [ + 45, + 57, + ], + "pos": Array [ + Object { + "cols": Array [ + 6, + 25, + ], + "file": "/AnnotatorTest.cpp", + "rows": Array [ + 10, + 10, + ], + }, + ], + }, + ], + "rows": Array [ + 10, + 10, + ], + }, + ], +} +`; diff --git a/tests/api.spec.ts b/tests/api.spec.ts index d55e5821..e89957f6 100644 --- a/tests/api.spec.ts +++ b/tests/api.spec.ts @@ -405,55 +405,7 @@ describe('Requests to public API', () => { expect(suggestion.tags).toEqual(['maintenance', 'express', 'server', 'helmet']); expect(Object.keys(response.value.analysisResults.files).length).toEqual(4); const filePath = `/AnnotatorTest.cpp`; - expect(response.value.analysisResults.files[filePath]).toEqual({ - '2': [ - { - cols: [8, 27], - markers: [], - rows: [5, 5], - fingerprints: [ - { - fingerprint: 'f8e3391465a47f6586489cffd1f44ae47a1c4885c722de596d6eb931fe43bb16', - version: 0, - }, - ], - }, - ], - '3': [ - { - cols: [6, 25], - markers: [ - { - msg: [25, 36], - pos: [ - { - cols: [7, 14], - rows: [8, 8], - file: filePath, - }, - ], - }, - { - msg: [45, 57], - pos: [ - { - cols: [6, 25], - rows: [10, 10], - file: filePath, - }, - ], - }, - ], - rows: [10, 10], - fingerprints: [ - { - fingerprint: '3e40a81739245db8fff4903a7e28e08bffa03486a677e7c91594cfdf15fb5a1d', - version: 0, - }, - ], - }, - ], - }); + expect(response.value.analysisResults.files[filePath]).toMatchSnapshot(); expect(response.value.analysisResults.timing.analysis).toBeGreaterThanOrEqual( response.value.analysisResults.timing.fetchingCode, diff --git a/tests/cache.spec.ts b/tests/cache.spec.ts new file mode 100644 index 00000000..e7aff1a8 --- /dev/null +++ b/tests/cache.spec.ts @@ -0,0 +1,58 @@ +import path from 'path'; +import write from 'write'; +import fs from 'fs'; +import { Cache, readJSON } from '../src/cache'; +describe('Cache', () => { + afterAll(() => { + const dir = path.resolve(__dirname, '../fixtures'); + fs.rmdir(dir, { recursive: true }, err => { + if (err) { + throw err; + } + }); + }); + it('should not crash if the cache file exists but it is an empty string', function () { + const cachePath = path.resolve(__dirname, '../fixtures/.cache2'); + write.sync(path.join(cachePath, 'someId'), ''); + + expect(function () { + const cache = new Cache('someId', cachePath); + expect(cache.persisted).toEqual({}); + }).not.toThrow(Error); + }); + + it('should not crash if the cache file exists but it is an invalid JSON string', function () { + const cachePath = path.resolve(__dirname, '../fixtures/.cache2'); + write.sync(path.join(cachePath, 'someId'), '{ "foo": "fookey", "bar" '); + + expect(function () { + const cache = new Cache('someId', cachePath); + expect(cache.persisted).toEqual({}); + }).not.toThrow(Error); + }); + + describe('loading an existing cache custom directory', function () { + beforeEach(function () { + const cache = new Cache('someId', path.resolve(__dirname, '../fixtures/.cache2')); + cache.setKey('foo', { + bar: 'baz', + }); + cache.setKey('bar', { + foo: 'baz', + }); + cache.save(); + }); + + it('should load an existing cache', function () { + const cache = new Cache('someId', path.resolve(__dirname, '../fixtures/.cache2')); + expect(readJSON(path.resolve(__dirname, '../fixtures/.cache2/someId'))).toEqual(cache.persisted); + }); + + it('should return the same structure if load called twice with the same docId', function () { + const cache = new Cache('someId', path.resolve(__dirname, '../fixtures/.cache2')); + const cache2 = new Cache('someId', path.resolve(__dirname, '../fixtures/.cache2')); + + expect(cache.persisted).toEqual(cache2.persisted); + }); + }); +});