Skip to content

Commit

Permalink
fix: flat-cache removed own custom implementation no support for circ…
Browse files Browse the repository at this point in the history
…ularJSON
  • Loading branch information
j-sp4 committed Mar 5, 2021
1 parent 723a957 commit dd86ef8
Show file tree
Hide file tree
Showing 6 changed files with 226 additions and 64 deletions.
5 changes: 3 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
"devDependencies": {
"@types/jest": "^26.0.12",
"@types/node": "^14.6.2",
"@types/write": "^2.0.0",
"@typescript-eslint/eslint-plugin": "^4.0.1",
"@typescript-eslint/parser": "^4.0.1",
"eslint": "^7.8.1",
Expand All @@ -48,7 +49,8 @@
"ts-jest": "^26.3.0",
"typescript": "^4.0.2",
"yalc": "^1.0.0-pre.44",
"jsonschema": "^1.2.11"
"jsonschema": "^1.2.11",
"write": "^2.0.0"
},
"dependencies": {
"@types/lodash.omit": "^4.5.6",
Expand All @@ -66,7 +68,6 @@
"queue": "^6.0.1",
"@snyk/fast-glob": "^3.2.6-patch",
"micromatch": "^4.0.2",
"flat-cache": "^3.0.4",
"@types/uuid": "^8.3.0",
"uuid": "^8.3.2"
}
Expand Down
72 changes: 72 additions & 0 deletions src/cache.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
//This is our own implementation of flat-cache without the use of flattened as we do not need cicular JSON support
//and the executable for flattened was broken
import path from 'path';
import fs from 'fs';

export class Cache {
public visited = {};
public persisted = {};
public pathToFile = '';
constructor(docId: string, cacheDir?: any) {
this.pathToFile = cacheDir ? path.resolve(cacheDir, docId) : path.resolve(__dirname, '../.cache/', docId);
if (fs.existsSync(this.pathToFile)) {
this.persisted = tryParse(this.pathToFile, {});
}
}

public save(noPrune: boolean = false): void {
!noPrune && this.prune();
writeJSON(this.pathToFile, this.persisted);
}

public getKey(key: string): any {
this.visited[key] = true;
return this.persisted[key];
}

public setKey(key: string, value: any): void {
this.visited[key] = true;
this.persisted[key] = value;
}
private prune() {
let obj = {};

const keys = Object.keys(this.visited);

// no keys visited for either get or set value
if (keys.length === 0) {
return;
}

keys.forEach(key => {
obj[key] = this.persisted[key];
});

this.visited = {};
this.persisted = obj;
}
}

function writeJSON(filePath: string, data: any): void {
fs.mkdirSync(path.dirname(filePath), {
recursive: true,
});
fs.writeFileSync(filePath, JSON.stringify(data));
}
function tryParse(filePath: string, defaultValue: any): JSON {
let result;
try {
result = readJSON(filePath);
} catch (ex) {
result = defaultValue;
}
return result;
}

export function readJSON(filePath: string): JSON {
return JSON.parse(
fs.readFileSync(filePath, {
encoding: 'utf8',
}),
);
}
18 changes: 5 additions & 13 deletions src/files.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,7 @@ import micromatch from 'micromatch';
import crypto from 'crypto';
import union from 'lodash.union';
import util from 'util';
import * as flatCache from 'flat-cache';

import { Cache } from './cache';
import { HASH_ALGORITHM, ENCODE_TYPE, MAX_PAYLOAD, IGNORES_DEFAULT, IGNORE_FILES_NAMES, CACHE_KEY } from './constants';

import { ISupportedFiles, IFileInfo } from './interfaces/files.interface';
Expand Down Expand Up @@ -165,8 +164,7 @@ export async function* collectBundleFiles(
maxFileSize = MAX_PAYLOAD,
symlinksEnabled = false,
): AsyncGenerator<IFileInfo> {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
const cache = flatCache.load(CACHE_KEY, baseDir);
const cache = new Cache(CACHE_KEY, baseDir);

const files = [];
const dirs = [];
Expand Down Expand Up @@ -208,7 +206,6 @@ export async function* collectBundleFiles(
}
}

// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
cache.save();
}

Expand All @@ -222,7 +219,7 @@ export async function prepareExtendingBundle(
): Promise<{ files: IFileInfo[]; removedFiles: string[] }> {
let removedFiles: string[] = [];
let bundleFiles: IFileInfo[] = [];
const cache = flatCache.load(CACHE_KEY, baseDir);
const cache = new Cache(CACHE_KEY, baseDir);

// Filter for supported extensions/files only
let processingFiles: string[] = filterSupportedFiles(files, supportedFiles);
Expand Down Expand Up @@ -283,7 +280,7 @@ export async function getFileInfo(
filePath: string,
baseDir: string,
withContent = false,
cache: flatCache.Cache | null = null,
cache: Cache | null = null,
): Promise<IFileInfo | null> {
const fileStats = await lStat(filePath);
if (fileStats === null) {
Expand All @@ -300,11 +297,9 @@ export async function getFileInfo(
let fileHash = '';
if (!withContent && !!cache) {
// Try to get hash from cache
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
const cachedData: CachedData | null = cache.getKey(filePath);
if (cachedData) {
if (cachedData[0] === fileStats.size && cachedData[1] === fileStats.mtimeMs) {
// eslint-disable-next-line prefer-destructuring
fileHash = cachedData[2];
} else {
// console.log(`did not match cache for: ${filePath} | ${cachedData} !== ${[fileStats.size, fileStats.mtime]}`);
Expand All @@ -316,7 +311,6 @@ export async function getFileInfo(
try {
fileContent = fs.readFileSync(filePath, { encoding: 'utf8' });
fileHash = calcHash(fileContent);
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
cache?.setKey(filePath, [fileStats.size, fileStats.mtimeMs, fileHash]);
} catch (err) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
Expand All @@ -338,15 +332,13 @@ export async function getFileInfo(
}

export async function resolveBundleFiles(baseDir: string, bundleMissingFiles: string[]): Promise<IFileInfo[]> {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
const cache = flatCache.load('.dccache', baseDir);
const cache = new Cache('.dccache', baseDir);
const tasks = bundleMissingFiles.map(mf => {
const filePath = resolveBundleFilePath(baseDir, mf);
return getFileInfo(filePath, baseDir, true, cache);
});

const res = (await Promise.all(tasks)).filter(notEmpty);
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
cache.save(true);
return res;
}
Expand Down
87 changes: 87 additions & 0 deletions tests/__snapshots__/api.spec.ts.snap
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP

exports[`Requests to public API test successful workflow with and without linters 1`] = `
Object {
"2": Array [
Object {
"cols": Array [
8,
27,
],
"fingerprints": Array [
Object {
"fingerprint": "f8e3391465a47f6586489cffd1f44ae47a1c4885c722de596d6eb931fe43bb16",
"version": 0,
},
],
"markers": Array [],
"rows": Array [
5,
5,
],
},
],
"3": Array [
Object {
"cols": Array [
6,
25,
],
"fingerprints": Array [
Object {
"fingerprint": "3e40a81739245db8fff4903a7e28e08bffa03486a677e7c91594cfdf15fb5a1d",
"version": 0,
},
Object {
"fingerprint": "57664a44.2c254dac.98501263.9e345555.da547a36.9509b717.a713c1c8.45d76bdf.57664a44.2c254dac.98501263.9e345555.da547a36.9509b717.a713c1c8.45d76bdf",
"version": 1,
},
],
"markers": Array [
Object {
"msg": Array [
25,
36,
],
"pos": Array [
Object {
"cols": Array [
7,
14,
],
"file": "/AnnotatorTest.cpp",
"rows": Array [
8,
8,
],
},
],
},
Object {
"msg": Array [
45,
57,
],
"pos": Array [
Object {
"cols": Array [
6,
25,
],
"file": "/AnnotatorTest.cpp",
"rows": Array [
10,
10,
],
},
],
},
],
"rows": Array [
10,
10,
],
},
],
}
`;
50 changes: 1 addition & 49 deletions tests/api.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -405,55 +405,7 @@ describe('Requests to public API', () => {
expect(suggestion.tags).toEqual(['maintenance', 'express', 'server', 'helmet']);
expect(Object.keys(response.value.analysisResults.files).length).toEqual(4);
const filePath = `/AnnotatorTest.cpp`;
expect(response.value.analysisResults.files[filePath]).toEqual({
'2': [
{
cols: [8, 27],
markers: [],
rows: [5, 5],
fingerprints: [
{
fingerprint: 'f8e3391465a47f6586489cffd1f44ae47a1c4885c722de596d6eb931fe43bb16',
version: 0,
},
],
},
],
'3': [
{
cols: [6, 25],
markers: [
{
msg: [25, 36],
pos: [
{
cols: [7, 14],
rows: [8, 8],
file: filePath,
},
],
},
{
msg: [45, 57],
pos: [
{
cols: [6, 25],
rows: [10, 10],
file: filePath,
},
],
},
],
rows: [10, 10],
fingerprints: [
{
fingerprint: '3e40a81739245db8fff4903a7e28e08bffa03486a677e7c91594cfdf15fb5a1d',
version: 0,
},
],
},
],
});
expect(response.value.analysisResults.files[filePath]).toMatchSnapshot();

expect(response.value.analysisResults.timing.analysis).toBeGreaterThanOrEqual(
response.value.analysisResults.timing.fetchingCode,
Expand Down
58 changes: 58 additions & 0 deletions tests/cache.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import path from 'path';
import write from 'write';
import fs from 'fs';
import { Cache, readJSON } from '../src/cache';
describe('Cache', () => {
afterAll(() => {
const dir = path.resolve(__dirname, '../fixtures');
fs.rmdir(dir, { recursive: true }, err => {
if (err) {
throw err;
}
});
});
it('should not crash if the cache file exists but it is an empty string', function () {
const cachePath = path.resolve(__dirname, '../fixtures/.cache2');
write.sync(path.join(cachePath, 'someId'), '');

expect(function () {
const cache = new Cache('someId', cachePath);
expect(cache.persisted).toEqual({});
}).not.toThrow(Error);
});

it('should not crash if the cache file exists but it is an invalid JSON string', function () {
const cachePath = path.resolve(__dirname, '../fixtures/.cache2');
write.sync(path.join(cachePath, 'someId'), '{ "foo": "fookey", "bar" ');

expect(function () {
const cache = new Cache('someId', cachePath);
expect(cache.persisted).toEqual({});
}).not.toThrow(Error);
});

describe('loading an existing cache custom directory', function () {
beforeEach(function () {
const cache = new Cache('someId', path.resolve(__dirname, '../fixtures/.cache2'));
cache.setKey('foo', {
bar: 'baz',
});
cache.setKey('bar', {
foo: 'baz',
});
cache.save();
});

it('should load an existing cache', function () {
const cache = new Cache('someId', path.resolve(__dirname, '../fixtures/.cache2'));
expect(readJSON(path.resolve(__dirname, '../fixtures/.cache2/someId'))).toEqual(cache.persisted);
});

it('should return the same structure if load called twice with the same docId', function () {
const cache = new Cache('someId', path.resolve(__dirname, '../fixtures/.cache2'));
const cache2 = new Cache('someId', path.resolve(__dirname, '../fixtures/.cache2'));

expect(cache.persisted).toEqual(cache2.persisted);
});
});
});

0 comments on commit dd86ef8

Please sign in to comment.