From a5097dd4613413f2460000b7c983f4a06f1e9ad5 Mon Sep 17 00:00:00 2001 From: deleterium Date: Thu, 7 Mar 2024 22:21:15 -0300 Subject: [PATCH] feat: adding collumn info during define replacements --- .../__tests__/preprocessor.spec.ts | 26 +-- src/preprocessor/preprocessor.ts | 157 ++++++++++++------ src/tokenizer/tokenizerV3.ts | 16 +- 3 files changed, 128 insertions(+), 71 deletions(-) diff --git a/src/preprocessor/__tests__/preprocessor.spec.ts b/src/preprocessor/__tests__/preprocessor.spec.ts index 98c65dd..ee12d6d 100644 --- a/src/preprocessor/__tests__/preprocessor.spec.ts +++ b/src/preprocessor/__tests__/preprocessor.spec.ts @@ -33,68 +33,68 @@ function createFakeProgram (sourceCode: string) { describe('preprocessor right tests', () => { it('#define test', () => { const code = createFakeProgram('#define MAX 4\nlong a; a=MAX; long MAXimus=2;') - const result = '\nlong a; a=4; long MAXimus=2;' + const result = '\nlong a; a=4#13#; long MAXimus=2;' // @ts-expect-error TS2345 expect(preprocessor(code)).toBe(result) }) it('#define test', () => { const code = createFakeProgram('#define MAX 4\n long a; a=MAX;\n #define MAX 2\n long MAXimus=MAX;') - const result = '\n long a; a=4;\n\n long MAXimus=2;' + const result = '\n long a; a=4#14#;\n\n long MAXimus=2#17#;' // @ts-expect-error TS2345 expect(preprocessor(code)).toBe(result) }) it('#define test', () => { const code = createFakeProgram('#define MAX 4\n long a; a=MAX;\n #define MAX \n long MAXimus=MAX;') - const result = '\n long a; a=4;\n\n long MAXimus=;' + const result = '\n long a; a=4#14#;\n\n long MAXimus=#17#;' // @ts-expect-error TS2345 expect(preprocessor(code)).toBe(result) }) it('#define test', () => { const code = createFakeProgram('#define 444 4\nlong a; a=444;\n #undef 444\nlong MAXimus=444;') - const result = '\nlong a; a=4;\n\nlong MAXimus=444;' + const result = '\nlong a; a=4#13#;\n\nlong MAXimus=444;' // @ts-expect-error TS2345 expect(preprocessor(code)).toBe(result) }) it('#define test', () => { const code = createFakeProgram('#define MAX 4\n#define MAX1 (MAX + 1)\n long a; if (a > MAX1) a++;') - const result = '\n\n long a; if (a > (4 + 1)) a++;' + const result = '\n\n long a; if (a > (4#4# + 1)#21#) a++;' // @ts-expect-error TS2345 expect(preprocessor(code)).toBe(result) }) it('#define test', () => { const code = createFakeProgram('#define MAX 4\n#define MAX1 (MAX + 1)\n#undef MAX\n long a; if (a > MAX1) a++;') - const result = '\n\n\n long a; if (a > (4 + 1)) a++;' + const result = '\n\n\n long a; if (a > (4#4# + 1)#21#) a++;' // @ts-expect-error TS2345 expect(preprocessor(code)).toBe(result) }) it('#define macro (simple)', () => { const code = createFakeProgram('#define DEF(top, bottom) (((top) << 8) | (bottom))\nlong a,b,c;\na = DEF(b, c);\n') - const result = '\nlong a,b,c;\na = (((b) << 8) | (c));\n' + const result = '\nlong a,b,c;\na = (((b) << 8) | (c))#13#;\n' // @ts-expect-error TS2345 expect(preprocessor(code)).toBe(result) }) it('#define macro (empty argument)', () => { const code = createFakeProgram('#define DEF() (Get_Current_Timestamp( ) >> 32)\nlong a;\na = DEF() ;\n') - const result = '\nlong a;\na = (Get_Current_Timestamp( ) >> 32) ;\n' + const result = '\nlong a;\na = (Get_Current_Timestamp( ) >> 32)#9# ;\n' // @ts-expect-error TS2345 expect(preprocessor(code)).toBe(result) }) it('#define macro (complex)', () => { const code = createFakeProgram('#define DEF(top, bottom) (((top) << 8) | (bottom))\nlong a,b,c;\na = DEF(mdv(a,b, c), c) + DEF(22, 25);\n') - const result = '\nlong a,b,c;\na = (((mdv(a,b, c)) << 8) | (c)) + (((22) << 8) | (25));\n' + const result = '\nlong a,b,c;\na = (((mdv(a,b, c)) << 8) | (c))#23# + (((22) << 8) | (25))#37#;\n' // @ts-expect-error TS2345 expect(preprocessor(code)).toBe(result) }) it('#define macro (with define constant)', () => { const code = createFakeProgram('#define ONE n1\n#define DEF(top, bottom) (((top) << 8) | (bottom + ONE))\n#undef ONE\nlong a,b,c;\na = DEF(mdv(a,b, c), c);\n') - const result = '\n\n\nlong a,b,c;\na = (((mdv(a,b, c)) << 8) | (c + n1));\n' + const result = '\n\n\nlong a,b,c;\na = (((mdv(a,b, c)) << 8) | (c + n1#29#))#23#;\n' // @ts-expect-error TS2345 expect(preprocessor(code)).toBe(result) }) it('#define replace multiple times same line', () => { const code = createFakeProgram('long a, b, c; if (a == NULL && b == NULL) c++;') - const result = 'long a, b, c; if (a == (void *)(0) && b == (void *)(0)) c++;' + const result = 'long a, b, c; if (a == (void *)(0)#27# && b == (void *)(0)#40#) c++;' // @ts-expect-error TS2345 expect(preprocessor(code)).toBe(result) }) @@ -166,13 +166,13 @@ describe('preprocessor right tests', () => { }) it('#define, #undef at line not active test', () => { const code = createFakeProgram('#ifdef debug\n#define A1 44\n#ifndef impossible\na++;\n#endif\n#define A2\n#undef true\n#endif\nlong a; a=A1+A2+true;') - const result = '\n\n\n\n\n\n\n\nlong a; a=A1+A2+1;' + const result = '\n\n\n\n\n\n\n\nlong a; a=A1+A2+1#20#;' // @ts-expect-error TS2345 expect(preprocessor(code)).toBe(result) }) it('#define, #undef at line active test', () => { const code = createFakeProgram('#define debug \n#ifdef debug\n#define A1 44\n#define A2\n#undef true\n#endif\nlong a; a=A1+A2+true;') - const result = '\n\n\n\n\n\nlong a; a=44++true;' + const result = '\n\n\n\n\n\nlong a; a=44#12#+#15#+true;' // @ts-expect-error TS2345 expect(preprocessor(code)).toBe(result) }) diff --git a/src/preprocessor/preprocessor.ts b/src/preprocessor/preprocessor.ts index fa8fa48..8d3805a 100644 --- a/src/preprocessor/preprocessor.ts +++ b/src/preprocessor/preprocessor.ts @@ -1,4 +1,4 @@ -import { BitField, StringStream, parseDecimalNumber } from '../repository/repository' +import { BitField, STREAM_PAIR, StringStream, parseDecimalNumber } from '../repository/repository' import { CONTRACT } from '../typings/contractTypes' type IF_INFO = { @@ -60,7 +60,7 @@ export default function preprocessor (Program: CONTRACT) : string { return retLines.join('\n') } - /** Moves escaped lines (ended with \ ) to the first line */ + /** Moves escaped lines (ended with \ ) to the first line, analizes and return the preprocessor token line */ function prepare () : PREP_LINE[] { const retTokens : PREP_LINE[] = [] let afterEscapedLine = false @@ -112,6 +112,7 @@ export default function preprocessor (Program: CONTRACT) : string { return retTokens } + /** Parse the line finding a directive and extracting the fields */ function parseDirective (codeLine: string, line: number) : string[] { const Stream = new StringStream(codeLine) const firstField = findFirstField() @@ -193,79 +194,127 @@ export default function preprocessor (Program: CONTRACT) : string { } } + /** OK... We parse the line backwards to find the string to be replaced. + * This way we can maintain the right collumn index for the remaining string + * in the most of cases. Collumns still wrong if there was a escaped line or + * a replacement inside other replacement. + */ function replaceDefines (codeline: string, line: string) : string { let retLine = codeline - preprocessorReplacements.forEach((Replacement) => { - if (Replacement.macro) { - while (true) { - const foundCname = Replacement.regex.exec(retLine) - if (foundCname === null) { - return + let current : STREAM_PAIR + let state = 0 + let wordEndIndex = 0 + let word : string + let Replacement: REPLACEMENTS | undefined + while (true) { + let wasReplaced = false + const Stream = new StringStream(retLine) + Stream.setBack() + while (true) { + current = Stream.back() + switch (state) { + case 0: + if (BitField.typeTable[current.code] & BitField.isWord || BitField.typeTable[current.code] & BitField.isDigit) { + wordEndIndex = Stream.col + state = 1 + } + if (current.char) { + continue } - let replaced = Replacement.macro - const currExtArgs = extractArgs(retLine, foundCname.index + 1, line) - const origExtArgs = extractArgs(Replacement.value, 0, line) - if (origExtArgs.argArray.length !== currExtArgs.argArray.length) { - throw new Error(Program.Context.formatError(line, - `Wrong number of arguments for macro '${Replacement.cname}'. ` + - `Expected ${origExtArgs.argArray.length}, got ${currExtArgs.argArray.length}.`)) + break + case 1: + if (BitField.typeTable[current.code] & BitField.isWord || BitField.typeTable[current.code] & BitField.isDigit) { + continue } - for (let currArg = 0; currArg < origExtArgs.argArray.length; currArg++) { - replaced = replaced.replace(new RegExp(`\\b${origExtArgs.argArray[currArg]}\\b`, 'g'), currExtArgs.argArray[currArg]) + word = retLine.slice(Stream.col, wordEndIndex) + Replacement = preprocessorReplacements.find(item => item.cname === word) + if (Replacement) { + retLine = executeReplacement(Replacement, retLine, Stream.col, wordEndIndex, line) + wasReplaced = true } - retLine = retLine.slice(0, foundCname.index) + replaced + retLine.slice(currExtArgs.endPosition) + state = 0 + } + if (wasReplaced || current.char === undefined) { + // We need to reload the string OR end of parse + break } } - retLine = retLine.replace(Replacement.regex, Replacement.value) - }) + if (Stream.index < 0) { + break + } + } return retLine } + function executeReplacement (Replacement: REPLACEMENTS, code: string, startIndex: number, endIndex: number, line: string) { + if (Replacement.macro === undefined) { + return code.slice(0, startIndex) + Replacement.value + '#' + endIndex + '#' + code.slice(endIndex) + } + let replaced = Replacement.macro + const currExtArgs = extractArgs(code, startIndex + 1, line) + const origExtArgs = extractArgs(Replacement.value, 0, line) + if (origExtArgs.argArray.length !== currExtArgs.argArray.length) { + throw new Error(Program.Context.formatError(line, + `Wrong number of arguments for macro '${Replacement.cname}'. ` + + `Expected ${origExtArgs.argArray.length}, got ${currExtArgs.argArray.length}.`)) + } + for (let currArg = 0; currArg < origExtArgs.argArray.length; currArg++) { + replaced = replaced.replace(new RegExp(`\\b${origExtArgs.argArray[currArg]}\\b`, 'g'), currExtArgs.argArray[currArg]) + } + return code.slice(0, startIndex) + replaced + `#${currExtArgs.endPosition}#` + code.slice(currExtArgs.endPosition) + } + function extractArgs (fnArgString: string, needle: number, line: string): { argArray: string[], endPosition: number} { const argArray : string [] = [] let currArg: string = '' let pLevel = 0 - let started = false - for (;;needle++) { - const currChar = fnArgString.charAt(needle) - if (currChar === '') { + const Stream = new StringStream(fnArgString) + Stream.index = needle - 1 + let stage = 0 + while (true) { + const current = Stream.advance() + if (current.char === undefined) { throw new Error(Program.Context.formatError(line, 'Unmatched parenthesis or unexpected end of line.')) } - if (currChar === '(') { - pLevel++ - if (pLevel === 1) { - started = true - continue + switch (stage) { + case 0: + if (current.char === '(') { + pLevel = 1 + stage = 1 } - } - if (!started) continue - if (currChar === ')') { - pLevel-- - if (pLevel === 0) { - const endArg = currArg.trim() - if (endArg.length === 0 && argArray.length !== 0) { - throw new Error(Program.Context.formatError(line, 'Found empty argument on macro declaration.')) - } - if (endArg.length !== 0) { - argArray.push(currArg.trim()) - } + continue + case 1: + if (current.char === '(') { + pLevel++ break } - } - if (currChar === ',' && pLevel === 1) { - const newArg = currArg.trim() - if (newArg.length === 0) { - throw new Error(Program.Context.formatError(line, 'Found empty argument on macro declaration.')) + if (current.char === ')') { + pLevel-- + if (pLevel === 0) { + const endArg = currArg.trim() + if (endArg.length === 0 && argArray.length !== 0) { + throw new Error(Program.Context.formatError(line, 'Found empty argument on macro declaration.')) + } + if (endArg.length !== 0) { + argArray.push(currArg.trim()) + } + return { + argArray, + endPosition: Stream.index + 1 + } + } + } + if (current.char === ',' && pLevel === 1) { + const newArg = currArg.trim() + if (newArg.length === 0) { + throw new Error(Program.Context.formatError(line, 'Found empty argument on macro declaration.')) + } + argArray.push(currArg.trim()) + currArg = '' + continue } - argArray.push(currArg.trim()) - currArg = '' - continue } - currArg += currChar - } - return { - argArray, - endPosition: needle + 1 + currArg += current.char } } diff --git a/src/tokenizer/tokenizerV3.ts b/src/tokenizer/tokenizerV3.ts index d48875f..fdf7f53 100644 --- a/src/tokenizer/tokenizerV3.ts +++ b/src/tokenizer/tokenizerV3.ts @@ -251,15 +251,23 @@ export default function tokenizer (Program: CONTRACT, inputSourceCode: string): } } - function stateReadMacro () : PRE_TOKEN { + function stateReadMacro () : undefined { const line = Stream.lineCol const tokenStartIndex = Stream.index let current: STREAM_PAIR - do { + while (true) { current = Stream.advance() - } while (current.char && current.code !== '\n'.charCodeAt(0)) + if (BitField.typeTable[current.code] & BitField.isDigit) { + continue + } + if (current.char === '#') { + break + } + throw new Error(Program.Context.formatError(line, 'Wrong use of preprocessor directive')) + } const tokenValue = inputSourceCode.slice(tokenStartIndex + 1, Stream.index) - return { type: 'Macro', precedence: 0, value: tokenValue, line } + Stream.col = parseInt(tokenValue) + return undefined } function stateReadString () : PRE_TOKEN {