diff --git a/src/parser.js b/src/parser.js index 0cf9c76..e3e2bd2 100644 --- a/src/parser.js +++ b/src/parser.js @@ -6,12 +6,13 @@ class RiScriptParser extends CstParser { constructor(allTokens) { super(allTokens, { nodeLocationTracking: "full" }); this.atomTypes = ['silent', 'assign', 'symbol', 'choice', 'pgate', 'text', 'entity']; + this.rawTypes = ['Raw', 'STAT']; this.buildRules(); } parse(opts) { this.input = opts.tokens; // superclass member (do not change) - + let cst = this.script(); if (this.errors.length > 0) throw Error ("[PARSING]\n" + this.errors[0].message); @@ -113,7 +114,13 @@ class RiScriptParser extends CstParser { }); $.RULE("text", () => { - $.CONSUME(Tokens.Raw); + // this.rawTypes.map(t => console.log(t, Tokens[t])); + $.OR(this.rawTypes.map(t => ({ ALT: () => $.CONSUME(Tokens[t]) }))); + + // $.OR([ + // { ALT: () => $.CONSUME(Tokens.STAT) }, + // { ALT: () => $.CONSUME(Tokens.Raw) }, + // ]) }); this.performSelfAnalysis(); // keep diff --git a/src/riscript.js b/src/riscript.js index d531dfb..276015a 100644 --- a/src/riscript.js +++ b/src/riscript.js @@ -76,23 +76,28 @@ class RiScript { this.visitor = 0; // created in evaluate() or passed in here this.v2Compatible = opts.compatibility === 2; const { Constants, tokens } = getTokens(this.v2Compatible); - this.Escaped = Constants.Escaped; - this.Symbols = Constants.Symbols; + const { Escaped, Symbols } = Constants; - const anysym = Constants.Escaped.STATIC + Constants.Escaped.DYNAMIC; - const open = Constants.Escaped.OPEN_CHOICE; - const close = Constants.Escaped.CLOSE_CHOICE; + this.Escaped = Escaped; + this.Symbols = Symbols; + + const open = Escaped.OPEN_CHOICE; + const close = Escaped.CLOSE_CHOICE; + const anysym = Escaped.STATIC + Escaped.DYNAMIC; this.JSOLIdentRE = new RegExp(`([${anysym}]?[A-Za-z_0-9][A-Za-z_0-9]*)\\s*:`, 'g'); this.RawAssignRE = new RegExp(`^[${anysym}][A-Za-z_0-9][A-Za-z_0-9]*\\s*=`); this.ChoiceWrapRE = new RegExp('^' + open + '[^' + open + close + ']*' + close + '$'); this.EntityRE = tokens.modes.normal.filter(t => t.name === 'Entity')[0].PATTERN; - this.SpecialRE = new RegExp(`[${this.Escaped.SPECIAL.replace('&', '')}]`); - this.ContinueRE = new RegExp(this.Escaped.CONTINUATION + '\\r?\\n', 'g'); + this.SpecialRE = new RegExp(`[${Escaped.SPECIAL.replace('&', '')}]`); + this.ContinueRE = new RegExp(Escaped.CONTINUATION + '\\r?\\n', 'g'); this.WhitespaceRE = /[\u00a0\u2000-\u200b\u2028-\u2029\u3000]+/g; + this.StaticSymbol = new RegExp(Escaped.STATIC + '[A-Za-z_0-9][A-Za-z_0-9]*'); + this.ValidSymbolRE = new RegExp('(' + Escaped.DYNAMIC + '|' + Escaped.STATIC + '[A-Za-z_0-9])[A-Za-z_0-9]*'); this.AnySymbolRE = new RegExp(`[${anysym}]`); // added + this.silent = false; this.lexer = new Lexer(tokens); this.parser = new RiScriptParser(tokens); @@ -175,7 +180,7 @@ class RiScript { // check for unresolved symbols ([$#]) after removing HTML entities if (!this.silent && !this.RiTa.SILENT) { - if (this.AnySymbolRE.test(expr.replace(HtmlEntities, ''))) { + if (this.ValidSymbolRE.test(expr.replace(HtmlEntities, ''))) { console.warn('[WARN] Unresolved symbol(s) in "' + expr.replace(/\n/g, '\\n') + '" '); } } diff --git a/src/tokens.js b/src/tokens.js index 582180a..ec11350 100644 --- a/src/tokens.js +++ b/src/tokens.js @@ -39,7 +39,7 @@ function getTokens(v2Compatible) { const ENTITY_PATTERN = /&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-fA-F]{1,6});/i; const PENDING_GATE_PATTERN = new RegExp(`${Escaped.PENDING_GATE}([0-9]{9,11})`) - Escaped.SPECIAL = Object.values(Escaped).join('').replace(/[<>]/g, ''); // allow <>& for html + Escaped.SPECIAL = Object.values(Escaped).join('').replace(/[<>]/g, ''); // allow <> for html Symbols.PENDING_GATE_RE = new RegExp(PENDING_GATE_PATTERN.source, 'g'); // for unresolved gates const ExitGate = createToken({ @@ -64,7 +64,8 @@ function getTokens(v2Compatible) { push_mode: "gate_mode" }); - + const DYN = createToken({ name: "DYN", pattern: new RegExp(Escaped.DYNAMIC) }); + const STAT = createToken({ name: "STAT", pattern: new RegExp(Escaped.STATIC) }); const OC = createToken({ name: "OC", pattern: new RegExp(Escaped.OPEN_CHOICE + '\\s*') }); const CC = createToken({ name: "CC", pattern: new RegExp(`\\s*${Escaped.CLOSE_CHOICE}`) }); const OR = createToken({ name: "OR", pattern: /\s*\|\s*/ }); @@ -73,13 +74,13 @@ function getTokens(v2Compatible) { const TF = createToken({ name: "TF", pattern: /\.[A-Za-z_0-9][A-Za-z_0-9]*(\(\))?/ }); const OS = createToken({ name: "OS", pattern: new RegExp(`${Escaped.OPEN_SILENT}\\s*`) }); const CS = createToken({ name: "CS", pattern: new RegExp(`\\s*${Escaped.CLOSE_SILENT}`) }); - const SYM = createToken({ name: "SYM", pattern: new RegExp(`[${Escaped.DYNAMIC}${Escaped.STATIC}][A-Za-z_0-9]*`) }); + const SYM = createToken({ name: "SYM", pattern: new RegExp(`(${Escaped.DYNAMIC}|${Escaped.STATIC}[A-Za-z_0-9])[A-Za-z_0-9]*`) }); const Entity = createToken({ name: "Entity", pattern: ENTITY_PATTERN }); const Weight = createToken({ name: "Weight", pattern: new RegExp(`\\s*${Escaped.OPEN_WEIGHT}.+${Escaped.CLOSE_WEIGHT}\\s*`) }); const Raw = createToken({ name: "Raw", pattern: new RegExp(`[^${Escaped.SPECIAL}]+`) }); - const normalMode = [Entity, Weight, ELSE, OC, CC, OR, EQ, SYM, TF, OS, CS, PendingGate, Raw, EnterGate]; + const normalMode = [Entity, Weight, ELSE, OC, CC, OR, EQ, SYM, DYN, STAT, TF, OS, CS, PendingGate, Raw, EnterGate]; const gateMode = [Gate, ExitGate]; const multiMode = { diff --git a/src/visitor.js b/src/visitor.js index 8fd7da1..fbdb6eb 100644 --- a/src/visitor.js +++ b/src/visitor.js @@ -244,9 +244,10 @@ class RiScriptVisitor extends BaseVisitor { } text(ctx) { - if (ctx.Raw.length !== 1) throw Error('[1] invalid text'); + // if (ctx.Raw.length !== 1 && ctx.STAT.length !== 1 ) throw Error('[1] invalid text'); if (Object.keys(ctx).length !== 1) throw Error('[2] invalid text'); - const image = ctx.Raw[0].image; + const tok = ctx?.Raw || ctx?.STAT; + const image = tok[0].image; this.print('text', this.RiScript._escapeText("'" + image + "'")); return image; } @@ -272,7 +273,7 @@ class RiScriptVisitor extends BaseVisitor { // lookup: result is either a value, a function, or undef let { result, isStatic, isUser, resolved } = this.checkContext(ident); - if (!isStatic && symbol.startsWith(this.symbols.STATIC)) { + if (!isStatic && this.scripting.StaticSymbol.test(symbol)) { if (!this.scripting.EntityRE.test(symbol)) { throw Error(`Attempt to refer to dynamic symbol '${ident}' as` + ` ${this.symbols.STATIC}${ident}, did you mean $${ident}?`); diff --git a/test/riscript.tests.js b/test/riscript.tests.js index 84d14a0..d0256cd 100644 --- a/test/riscript.tests.js +++ b/test/riscript.tests.js @@ -35,6 +35,33 @@ describe('RiScript.v3', function () { LTR && describe('OneOff', function () { it('Should be a single problematic test', function () { }); }); + describe('Markdown', function () { + it('Should handle markdown headers ', function () { + const res = riscript.evaluate('### Header'); + expect(res).eq('### Header'); + }); + + it('Should handle markdown @italics', function () { + // TODO: if @ is not followed by {, escaped it automatically? + const res = riscript.evaluate(`Some [RiScript](/\\@dhowe/riscript) *code*`); + expect(res).eq('Some RiScript(/@dhowe/riscript) *code*'); + }); + + it('Should handle markdown lines', function () { + let input = `### A Title + Some [RiScript](/\\@dhowe/riscript) code + that we can [format|format|format] + with *[inline | inline]* Markdown + and rerun [once per | once per] second + [using|using|using] the **[pulse].qq** function below`; + let expected = '### A Title \n Some RiScript(/@dhowe/riscript) code\n that we can format\n with *inline* Markdown\n and rerun once per second\n using the **“pulse”** function below'; + // TODO: if @ is not followed by {, escaped it automatically? + const res = riscript.evaluate(input); + //console.log(res); + expect(res).eq(expected); + }); + }); + describe('Sequences', function () { it('Should support norepeat choice transforms', function () {