fixes for markdown issues

src/parser.js

@@ -6,12 +6,13 @@ class RiScriptParser extends CstParser {
   constructor(allTokens) {
     super(allTokens, { nodeLocationTracking: "full" });
     this.atomTypes = ['silent', 'assign', 'symbol', 'choice', 'pgate', 'text', 'entity'];
+    this.rawTypes = ['Raw', 'STAT'];
     this.buildRules();
   }
 
   parse(opts) {
     this.input = opts.tokens; // superclass member (do not change)
-    
+
     let cst = this.script();
     if (this.errors.length > 0) throw Error
       ("[PARSING]\n" + this.errors[0].message);
@@ -113,7 +114,13 @@ class RiScriptParser extends CstParser {
     });
 
     $.RULE("text", () => {
-      $.CONSUME(Tokens.Raw);
+      // this.rawTypes.map(t => console.log(t, Tokens[t]));
+      $.OR(this.rawTypes.map(t => ({ ALT: () => $.CONSUME(Tokens[t]) })));
+
+      // $.OR([
+      //   { ALT: () => $.CONSUME(Tokens.STAT) },
+      //   { ALT: () => $.CONSUME(Tokens.Raw) },
+      // ])
     });
 
     this.performSelfAnalysis(); // keep

src/riscript.js

@@ -76,23 +76,28 @@ class RiScript {
     this.visitor = 0; // created in evaluate() or passed in here
     this.v2Compatible = opts.compatibility === 2;
     const { Constants, tokens } = getTokens(this.v2Compatible);
-    this.Escaped = Constants.Escaped;
-    this.Symbols = Constants.Symbols;
+    const { Escaped, Symbols } = Constants;
 
-    const anysym = Constants.Escaped.STATIC + Constants.Escaped.DYNAMIC;
-    const open = Constants.Escaped.OPEN_CHOICE;
-    const close = Constants.Escaped.CLOSE_CHOICE;
+    this.Escaped = Escaped;
+    this.Symbols = Symbols;
+
+    const open = Escaped.OPEN_CHOICE;
+    const close = Escaped.CLOSE_CHOICE;
+    const anysym = Escaped.STATIC + Escaped.DYNAMIC;
 
     this.JSOLIdentRE = new RegExp(`([${anysym}]?[A-Za-z_0-9][A-Za-z_0-9]*)\\s*:`, 'g');
     this.RawAssignRE = new RegExp(`^[${anysym}][A-Za-z_0-9][A-Za-z_0-9]*\\s*=`);
     this.ChoiceWrapRE = new RegExp('^' + open + '[^' + open + close + ']*' + close + '$');
 
     this.EntityRE = tokens.modes.normal.filter(t => t.name === 'Entity')[0].PATTERN;
-    this.SpecialRE = new RegExp(`[${this.Escaped.SPECIAL.replace('&', '')}]`);
-    this.ContinueRE = new RegExp(this.Escaped.CONTINUATION + '\\r?\\n', 'g');
+    this.SpecialRE = new RegExp(`[${Escaped.SPECIAL.replace('&', '')}]`);
+    this.ContinueRE = new RegExp(Escaped.CONTINUATION + '\\r?\\n', 'g');
     this.WhitespaceRE = /[\u00a0\u2000-\u200b\u2028-\u2029\u3000]+/g;
+    this.StaticSymbol = new RegExp(Escaped.STATIC + '[A-Za-z_0-9][A-Za-z_0-9]*');
+    this.ValidSymbolRE = new RegExp('(' + Escaped.DYNAMIC + '|' + Escaped.STATIC + '[A-Za-z_0-9])[A-Za-z_0-9]*');
     this.AnySymbolRE = new RegExp(`[${anysym}]`); // added
 
+
     this.silent = false;
     this.lexer = new Lexer(tokens);
     this.parser = new RiScriptParser(tokens);
@@ -175,7 +180,7 @@ class RiScript {
 
     // check for unresolved symbols ([$#]) after removing HTML entities
     if (!this.silent && !this.RiTa.SILENT) {
-      if (this.AnySymbolRE.test(expr.replace(HtmlEntities, ''))) {
+      if (this.ValidSymbolRE.test(expr.replace(HtmlEntities, ''))) {
         console.warn('[WARN] Unresolved symbol(s) in "' + expr.replace(/\n/g, '\\n') + '" ');
       }
     }

src/tokens.js

@@ -39,7 +39,7 @@ function getTokens(v2Compatible) {
   const ENTITY_PATTERN = /&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-fA-F]{1,6});/i;
   const PENDING_GATE_PATTERN = new RegExp(`${Escaped.PENDING_GATE}([0-9]{9,11})`)
 
-  Escaped.SPECIAL = Object.values(Escaped).join('').replace(/[<>]/g, ''); // allow <>& for html 
+  Escaped.SPECIAL = Object.values(Escaped).join('').replace(/[<>]/g, ''); // allow <> for html 
   Symbols.PENDING_GATE_RE = new RegExp(PENDING_GATE_PATTERN.source, 'g'); // for unresolved gates
 
   const ExitGate = createToken({
@@ -64,7 +64,8 @@ function getTokens(v2Compatible) {
     push_mode: "gate_mode"
   });
 
-
+  const DYN = createToken({ name: "DYN", pattern: new RegExp(Escaped.DYNAMIC) });
+  const STAT = createToken({ name: "STAT", pattern: new RegExp(Escaped.STATIC) });
   const OC = createToken({ name: "OC", pattern: new RegExp(Escaped.OPEN_CHOICE + '\\s*') });
   const CC = createToken({ name: "CC", pattern: new RegExp(`\\s*${Escaped.CLOSE_CHOICE}`) });
   const OR = createToken({ name: "OR", pattern: /\s*\|\s*/ });
@@ -73,13 +74,13 @@ function getTokens(v2Compatible) {
   const TF = createToken({ name: "TF", pattern: /\.[A-Za-z_0-9][A-Za-z_0-9]*(\(\))?/ });
   const OS = createToken({ name: "OS", pattern: new RegExp(`${Escaped.OPEN_SILENT}\\s*`) });
   const CS = createToken({ name: "CS", pattern: new RegExp(`\\s*${Escaped.CLOSE_SILENT}`) });
-  const SYM = createToken({ name: "SYM", pattern: new RegExp(`[${Escaped.DYNAMIC}${Escaped.STATIC}][A-Za-z_0-9]*`) });
+  const SYM = createToken({ name: "SYM", pattern: new RegExp(`(${Escaped.DYNAMIC}|${Escaped.STATIC}[A-Za-z_0-9])[A-Za-z_0-9]*`) });
 
   const Entity = createToken({ name: "Entity", pattern: ENTITY_PATTERN });
   const Weight = createToken({ name: "Weight", pattern: new RegExp(`\\s*${Escaped.OPEN_WEIGHT}.+${Escaped.CLOSE_WEIGHT}\\s*`) });
   const Raw = createToken({ name: "Raw", pattern: new RegExp(`[^${Escaped.SPECIAL}]+`) });
 
-  const normalMode = [Entity, Weight, ELSE, OC, CC, OR, EQ, SYM, TF, OS, CS, PendingGate, Raw, EnterGate];
+  const normalMode = [Entity, Weight, ELSE, OC, CC, OR, EQ, SYM, DYN, STAT, TF, OS, CS, PendingGate, Raw, EnterGate];
   const gateMode = [Gate, ExitGate];
 
   const multiMode = {

src/visitor.js

@@ -244,9 +244,10 @@ class RiScriptVisitor extends BaseVisitor {
   }
 
   text(ctx) {
-    if (ctx.Raw.length !== 1) throw Error('[1] invalid text');
+    // if (ctx.Raw.length !== 1 && ctx.STAT.length !== 1 ) throw Error('[1] invalid text');
     if (Object.keys(ctx).length !== 1) throw Error('[2] invalid text');
-    const image = ctx.Raw[0].image;
+    const tok = ctx?.Raw || ctx?.STAT;
+    const image = tok[0].image;
     this.print('text', this.RiScript._escapeText("'" + image + "'"));
     return image;
   }
@@ -272,7 +273,7 @@ class RiScriptVisitor extends BaseVisitor {
     // lookup: result is either a value, a function, or undef
     let { result, isStatic, isUser, resolved } = this.checkContext(ident);
 
-    if (!isStatic && symbol.startsWith(this.symbols.STATIC)) {
+    if (!isStatic && this.scripting.StaticSymbol.test(symbol)) {
       if (!this.scripting.EntityRE.test(symbol)) {
         throw Error(`Attempt to refer to dynamic symbol '${ident}' as` +
           ` ${this.symbols.STATIC}${ident}, did you mean $${ident}?`);

test/riscript.tests.js

@@ -35,6 +35,33 @@ describe('RiScript.v3', function () {
   LTR && describe('OneOff', function () {
     it('Should be a single problematic test', function () { });
   });
+  describe('Markdown', function () {
+    it('Should handle markdown headers ', function () {
+      const res = riscript.evaluate('### Header');
+      expect(res).eq('### Header');
+    });
+
+    it('Should handle markdown @italics', function () {
+      // TODO: if @ is not followed by {, escaped it automatically?
+      const res = riscript.evaluate(`Some [RiScript](/\\@dhowe/riscript) *code*`);
+      expect(res).eq('Some RiScript(/@dhowe/riscript) *code*');
+    });
+
+    it('Should handle markdown lines', function () {
+      let input = `### A Title 
+      Some [RiScript](/\\@dhowe/riscript) code
+        that we can [format|format|format]
+           with *[inline | inline]* Markdown
+             and rerun [once per | once per] second
+               [using|using|using] the **[pulse].qq** function below`;
+      let expected = '### A Title \n      Some RiScript(/@dhowe/riscript) code\n        that we can format\n           with *inline* Markdown\n             and rerun once per second\n               using the **“pulse”** function below';
+      // TODO: if @ is not followed by {, escaped it automatically?
+      const res = riscript.evaluate(input);
+      //console.log(res);
+      expect(res).eq(expected);
+    });
+  });
+
 
   describe('Sequences', function () {
     it('Should support norepeat choice transforms', function () {