feat: integrate expression evaluation

package.json

@@ -21,7 +21,6 @@
     "networks.json"
   ],
   "dependencies": {
-    "@subsquid/manifest-expr": "^0.0.1",
     "joi": "17.13.3",
     "js-yaml": "^4.1.0",
     "lodash": "^4.17.21"

src/expression.ts

@@ -0,0 +1,197 @@
+import assert from 'assert';
+
+export const EXPR_PATTERN = /(\${{[^}]*}})/;
+
+export class Parser {
+  constructor() {}
+
+  parse(str: string): Expression {
+    const tokens: (string | Token)[] = [];
+
+    let pos = 0;
+    str.split(EXPR_PATTERN).map(i => {
+      if (EXPR_PATTERN.test(i)) {
+        tokens.push(new Tokenizer(i.slice(3, i.length - 2), pos + 3).tokenize());
+      } else {
+        tokens.push(i);
+      }
+      pos += i.length;
+    });
+
+    return new Expression(tokens);
+  }
+}
+
+export class Tokenizer {
+  private pos = 0;
+
+  constructor(
+    private str: string,
+    private offset = 0,
+  ) {}
+
+  tokenize(): Token {
+    let token: Token | undefined;
+
+    while (this.str[this.pos]) {
+      while (this.str[this.pos] === ' ') {
+        this.pos++;
+      }
+
+      switch (this.str[this.pos]) {
+        case undefined:
+          break;
+        case '.':
+          if (token) {
+            this.pos++;
+            token = new Token(TokenType.MemberAccess, [token, this.tokenize()]);
+          } else {
+            throw this.error("Unexpected '.'");
+          }
+          break;
+        default:
+          const value = this.id();
+          if (value) {
+            token = new Token(TokenType.Identifier, [value]);
+          } else {
+            throw this.error(`Unexpected '${this.str[this.pos]}'`);
+          }
+          break;
+      }
+    }
+
+    if (!token) {
+      throw this.error(`Unexpected EOF`);
+    }
+
+    return token;
+  }
+
+  id() {
+    const start = this.pos;
+    while (
+      this.str[this.pos] &&
+      (/[a-zA-Z_$]/.test(this.str[this.pos]) ||
+        (this.pos > start && /[0-9]/.test(this.str[this.pos])))
+    ) {
+      this.pos++;
+    }
+
+    return this.pos > start ? this.str.slice(start, this.pos) : undefined;
+  }
+
+  error(msg: string) {
+    return new ParsingError(msg, [0, this.pos + this.offset]);
+  }
+}
+
+export class ParsingError extends Error {
+  constructor(message: string, pos: [number, number]) {
+    super(message + ` [${pos}]`);
+  }
+}
+
+export enum TokenType {
+  Identifier,
+  MemberAccess,
+}
+
+export class Token {
+  constructor(
+    private type: TokenType,
+    private nodes: (string | Token)[],
+  ) {}
+
+  eval(ctx: any, ctxPath: string[]): { value: unknown; path: string[] } {
+    switch (this.type) {
+      case TokenType.MemberAccess: {
+        const [n0, n1] = this.nodes;
+        assert(n0 instanceof Token);
+        assert(n1 instanceof Token);
+
+        const { value, path } = n0.eval(ctx, ctxPath);
+
+        return n1.eval(value, path);
+      }
+      case TokenType.Identifier: {
+        const [n0] = this.nodes;
+        assert(typeof n0 === 'string');
+
+        const path = [...ctxPath, n0];
+        const value = !!ctx?.hasOwnProperty(n0) ? ctx[n0] : undefined;
+
+        if (value) {
+          return { value, path };
+        } else {
+          throw new EvaluationError(`"${path.join('.')}" is not defined`);
+        }
+      }
+    }
+  }
+
+  variables(path: string[] = []): string[] {
+    const res: Set<string> = new Set();
+
+    switch (this.type) {
+      case TokenType.MemberAccess: {
+        const [n0, n1] = this.nodes;
+        assert(n0 instanceof Token);
+        assert(n1 instanceof Token);
+
+        const obj = n0.variables()[0];
+        if (obj && path.length === 0) {
+          res.add(obj);
+        } else if (obj === path[0]) {
+          n1.variables(path.slice(1)).forEach(v => res.add(v));
+        }
+
+        break;
+      }
+      case TokenType.Identifier: {
+        const [n0] = this.nodes;
+        assert(typeof n0 === 'string');
+
+        if (path.length === 0) {
+          res.add(n0);
+        }
+
+        break;
+      }
+    }
+
+    return [...res];
+  }
+}
+
+export class EvaluationError extends Error {}
+
+export class Expression {
+  constructor(readonly tokens: (string | Token)[]) {}
+
+  eval(context: Record<string, any> = {}): string {
+    const res: (string | undefined)[] = [];
+
+    for (const token of this.tokens) {
+      if (typeof token === 'string') {
+        res.push(token);
+      } else {
+        res.push(token.eval(context, []).value?.toString() ?? '');
+      }
+    }
+
+    return res.join('');
+  }
+
+  variables(path: string[] = []) {
+    const res: Set<string> = new Set();
+
+    for (const token of this.tokens) {
+      if (typeof token === 'string') {
+      } else {
+        token.variables(path).forEach(v => res.add(v));
+      }
+    }
+
+    return [...res];
+  }
+}

src/manifest.ts

@@ -1,8 +1,8 @@
-import { Expression, Parser } from '@subsquid/manifest-expr';
 import yaml from 'js-yaml';
 import { cloneDeep, defaultsDeep, get, mapValues, set } from 'lodash';
 
 import { ManifestEvaluatingError, ManifestParsingError } from './errors';
+import { Expression, Parser } from './expression';
 import { manifestSchema } from './schema';
 import { DeepPartial, ManifestDeploymentConfig, ManifestProcessor, ManifestValue } from './types';
 
@@ -334,4 +334,4 @@ function getError(path: string, expression: string | undefined, error: any) {
     `Manifest env variable "${path}" can not be mapped${exprIn}`,
     error instanceof Error ? error.message : error.toString(),
   ].join(': ');
-}
+}

test/eval.test.ts

@@ -2,7 +2,7 @@ import { Manifest, ManifestEvaluatingError } from '../src';
 
 describe('Env Evaluation', () => {
   describe('eval', () => {
-    it('should not add an objects if it doesnt exist in the source manifest', () => {
+    it('should not add an objects if it does not exist in the source manifest', () => {
       const res = new Manifest({
         manifest_version: 'subsquid.io/v0.1',
         name: 'test',
@@ -158,7 +158,7 @@ describe('Env Evaluation', () => {
         }),
       ).toThrow(
         new ManifestEvaluatingError([
-          'Manifest env variable "deploy.env.baz" can not be mapped for "${{baz}}" expression: "baz" is not found in the context',
+          'Manifest env variable "deploy.env.baz" can not be mapped for "${{baz}}" expression: "baz" is not defined',
         ]),
       );
     });

test/expression.test.ts

@@ -0,0 +1,90 @@
+import { ParsingError, Parser, EvaluationError } from '../src/expression';
+
+describe('Expression', () => {
+  const parser = new Parser();
+
+  it('string', () => {
+    const value = parser.parse('hello').eval();
+    expect(value).toEqual('hello');
+  });
+
+  it('should resolve identifier', () => {
+    const value = parser.parse('${{foo}}').eval({ foo: 'foo' });
+    expect(value).toEqual('foo');
+  });
+
+  it('should throw on not defined identifier', () => {
+    expect(() => parser.parse('${{foo}}').eval({})).toThrow(
+      new EvaluationError('"foo" is not defined'),
+    );
+  });
+
+  it('should resolve member access', () => {
+    const value = parser.parse('${{foo.bar}}').eval({ foo: { bar: 'bar' } });
+    expect(value).toEqual('bar');
+  });
+
+  it('should resolve chained member access', () => {
+    const value = parser.parse('${{foo.bar.baz}}').eval({ foo: { bar: { baz: 'baz' } } });
+    expect(value).toEqual('baz');
+  });
+
+  it('should resolve member access with spaces', () => {
+    const value = parser.parse('${{foo  .    bar}}').eval({ foo: { bar: 'bar' } });
+    expect(value).toEqual('bar');
+  });
+
+  it('should throw on member access with not defined child', () => {
+    expect(() => parser.parse('${{foo.bar.baz}}').eval({ foo: { bar: {} } })).toThrow(
+      new EvaluationError('"foo.bar.baz" is not defined'),
+    );
+  });
+
+  it('should throw on member access with not defined parent', () => {
+    expect(() => parser.parse('${{foo.bar.baz}}').eval({ foo: {} })).toThrow(
+      new EvaluationError('"foo.bar" is not defined'),
+    );
+  });
+
+  it('should throw on unexpected char', () => {
+    expect(() => parser.parse('${{foo @ bar}}')).toThrow(
+      new ParsingError(`Unexpected '@'`, [0, 7]),
+    );
+  });
+
+  it('should throw on unexpected char 2', () => {
+    expect(() => parser.parse('${{foo.9bar}}')).toThrow(new ParsingError(`Unexpected '9'`, [0, 7]));
+  });
+
+  it('should throw on unexpected char', () => {
+    expect(() => parser.parse('${{.foo}}')).toThrow(new ParsingError(`Unexpected '.'`, [0, 3]));
+  });
+
+  it('should throw on unexpected EOF', () => {
+    expect(() => parser.parse('${{foo.}}')).toThrow(new ParsingError(`Unexpected EOF`, [0, 7]));
+  });
+
+  it('should throw on empty expression', () => {
+    expect(() => parser.parse('${{    }}')).toThrow(new ParsingError(`Unexpected EOF`, [0, 7]));
+  });
+
+  it('should resolve wrapped expression', () => {
+    const value = parser.parse('hello ${{foo}} world').eval({ foo: 'foo' });
+    expect(value).toEqual('hello foo world');
+  });
+
+  it('should resolve double expression', () => {
+    const value = parser.parse('${{foo}} ${{bar}}').eval({ foo: 'foo', bar: 'bar' });
+    expect(value).toEqual('foo bar');
+  });
+
+  it('should resolve variables', () => {
+    const variables = parser.parse('${{foo.bar}}').variables();
+    expect(variables).toEqual(['foo']);
+  });
+
+  it('should resolve variables with path', () => {
+    const variables = parser.parse('${{foo.bar}}').variables(['foo']);
+    expect(variables).toEqual(['bar']);
+  });
+});

yarn.lock

@@ -682,11 +682,6 @@
   dependencies:
     "@sinonjs/commons" "^3.0.0"
 
-"@subsquid/manifest-expr@^0.0.1":
-  version "0.0.1"
-  resolved "https://registry.yarnpkg.com/@subsquid/manifest-expr/-/manifest-expr-0.0.1.tgz#da8078dcdc8e765b6b82cca1a0db9dbf5bff0297"
-  integrity sha512-QKrgq+BPVX03SPm/kxXr3kDRDFn+ddBAuA9ysvae3PBA0jlj5DUSFxNjIoGZoy+RBwu82t4T0Xt4oTXR8e0OBw==
-
 "@tsconfig/node10@^1.0.7":
   version "1.0.9"
   resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.9.tgz#df4907fc07a886922637b15e02d4cebc4c0021b2"