Fix transpilation of JS placeholders inside literal SQL strings. (#962)

* Fix transpilation of JS placeholders inside literal SQL strings.

* bump version

core/compilers.ts

@@ -253,33 +253,30 @@ function getFunctionPropertyNames(prototype: any) {
 
 function createEscapedStatements(nodes: Array<string | SyntaxTreeNode>) {
   const results = [""];
-  nodes.map(escapeNode).forEach(node => {
+  nodes.forEach(node => {
     if (typeof node !== "string" && node.type === SyntaxTreeNodeType.SQL_STATEMENT_SEPARATOR) {
       results.push("");
       return;
     }
-    results[results.length - 1] += typeof node === "string" ? node : node.concatenate();
+    results[results.length - 1] += escapeNode(node);
   });
   return results;
 }
 
+const SQL_STATEMENT_ESCAPERS = new Map([
+  [
+    SyntaxTreeNodeType.SQL_COMMENT,
+    (str: string) => str.replace(/`/g, "\\`").replace(/\${/g, "\\${")
+  ],
+  [
+    SyntaxTreeNodeType.SQL_LITERAL_STRING,
+    (str: string) => str.replace(/\\/g, "\\\\").replace(/\`/g, "\\`")
+  ]
+]);
+
 function escapeNode(node: string | SyntaxTreeNode) {
   if (typeof node === "string") {
-    return node.replace(/\\/g, "\\\\").replace(/\`/g, "\\`");
-  }
-  switch (node.type) {
-    case SyntaxTreeNodeType.SQL_COMMENT:
-      // Any code (i.e. JavaScript placeholder strings) inside comments should not run, so escape it.
-      return node
-        .concatenate()
-        .replace(/`/g, "\\`")
-        .replace(/\${/g, "\\${");
-    case SyntaxTreeNodeType.SQL_LITERAL_STRING:
-      // Literal strings may contain backslashes or backticks which need to be escaped.
-      return node
-        .concatenate()
-        .replace(/\\/g, "\\\\")
-        .replace(/\`/g, "\\`");
+    return SQL_STATEMENT_ESCAPERS.get(SyntaxTreeNodeType.SQL_LITERAL_STRING)(node);
   }
-  return node;
+  return node.concatenate(SQL_STATEMENT_ESCAPERS);
 }

sqlx/lexer.ts

@@ -20,7 +20,6 @@ const SQL_LEXER_TOKEN_NAMES = {
   SINGLE_LINE_COMMENT: LEXER_STATE_NAMES.SQL + "_singleLineComment",
   MULTI_LINE_COMMENT: LEXER_STATE_NAMES.SQL + "_multiLineComment",
   START_JS_PLACEHOLDER: LEXER_STATE_NAMES.SQL + "_startJsPlaceholder",
-  BACKSLASH: LEXER_STATE_NAMES.SQL + "_backslash",
   BACKTICK: LEXER_STATE_NAMES.SQL + "_backtick",
   START_QUOTE_SINGLE: LEXER_STATE_NAMES.INNER_SQL_BLOCK + "_startQuoteSingle",
   START_QUOTE_DOUBLE: LEXER_STATE_NAMES.INNER_SQL_BLOCK + "_startQuoteDouble",
@@ -52,7 +51,6 @@ const INNER_SQL_BLOCK_LEXER_TOKEN_NAMES = {
   MULTI_LINE_COMMENT: LEXER_STATE_NAMES.INNER_SQL_BLOCK + "_multiLineComment",
   START_JS_PLACEHOLDER: LEXER_STATE_NAMES.INNER_SQL_BLOCK + "_startJsPlaceholder",
   CLOSE_BLOCK: LEXER_STATE_NAMES.INNER_SQL_BLOCK + "_closeBlock",
-  BACKSLASH: LEXER_STATE_NAMES.INNER_SQL_BLOCK + "_backslash",
   BACKTICK: LEXER_STATE_NAMES.INNER_SQL_BLOCK + "_backtick",
   START_QUOTE_SINGLE: LEXER_STATE_NAMES.INNER_SQL_BLOCK + "_startQuoteSingle",
   START_QUOTE_DOUBLE: LEXER_STATE_NAMES.INNER_SQL_BLOCK + "_startQuoteDouble",
@@ -62,15 +60,15 @@ const INNER_SQL_BLOCK_LEXER_TOKEN_NAMES = {
 const SQL_SINGLE_QUOTE_STRING_LEXER_TOKEN_NAMES = {
   ESCAPED_BACKSLASH: LEXER_STATE_NAMES.SQL_SINGLE_QUOTE_STRING + "_escapedBackslash",
   ESCAPED_QUOTE: LEXER_STATE_NAMES.SQL_SINGLE_QUOTE_STRING + "_escapedQuoteSingle",
-  BACKSLASH: LEXER_STATE_NAMES.SQL_SINGLE_QUOTE_STRING + "_backslash",
+  START_JS_PLACEHOLDER: LEXER_STATE_NAMES.SQL_SINGLE_QUOTE_STRING + "_startJsPlaceholder",
   CLOSE_QUOTE: LEXER_STATE_NAMES.SQL_SINGLE_QUOTE_STRING + "_closeQuoteSingle",
   CAPTURE_EVERYTHING_ELSE: LEXER_STATE_NAMES.SQL_SINGLE_QUOTE_STRING + "_captureEverythingElse"
 };
 
 const SQL_DOUBLE_QUOTE_STRING_LEXER_TOKEN_NAMES = {
   ESCAPED_BACKSLASH: LEXER_STATE_NAMES.SQL_DOUBLE_QUOTE_STRING + "_escapedBackslash",
   ESCAPED_QUOTE: LEXER_STATE_NAMES.SQL_DOUBLE_QUOTE_STRING + "_escapedQuoteDouble",
-  BACKSLASH: LEXER_STATE_NAMES.SQL_DOUBLE_QUOTE_STRING + "_backslash",
+  START_JS_PLACEHOLDER: LEXER_STATE_NAMES.SQL_DOUBLE_QUOTE_STRING + "_startJsPlaceholder",
   CLOSE_QUOTE: LEXER_STATE_NAMES.SQL_DOUBLE_QUOTE_STRING + "_closeQuoteDouble",
   CAPTURE_EVERYTHING_ELSE: LEXER_STATE_NAMES.SQL_DOUBLE_QUOTE_STRING + "_captureEverythingElse"
 };
@@ -109,7 +107,17 @@ const START_TOKEN_NODE_MAPPINGS = new Map<string, SyntaxTreeNodeType>([
     SyntaxTreeNodeType.JAVASCRIPT_TEMPLATE_STRING_PLACEHOLDER
   ],
   [INNER_SQL_BLOCK_LEXER_TOKEN_NAMES.START_QUOTE_SINGLE, SyntaxTreeNodeType.SQL_LITERAL_STRING],
-  [INNER_SQL_BLOCK_LEXER_TOKEN_NAMES.START_QUOTE_DOUBLE, SyntaxTreeNodeType.SQL_LITERAL_STRING]
+  [INNER_SQL_BLOCK_LEXER_TOKEN_NAMES.START_QUOTE_DOUBLE, SyntaxTreeNodeType.SQL_LITERAL_STRING],
+
+  [
+    SQL_SINGLE_QUOTE_STRING_LEXER_TOKEN_NAMES.START_JS_PLACEHOLDER,
+    SyntaxTreeNodeType.JAVASCRIPT_TEMPLATE_STRING_PLACEHOLDER
+  ],
+
+  [
+    SQL_DOUBLE_QUOTE_STRING_LEXER_TOKEN_NAMES.START_JS_PLACEHOLDER,
+    SyntaxTreeNodeType.JAVASCRIPT_TEMPLATE_STRING_PLACEHOLDER
+  ]
 ]);
 
 const CLOSE_TOKEN_TYPES = new Set<string>([
@@ -176,11 +184,12 @@ export class SyntaxTreeNode {
     return this.allChildren.slice();
   }
 
-  public concatenate(): string {
+  public concatenate(mutators?: Map<SyntaxTreeNodeType, (str: string) => string>): string {
+    const mutator = mutators?.has(this.type) ? mutators.get(this.type) : (str: string) => str;
     return this.allChildren
       .map(child => {
         if (typeof child === "string") {
-          return child;
+          return mutator(child);
         }
         return child.concatenate();
       })
@@ -259,7 +268,6 @@ function buildSqlxLexer(): { [x: string]: moo.Rules } {
     match: "${",
     push: LEXER_STATE_NAMES.JS_BLOCK
   };
-  sqlLexer[SQL_LEXER_TOKEN_NAMES.BACKSLASH] = "\\";
   sqlLexer[SQL_LEXER_TOKEN_NAMES.BACKTICK] = "`";
   sqlLexer[SQL_LEXER_TOKEN_NAMES.START_QUOTE_SINGLE] = {
     match: "'",
@@ -337,6 +345,10 @@ function buildSqlxLexer(): { [x: string]: moo.Rules } {
   const innerSingleQuoteLexer: moo.Rules = {};
   innerSingleQuoteLexer[SQL_SINGLE_QUOTE_STRING_LEXER_TOKEN_NAMES.ESCAPED_BACKSLASH] = "\\\\";
   innerSingleQuoteLexer[SQL_SINGLE_QUOTE_STRING_LEXER_TOKEN_NAMES.ESCAPED_QUOTE] = "\\'";
+  innerSingleQuoteLexer[SQL_SINGLE_QUOTE_STRING_LEXER_TOKEN_NAMES.START_JS_PLACEHOLDER] = {
+    match: "${",
+    push: LEXER_STATE_NAMES.JS_BLOCK
+  };
   innerSingleQuoteLexer[SQL_SINGLE_QUOTE_STRING_LEXER_TOKEN_NAMES.CLOSE_QUOTE] = {
     match: "'",
     pop: 1
@@ -351,6 +363,10 @@ function buildSqlxLexer(): { [x: string]: moo.Rules } {
   innerDoubleQuoteLexer[SQL_DOUBLE_QUOTE_STRING_LEXER_TOKEN_NAMES.ESCAPED_QUOTE] = {
     match: '\\"'
   };
+  innerSingleQuoteLexer[SQL_DOUBLE_QUOTE_STRING_LEXER_TOKEN_NAMES.START_JS_PLACEHOLDER] = {
+    match: "${",
+    push: LEXER_STATE_NAMES.JS_BLOCK
+  };
   innerDoubleQuoteLexer[SQL_DOUBLE_QUOTE_STRING_LEXER_TOKEN_NAMES.CLOSE_QUOTE] = {
     match: '"',
     pop: 1

tests/core/core.spec.ts

@@ -929,5 +929,17 @@ post_operations {
         )
       ).eql(await fs.readFile("tests/core/strings-act-literally.js.test", "utf8"));
     });
+    test("JS placeholders inside SQL strings", async () => {
+      expect(
+        compilers.compile(
+          `
+select '\${\`bar\`}'
+`,
+          "file.sqlx"
+        )
+      ).eql(
+        await fs.readFile("tests/core/js-placeholder-strings-inside-sql-strings.js.test", "utf8")
+      );
+    });
   });
 });

tests/core/js-placeholder-strings-inside-sql-strings.js.test

@@ -0,0 +1,26 @@
+session.sqlxAction({
+  sqlxConfig: {
+    name: "file",
+    type: "operations",
+    ...{}
+  },
+  sqlStatementCount: 1,
+  sqlContextable: (ctx) => {
+    const self = ctx.self ? ctx.self.bind(ctx) : undefined;
+const ref = ctx.ref ? ctx.ref.bind(ctx) : undefined;
+const resolve = ctx.resolve ? ctx.resolve.bind(ctx) : undefined;
+const name = ctx.name ? ctx.name.bind(ctx) : undefined;
+const when = ctx.when ? ctx.when.bind(ctx) : undefined;
+const incremental = ctx.incremental ? ctx.incremental.bind(ctx) : undefined;
+
+    return [`
+select '${`bar`}'
+`];
+  },
+  incrementalWhereContextable: undefined,
+  preOperationsContextable: undefined,
+  postOperationsContextable: undefined,
+  inputContextables: [
+
+  ]
+});

version.bzl

@@ -1,3 +1,3 @@
 # NOTE: If you change the format of this line, you must change the bash command
 # in /scripts/publish to extract the version string correctly.
-DF_VERSION = "1.9.2"
+DF_VERSION = "1.9.3"