Fix: TM language would tokenize too many things after line comment (#…

…2482) fix #2471
microsoft · Sep 26, 2023 · b9452d0 · b9452d0
1 parent fb76f02
commit b9452d0
Show file tree

Hide file tree

Showing 3 changed files with 68 additions and 1 deletion.
diff --git a/common/changes/@typespec/compiler/fix-comment-tmlanguage_2023-09-25-20-36.json b/common/changes/@typespec/compiler/fix-comment-tmlanguage_2023-09-25-20-36.json
@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@typespec/compiler",
+      "comment": "Fix: Colorization of line comment was bleeding over to the next line(s).",
+      "type": "none"
+    }
+  ],
+  "packageName": "@typespec/compiler"
+}
diff --git a/packages/compiler/src/server/tmlanguage.ts b/packages/compiler/src/server/tmlanguage.ts
@@ -205,7 +205,7 @@ const docCommentBlock: IncludeRule = {
 const docComment: BeginEndRule = {
   key: "doc-comment",
   scope: "comment.block.tsp",
-  begin: "/\\**",
+  begin: "/\\*\\*",
   beginCaptures: {
     "0": { scope: "comment.block.tsp" },
   },

diff --git a/packages/compiler/test/server/colorization.test.ts b/packages/compiler/test/server/colorization.test.ts
@@ -821,6 +821,63 @@ function testColorization(description: string, tokenize: Tokenize) {
       });
     });
 
+    if (tokenize === tokenizeTMLanguage) {
+      describe("comments", () => {
+        it("tokenize empty line comment", async () => {
+          const tokens = await tokenize(`
+        //
+        `);
+          deepStrictEqual(tokens, [Token.comment.line("//")]);
+        });
+        it("tokenize line comment", async () => {
+          const tokens = await tokenize(`
+        // This is a line comment
+        `);
+          deepStrictEqual(tokens, [Token.comment.line("// This is a line comment")]);
+        });
+        it("tokenize line comment before statement", async () => {
+          const tokens = await tokenize(`
+        // Comment
+        model Foo {}
+        `);
+          deepStrictEqual(tokens, [
+            Token.comment.line("// Comment"),
+            Token.keywords.model,
+            Token.identifiers.type("Foo"),
+            Token.punctuation.openBrace,
+            Token.punctuation.closeBrace,
+          ]);
+        });
+
+        it("tokenize single line block comment", async () => {
+          const tokens = await tokenize(`
+          /* Comment */
+          `);
+          deepStrictEqual(tokens, [
+            Token.comment.block("/*"),
+            Token.comment.block(" Comment "),
+            Token.comment.block("*/"),
+          ]);
+        });
+
+        it("tokenize multi line block comment", async () => {
+          const tokens = await tokenize(`
+          /*
+            Comment
+            on multi line
+          */
+          `);
+          deepStrictEqual(tokens, [
+            Token.comment.block("/*"),
+            Token.comment.block("            Comment"),
+            Token.comment.block("            on multi line"),
+            Token.comment.block("          "),
+            Token.comment.block("*/"),
+          ]);
+        });
+      });
+    }
+
     /**
      * Doc comment
      * @param foo Foo desc