From 51c8dfb6ec68f904059c59414e3f8eff4ce07690 Mon Sep 17 00:00:00 2001 From: XFFXFF <1247714429@qq.com> Date: Sun, 12 Nov 2023 08:21:38 +0800 Subject: [PATCH 1/3] fix incorrect spans --- components/lox-ir/src/token_tree.rs | 1 + components/lox-lex/src/lex.rs | 9 +++++- components/lox-parse/src/parser.rs | 24 ++++++++++++++ components/lox-parse/src/tokens.rs | 11 +++++-- .../diagnostics/error_in_function/bytecode | 2 +- .../diagnostics/error_in_function/output | 14 ++++---- .../diagnostics/error_in_function/syntax | 2 +- lox_tests/diagnostics/error_in_function/token | 32 +++++++++++++++++++ 8 files changed, 83 insertions(+), 12 deletions(-) diff --git a/components/lox-ir/src/token_tree.rs b/components/lox-ir/src/token_tree.rs index 83ac788..7977369 100644 --- a/components/lox-ir/src/token_tree.rs +++ b/components/lox-ir/src/token_tree.rs @@ -22,6 +22,7 @@ impl<'db> DebugWithDb for TokenTree { ) -> std::fmt::Result { f.debug_struct("TokenTree") .field("source text", &self.input_file(db).source_text(db)) + .field("span", &self.span(db)) .field("tokens", &self.tokens(db).debug(db)) .finish() } diff --git a/components/lox-lex/src/lex.rs b/components/lox-lex/src/lex.rs index e939b81..af35ee6 100644 --- a/components/lox-lex/src/lex.rs +++ b/components/lox-lex/src/lex.rs @@ -46,8 +46,10 @@ where }; let mut end_pos = 0; + let mut start_pos = self.input_file.source_text(self.db).len(); while let Some((pos, ch)) = self.chars.peek().cloned() { end_pos = end_pos.max(pos); + start_pos = start_pos.min(pos); if Some(ch) == end_ch { break; @@ -104,7 +106,12 @@ where } } - TokenTree::new(self.db, self.input_file, Span::from(0u32, end_pos), tokens) + TokenTree::new( + self.db, + self.input_file, + Span::from(start_pos, end_pos), + tokens, + ) } /// Accumulate `ch0` and following characters while `matches` returns true diff --git a/components/lox-parse/src/parser.rs b/components/lox-parse/src/parser.rs index 40ed998..1767b0e 100644 --- a/components/lox-parse/src/parser.rs +++ b/components/lox-parse/src/parser.rs @@ -30,6 +30,7 @@ impl<'me> Parser<'me> { } pub(crate) fn parse(&mut self) -> Vec { + tracing::debug!("parsing {:?}", self.tokens.tokens); let mut stmts = vec![]; while let Some(stmt) = self.declaration() { stmts.push(stmt); @@ -42,6 +43,7 @@ impl<'me> Parser<'me> { stmts } + #[tracing::instrument(skip(self))] fn declaration(&mut self) -> Option { if self.eat(Keyword::Var).is_some() { self.var_declaration() @@ -52,6 +54,7 @@ impl<'me> Parser<'me> { } } + #[tracing::instrument(skip(self))] fn func_declaration(&mut self) -> Option { let name = self.eat(Identifier)?.1; let parameters_tree = self.delimited('(')?.1; @@ -71,6 +74,7 @@ impl<'me> Parser<'me> { } // "var" IDENTIFIER ( "=" expression )? ";" ; + #[tracing::instrument(skip(self))] fn var_declaration(&mut self) -> Option { if let Some((_, id)) = self.eat(Identifier) { let initializer = if self.eat_op(Op::Equal).is_some() { @@ -90,6 +94,7 @@ impl<'me> Parser<'me> { } } + #[tracing::instrument(skip(self))] fn stmt(&mut self) -> Option { if self.eat(Keyword::Print).is_some() { return self.print_stmt(); @@ -110,6 +115,7 @@ impl<'me> Parser<'me> { self.expr_stmt() } + #[tracing::instrument(skip(self))] fn return_stmt(&mut self) -> Option { if self.eat(Token::Semicolon).is_some() { return Some(Stmt::Return(None)); @@ -123,6 +129,7 @@ impl<'me> Parser<'me> { // forStmt → "for" "(" ( varDecl | exprStmt | ";" ) // expression? ";" // expression? ")" statement ; + #[tracing::instrument(skip(self))] fn for_stmt(&mut self) -> Option { let (_, token_tree) = self.delimited('(')?; let mut sub_parser = Parser::new(self.db, token_tree); @@ -162,6 +169,7 @@ impl<'me> Parser<'me> { }) } + #[tracing::instrument(skip(self))] fn while_stmt(&mut self) -> Option { let (_, token_tree) = self.delimited('(')?; let condition = Parser::new(self.db, token_tree).parse_expr()?; @@ -172,6 +180,7 @@ impl<'me> Parser<'me> { }) } + #[tracing::instrument(skip(self))] fn if_stmt(&mut self) -> Option { let (_, token_tree) = self.delimited('(')?; let condition = Parser::new(self.db, token_tree).parse_expr()?; @@ -189,6 +198,7 @@ impl<'me> Parser<'me> { } // "print" expression ";" ; + #[tracing::instrument(skip(self))] fn print_stmt(&mut self) -> Option { let expr = self.parse_expr()?; self.eat(Token::Semicolon) @@ -196,6 +206,7 @@ impl<'me> Parser<'me> { Some(Stmt::Print(expr)) } + #[tracing::instrument(skip(self))] fn expr_stmt(&mut self) -> Option { let expr = self.parse_expr()?; self.eat(Token::Semicolon) @@ -219,6 +230,7 @@ impl<'me> Parser<'me> { // assignment -> IDENTIFIER "=" assignment | logic_or ; // assignment is not a statement, it is an expression + #[tracing::instrument(skip(self))] fn assignment(&mut self) -> Option { let expr = self.logic_or()?; if self.eat_op(Op::Equal).is_some() { @@ -238,6 +250,7 @@ impl<'me> Parser<'me> { } // logic_or -> logic_and ( "or" logic_and )* ; + #[tracing::instrument(skip(self))] fn logic_or(&mut self) -> Option { let mut left = self.logic_and()?; @@ -253,6 +266,7 @@ impl<'me> Parser<'me> { } // logic_and -> equality ( "and" equality )* ; + #[tracing::instrument(skip(self))] fn logic_and(&mut self) -> Option { let mut left = self.equality()?; @@ -267,6 +281,7 @@ impl<'me> Parser<'me> { Some(left) } + #[tracing::instrument(skip(self))] fn equality(&mut self) -> Option { let mut left = self.comparison()?; @@ -284,6 +299,7 @@ impl<'me> Parser<'me> { Some(left) } + #[tracing::instrument(skip(self))] fn comparison(&mut self) -> Option { let mut left = self.term()?; @@ -301,6 +317,7 @@ impl<'me> Parser<'me> { Some(left) } + #[tracing::instrument(skip(self))] fn term(&mut self) -> Option { let mut left = self.factor()?; @@ -316,6 +333,7 @@ impl<'me> Parser<'me> { Some(left) } + #[tracing::instrument(skip(self))] fn factor(&mut self) -> Option { let mut left = self.unary()?; @@ -347,6 +365,7 @@ impl<'me> Parser<'me> { None } + #[tracing::instrument(skip(self))] fn unary(&mut self) -> Option { for op in &[Op::Minus, Op::Bang] { if self.eat_op(*op).is_some() { @@ -357,6 +376,7 @@ impl<'me> Parser<'me> { self.call() } + #[tracing::instrument(skip(self))] fn call(&mut self) -> Option { let mut expr = self.primary()?; loop { @@ -382,6 +402,7 @@ impl<'me> Parser<'me> { Some(expr) } + #[tracing::instrument(skip(self))] fn primary(&mut self) -> Option { if self.eat(Keyword::True).is_some() { Some(Expr::BooleanLiteral(true)) @@ -415,9 +436,11 @@ impl<'me> Parser<'me> { /// returns the span + narrowed view. Otherwise returns None /// and has no effect. Returns None if there is no pending token. fn eat(&mut self, test: TT) -> Option<(Span, TT::Narrow)> { + tracing::debug!("trying to eat {:?}", test); let span = self.tokens.peek_span(); let narrow = self.peek(test)?; self.tokens.consume(); + tracing::debug!("ate {:?}", narrow); Some((span, narrow)) } @@ -469,6 +492,7 @@ impl<'me> Parser<'me> { /// If the next tokens match the given operator, consume it and /// return. + #[tracing::instrument(skip(self))] fn eat_op(&mut self, op: Op) -> Option { let (span, tokens) = self.test_op(op)?; self.tokens = tokens; diff --git a/components/lox-parse/src/tokens.rs b/components/lox-parse/src/tokens.rs index 9a69356..a2a8b3d 100644 --- a/components/lox-parse/src/tokens.rs +++ b/components/lox-parse/src/tokens.rs @@ -1,4 +1,5 @@ use lox_ir::{span::Span, token::Token, token_tree::TokenTree}; +use salsa::DebugWithDb; #[derive(Copy, Clone)] pub(crate) struct Tokens<'me> { @@ -11,7 +12,7 @@ pub(crate) struct Tokens<'me> { last_not_skipped_span: Span, skipped: Skipped, - tokens: &'me [Token], + pub(crate) tokens: &'me [Token], } #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug)] @@ -24,6 +25,7 @@ enum Skipped { impl<'me> Tokens<'me> { pub(crate) fn new(db: &'me dyn crate::Db, token_tree: TokenTree) -> Self { let start_span = token_tree.span(db).span_at_start(); + tracing::info!("start_span={:?}", start_span); let tokens = token_tree.tokens(db); let mut this = Tokens { db, @@ -84,12 +86,17 @@ impl<'me> Tokens<'me> { } /// Advance by one token and return the span + token just consumed (if any). + #[tracing::instrument(skip(self))] pub(crate) fn consume(&mut self) -> Option { let token = self.next_token(false)?; self.skip_tokens(); - tracing::debug!("consumed {token:?}, lookahead={:?}", self.peek()); + tracing::debug!( + "consumed {:?}, lookahead={:?}", + token.debug(self.db), + self.peek() + ); Some(token) } diff --git a/lox_tests/diagnostics/error_in_function/bytecode b/lox_tests/diagnostics/error_in_function/bytecode index 66637c5..d643392 100644 --- a/lox_tests/diagnostics/error_in_function/bytecode +++ b/lox_tests/diagnostics/error_in_function/bytecode @@ -6,7 +6,7 @@ CompiledFunction { Function( Function( Id { - value: 6, + value: 1, }, ), ), diff --git a/lox_tests/diagnostics/error_in_function/output b/lox_tests/diagnostics/error_in_function/output index c171d61..4ed465c 100644 --- a/lox_tests/diagnostics/error_in_function/output +++ b/lox_tests/diagnostics/error_in_function/output @@ -1,8 +1,8 @@ -Error: expected `;` - ╭─[/home/zhoufan/workspace/rust/lox/lox_tests/diagnostics/error_in_function.lox:2:5] - │ - 2 │ print "hello" - │ ┬ - │ ╰── here -───╯ +Error: expected `;` + ╭─[/home/zhoufan/workspace/rust/lox/lox_tests/diagnostics/error_in_function.lox:2:5] + │ + 2 │     print "hello" +  │ ┬ +  │ ╰── here +───╯ diff --git a/lox_tests/diagnostics/error_in_function/syntax b/lox_tests/diagnostics/error_in_function/syntax index 51faa42..11e399e 100644 --- a/lox_tests/diagnostics/error_in_function/syntax +++ b/lox_tests/diagnostics/error_in_function/syntax @@ -1,7 +1,7 @@ FunctionDeclaration { function: Function( Id { - value: 6, + value: 1, }, ), } diff --git a/lox_tests/diagnostics/error_in_function/token b/lox_tests/diagnostics/error_in_function/token index b1d1d71..6fb4d5f 100644 --- a/lox_tests/diagnostics/error_in_function/token +++ b/lox_tests/diagnostics/error_in_function/token @@ -1,5 +1,13 @@ TokenTree { source text: "fun hello() {\n print \"hello\"\n}\n\nhello();", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 42, + ), + }, tokens: [ Alphabetic(fun), Whitespace(' '), @@ -8,6 +16,14 @@ TokenTree { Tree( TokenTree { source text: "fun hello() {\n print \"hello\"\n}\n\nhello();", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 10, + ), + }, tokens: [], }, ), @@ -17,6 +33,14 @@ TokenTree { Tree( TokenTree { source text: "fun hello() {\n print \"hello\"\n}\n\nhello();", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 32, + ), + }, tokens: [ Whitespace('\n'), Whitespace(' '), @@ -38,6 +62,14 @@ TokenTree { Tree( TokenTree { source text: "fun hello() {\n print \"hello\"\n}\n\nhello();", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 41, + ), + }, tokens: [], }, ), From 9daa17ee97cb9e0b47041387b32235d52b444975 Mon Sep 17 00:00:00 2001 From: XFFXFF <1247714429@qq.com> Date: Sun, 12 Nov 2023 08:22:27 +0800 Subject: [PATCH 2/3] update tests --- lox_tests/and/token | 8 +++ lox_tests/arithmetic/token | 8 +++ lox_tests/assignment/token | 8 +++ lox_tests/boolean/token | 8 +++ lox_tests/comparison/token | 8 +++ .../diagnostics/error_in_function/bytecode | 2 +- .../diagnostics/error_in_function/output | 8 +-- .../diagnostics/error_in_function/syntax | 2 +- lox_tests/diagnostics/error_in_function/token | 6 +- lox_tests/equality/token | 8 +++ lox_tests/fib/token | 64 +++++++++++++++++++ lox_tests/for/token | 16 +++++ lox_tests/func/token | 56 ++++++++++++++++ lox_tests/if/token | 32 ++++++++++ lox_tests/local_var/token | 24 +++++++ lox_tests/local_var_assign/token | 24 +++++++ lox_tests/or/token | 8 +++ lox_tests/print/token | 8 +++ lox_tests/string/token | 8 +++ lox_tests/var/token | 8 +++ lox_tests/while/token | 24 +++++++ 21 files changed, 329 insertions(+), 9 deletions(-) diff --git a/lox_tests/and/token b/lox_tests/and/token index 8365ffa..d14c8ea 100644 --- a/lox_tests/and/token +++ b/lox_tests/and/token @@ -1,5 +1,13 @@ TokenTree { source text: "false and true;", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 14, + ), + }, tokens: [ Alphabetic(false), Whitespace(' '), diff --git a/lox_tests/arithmetic/token b/lox_tests/arithmetic/token index 7ffaa6e..05ff05c 100644 --- a/lox_tests/arithmetic/token +++ b/lox_tests/arithmetic/token @@ -1,5 +1,13 @@ TokenTree { source text: "1 + 2;\n\n1 + 2 * 3;\n\n1 - 2 / 3;\n\n1 + 2 * -3;", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 42, + ), + }, tokens: [ Number(1), Whitespace(' '), diff --git a/lox_tests/assignment/token b/lox_tests/assignment/token index e63f65b..74ec8fb 100644 --- a/lox_tests/assignment/token +++ b/lox_tests/assignment/token @@ -1,5 +1,13 @@ TokenTree { source text: "var a = 1;\na = 2;\nprint a;\nprint a = 3;\nvar b = a;\nvar c;\nc = b;\nprint c;", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 72, + ), + }, tokens: [ Alphabetic(var), Whitespace(' '), diff --git a/lox_tests/boolean/token b/lox_tests/boolean/token index 08e0dc0..d47b997 100644 --- a/lox_tests/boolean/token +++ b/lox_tests/boolean/token @@ -1,5 +1,13 @@ TokenTree { source text: "true;\nfalse;\n!true;\n!false;", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 26, + ), + }, tokens: [ Alphabetic(true), Semicolon, diff --git a/lox_tests/comparison/token b/lox_tests/comparison/token index 8b0c993..0b130ba 100644 --- a/lox_tests/comparison/token +++ b/lox_tests/comparison/token @@ -1,5 +1,13 @@ TokenTree { source text: "1 < 2;\n1 <= 1;\n1 > 2;\n1 >= 1;", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 28, + ), + }, tokens: [ Number(1), Whitespace(' '), diff --git a/lox_tests/diagnostics/error_in_function/bytecode b/lox_tests/diagnostics/error_in_function/bytecode index d643392..66637c5 100644 --- a/lox_tests/diagnostics/error_in_function/bytecode +++ b/lox_tests/diagnostics/error_in_function/bytecode @@ -6,7 +6,7 @@ CompiledFunction { Function( Function( Id { - value: 1, + value: 6, }, ), ), diff --git a/lox_tests/diagnostics/error_in_function/output b/lox_tests/diagnostics/error_in_function/output index 4ed465c..4ee6e35 100644 --- a/lox_tests/diagnostics/error_in_function/output +++ b/lox_tests/diagnostics/error_in_function/output @@ -1,8 +1,8 @@ Error: expected `;` - ╭─[/home/zhoufan/workspace/rust/lox/lox_tests/diagnostics/error_in_function.lox:2:5] + ╭─[/home/zhoufan/workspace/rust/lox/lox_tests/diagnostics/error_in_function.lox:2:18] │ - 2 │     print "hello" -  │ ┬ -  │ ╰── here + 2 │     print "hello" +  │ ┬ +  │ ╰── here ───╯ diff --git a/lox_tests/diagnostics/error_in_function/syntax b/lox_tests/diagnostics/error_in_function/syntax index 11e399e..51faa42 100644 --- a/lox_tests/diagnostics/error_in_function/syntax +++ b/lox_tests/diagnostics/error_in_function/syntax @@ -1,7 +1,7 @@ FunctionDeclaration { function: Function( Id { - value: 1, + value: 6, }, ), } diff --git a/lox_tests/diagnostics/error_in_function/token b/lox_tests/diagnostics/error_in_function/token index 6fb4d5f..ac6b6b1 100644 --- a/lox_tests/diagnostics/error_in_function/token +++ b/lox_tests/diagnostics/error_in_function/token @@ -18,7 +18,7 @@ TokenTree { source text: "fun hello() {\n print \"hello\"\n}\n\nhello();", span: Span { start: Offset( - 0, + 10, ), end: Offset( 10, @@ -35,7 +35,7 @@ TokenTree { source text: "fun hello() {\n print \"hello\"\n}\n\nhello();", span: Span { start: Offset( - 0, + 13, ), end: Offset( 32, @@ -64,7 +64,7 @@ TokenTree { source text: "fun hello() {\n print \"hello\"\n}\n\nhello();", span: Span { start: Offset( - 0, + 41, ), end: Offset( 41, diff --git a/lox_tests/equality/token b/lox_tests/equality/token index a47b266..b04b93d 100644 --- a/lox_tests/equality/token +++ b/lox_tests/equality/token @@ -1,5 +1,13 @@ TokenTree { source text: "1 == 2;\n1 != 2;", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 14, + ), + }, tokens: [ Number(1), Whitespace(' '), diff --git a/lox_tests/fib/token b/lox_tests/fib/token index 168a6e8..1170878 100644 --- a/lox_tests/fib/token +++ b/lox_tests/fib/token @@ -1,5 +1,13 @@ TokenTree { source text: "fun fib(n) {\n if (n <= 1) { return n; }\n return fib(n - 1) + fib(n - 2);\n}\n\nprint fib(5);", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 94, + ), + }, tokens: [ Alphabetic(fun), Whitespace(' '), @@ -8,6 +16,14 @@ TokenTree { Tree( TokenTree { source text: "fun fib(n) {\n if (n <= 1) { return n; }\n return fib(n - 1) + fib(n - 2);\n}\n\nprint fib(5);", + span: Span { + start: Offset( + 8, + ), + end: Offset( + 9, + ), + }, tokens: [ Alphabetic(n), ], @@ -19,6 +35,14 @@ TokenTree { Tree( TokenTree { source text: "fun fib(n) {\n if (n <= 1) { return n; }\n return fib(n - 1) + fib(n - 2);\n}\n\nprint fib(5);", + span: Span { + start: Offset( + 12, + ), + end: Offset( + 79, + ), + }, tokens: [ Whitespace('\n'), Whitespace(' '), @@ -31,6 +55,14 @@ TokenTree { Tree( TokenTree { source text: "fun fib(n) {\n if (n <= 1) { return n; }\n return fib(n - 1) + fib(n - 2);\n}\n\nprint fib(5);", + span: Span { + start: Offset( + 21, + ), + end: Offset( + 27, + ), + }, tokens: [ Alphabetic(n), Whitespace(' '), @@ -47,6 +79,14 @@ TokenTree { Tree( TokenTree { source text: "fun fib(n) {\n if (n <= 1) { return n; }\n return fib(n - 1) + fib(n - 2);\n}\n\nprint fib(5);", + span: Span { + start: Offset( + 30, + ), + end: Offset( + 41, + ), + }, tokens: [ Whitespace(' '), Alphabetic(return), @@ -70,6 +110,14 @@ TokenTree { Tree( TokenTree { source text: "fun fib(n) {\n if (n <= 1) { return n; }\n return fib(n - 1) + fib(n - 2);\n}\n\nprint fib(5);", + span: Span { + start: Offset( + 58, + ), + end: Offset( + 63, + ), + }, tokens: [ Alphabetic(n), Whitespace(' '), @@ -88,6 +136,14 @@ TokenTree { Tree( TokenTree { source text: "fun fib(n) {\n if (n <= 1) { return n; }\n return fib(n - 1) + fib(n - 2);\n}\n\nprint fib(5);", + span: Span { + start: Offset( + 71, + ), + end: Offset( + 76, + ), + }, tokens: [ Alphabetic(n), Whitespace(' '), @@ -113,6 +169,14 @@ TokenTree { Tree( TokenTree { source text: "fun fib(n) {\n if (n <= 1) { return n; }\n return fib(n - 1) + fib(n - 2);\n}\n\nprint fib(5);", + span: Span { + start: Offset( + 92, + ), + end: Offset( + 93, + ), + }, tokens: [ Number(5), ], diff --git a/lox_tests/for/token b/lox_tests/for/token index 06178b8..e56c074 100644 --- a/lox_tests/for/token +++ b/lox_tests/for/token @@ -1,5 +1,13 @@ TokenTree { source text: "for (var i = 0; i < 10; i = i + 1) print i;", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 42, + ), + }, tokens: [ Alphabetic(for), Whitespace(' '), @@ -7,6 +15,14 @@ TokenTree { Tree( TokenTree { source text: "for (var i = 0; i < 10; i = i + 1) print i;", + span: Span { + start: Offset( + 5, + ), + end: Offset( + 33, + ), + }, tokens: [ Alphabetic(var), Whitespace(' '), diff --git a/lox_tests/func/token b/lox_tests/func/token index 5cd3488..441e253 100644 --- a/lox_tests/func/token +++ b/lox_tests/func/token @@ -1,5 +1,13 @@ TokenTree { source text: "fun hello() {\n print \"hello\";\n}\n\nhello();\nprint \"world\";\n\nfun add(a, b) {\n return a + b;\n}\n\nvar c = add(1, 2);\nprint c;", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 124, + ), + }, tokens: [ Alphabetic(fun), Whitespace(' '), @@ -8,6 +16,14 @@ TokenTree { Tree( TokenTree { source text: "fun hello() {\n print \"hello\";\n}\n\nhello();\nprint \"world\";\n\nfun add(a, b) {\n return a + b;\n}\n\nvar c = add(1, 2);\nprint c;", + span: Span { + start: Offset( + 10, + ), + end: Offset( + 10, + ), + }, tokens: [], }, ), @@ -17,6 +33,14 @@ TokenTree { Tree( TokenTree { source text: "fun hello() {\n print \"hello\";\n}\n\nhello();\nprint \"world\";\n\nfun add(a, b) {\n return a + b;\n}\n\nvar c = add(1, 2);\nprint c;", + span: Span { + start: Offset( + 13, + ), + end: Offset( + 33, + ), + }, tokens: [ Whitespace('\n'), Whitespace(' '), @@ -39,6 +63,14 @@ TokenTree { Tree( TokenTree { source text: "fun hello() {\n print \"hello\";\n}\n\nhello();\nprint \"world\";\n\nfun add(a, b) {\n return a + b;\n}\n\nvar c = add(1, 2);\nprint c;", + span: Span { + start: Offset( + 42, + ), + end: Offset( + 42, + ), + }, tokens: [], }, ), @@ -58,6 +90,14 @@ TokenTree { Tree( TokenTree { source text: "fun hello() {\n print \"hello\";\n}\n\nhello();\nprint \"world\";\n\nfun add(a, b) {\n return a + b;\n}\n\nvar c = add(1, 2);\nprint c;", + span: Span { + start: Offset( + 69, + ), + end: Offset( + 73, + ), + }, tokens: [ Alphabetic(a), Comma, @@ -72,6 +112,14 @@ TokenTree { Tree( TokenTree { source text: "fun hello() {\n print \"hello\";\n}\n\nhello();\nprint \"world\";\n\nfun add(a, b) {\n return a + b;\n}\n\nvar c = add(1, 2);\nprint c;", + span: Span { + start: Offset( + 76, + ), + end: Offset( + 95, + ), + }, tokens: [ Whitespace('\n'), Whitespace(' '), @@ -104,6 +152,14 @@ TokenTree { Tree( TokenTree { source text: "fun hello() {\n print \"hello\";\n}\n\nhello();\nprint \"world\";\n\nfun add(a, b) {\n return a + b;\n}\n\nvar c = add(1, 2);\nprint c;", + span: Span { + start: Offset( + 110, + ), + end: Offset( + 114, + ), + }, tokens: [ Number(1), Comma, diff --git a/lox_tests/if/token b/lox_tests/if/token index fcc1be3..b6beb5c 100644 --- a/lox_tests/if/token +++ b/lox_tests/if/token @@ -1,5 +1,13 @@ TokenTree { source text: "if (true) {\n print \"true\";\n} else {\n print \"false\";\n}", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 37, + ), + }, tokens: [ Alphabetic(if), Whitespace(' '), @@ -7,6 +15,14 @@ TokenTree { Tree( TokenTree { source text: "if (true) {\n print \"true\";\n} else {\n print \"false\";\n}", + span: Span { + start: Offset( + 4, + ), + end: Offset( + 8, + ), + }, tokens: [ Alphabetic(true), ], @@ -18,6 +34,14 @@ TokenTree { Tree( TokenTree { source text: "if (true) {\n print \"true\";\n} else {\n print \"false\";\n}", + span: Span { + start: Offset( + 11, + ), + end: Offset( + 30, + ), + }, tokens: [ Whitespace('\n'), Whitespace(' '), @@ -40,6 +64,14 @@ TokenTree { Tree( TokenTree { source text: "if (true) {\n print \"true\";\n} else {\n print \"false\";\n}", + span: Span { + start: Offset( + 38, + ), + end: Offset( + 58, + ), + }, tokens: [ Whitespace('\n'), Whitespace(' '), diff --git a/lox_tests/local_var/token b/lox_tests/local_var/token index 1af887c..f287676 100644 --- a/lox_tests/local_var/token +++ b/lox_tests/local_var/token @@ -1,10 +1,26 @@ TokenTree { source text: "{\n var a = 1;\n {\n var a = 2;\n print a;\n }\n print a;\n}", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 0, + ), + }, tokens: [ Delimiter({), Tree( TokenTree { source text: "{\n var a = 1;\n {\n var a = 2;\n print a;\n }\n print a;\n}", + span: Span { + start: Offset( + 1, + ), + end: Offset( + 78, + ), + }, tokens: [ Whitespace('\n'), Whitespace(' '), @@ -28,6 +44,14 @@ TokenTree { Tree( TokenTree { source text: "{\n var a = 1;\n {\n var a = 2;\n print a;\n }\n print a;\n}", + span: Span { + start: Offset( + 22, + ), + end: Offset( + 63, + ), + }, tokens: [ Whitespace('\n'), Whitespace(' '), diff --git a/lox_tests/local_var_assign/token b/lox_tests/local_var_assign/token index 50fd3b9..2667c3b 100644 --- a/lox_tests/local_var_assign/token +++ b/lox_tests/local_var_assign/token @@ -1,10 +1,26 @@ TokenTree { source text: "{\n var a = 1;\n {\n a = 2;\n print a;\n }\n print a;\n}", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 0, + ), + }, tokens: [ Delimiter({), Tree( TokenTree { source text: "{\n var a = 1;\n {\n a = 2;\n print a;\n }\n print a;\n}", + span: Span { + start: Offset( + 1, + ), + end: Offset( + 74, + ), + }, tokens: [ Whitespace('\n'), Whitespace(' '), @@ -28,6 +44,14 @@ TokenTree { Tree( TokenTree { source text: "{\n var a = 1;\n {\n a = 2;\n print a;\n }\n print a;\n}", + span: Span { + start: Offset( + 22, + ), + end: Offset( + 59, + ), + }, tokens: [ Whitespace('\n'), Whitespace(' '), diff --git a/lox_tests/or/token b/lox_tests/or/token index 399e649..c280bd1 100644 --- a/lox_tests/or/token +++ b/lox_tests/or/token @@ -1,5 +1,13 @@ TokenTree { source text: "true or false;", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 13, + ), + }, tokens: [ Alphabetic(true), Whitespace(' '), diff --git a/lox_tests/print/token b/lox_tests/print/token index f2ee942..da0bc6c 100644 --- a/lox_tests/print/token +++ b/lox_tests/print/token @@ -1,5 +1,13 @@ TokenTree { source text: "print 1 + 2;", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 11, + ), + }, tokens: [ Alphabetic(print), Whitespace(' '), diff --git a/lox_tests/string/token b/lox_tests/string/token index f7290eb..d5c7a66 100644 --- a/lox_tests/string/token +++ b/lox_tests/string/token @@ -1,5 +1,13 @@ TokenTree { source text: "\"hello\" + \" world\";", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 18, + ), + }, tokens: [ String(hello), Whitespace(' '), diff --git a/lox_tests/var/token b/lox_tests/var/token index 56e5884..e199043 100644 --- a/lox_tests/var/token +++ b/lox_tests/var/token @@ -1,5 +1,13 @@ TokenTree { source text: "var a = 1;\nvar b;\nprint a;\nvar str1 = \"hello\";\nvar str2 = \"world\";\nprint str1 + str2;", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 84, + ), + }, tokens: [ Alphabetic(var), Whitespace(' '), diff --git a/lox_tests/while/token b/lox_tests/while/token index af4b910..4609968 100644 --- a/lox_tests/while/token +++ b/lox_tests/while/token @@ -1,5 +1,13 @@ TokenTree { source text: "var a = 0;\nwhile (a < 5) {\n print \"hello\";\n a = a + 1;\n}", + span: Span { + start: Offset( + 0, + ), + end: Offset( + 25, + ), + }, tokens: [ Alphabetic(var), Whitespace(' '), @@ -16,6 +24,14 @@ TokenTree { Tree( TokenTree { source text: "var a = 0;\nwhile (a < 5) {\n print \"hello\";\n a = a + 1;\n}", + span: Span { + start: Offset( + 18, + ), + end: Offset( + 23, + ), + }, tokens: [ Alphabetic(a), Whitespace(' '), @@ -31,6 +47,14 @@ TokenTree { Tree( TokenTree { source text: "var a = 0;\nwhile (a < 5) {\n print \"hello\";\n a = a + 1;\n}", + span: Span { + start: Offset( + 26, + ), + end: Offset( + 61, + ), + }, tokens: [ Whitespace('\n'), Whitespace(' '), From a1580181135766e8eb6493772efa5ef4b1635fb4 Mon Sep 17 00:00:00 2001 From: XFFXFF <1247714429@qq.com> Date: Sun, 12 Nov 2023 08:31:03 +0800 Subject: [PATCH 3/3] print diagnositics without colors in testing context --- components/lox-error-format/src/format.rs | 2 +- components/lox-execute/src/execute.rs | 3 ++- components/lox-execute/src/vm.rs | 16 ++++++++++--- components/lox-web/src/lib.rs | 1 + .../diagnostics/error_in_function/output | 14 +++++------ src/main.rs | 24 +++++++------------ 6 files changed, 32 insertions(+), 28 deletions(-) diff --git a/components/lox-error-format/src/format.rs b/components/lox-error-format/src/format.rs index 1fa7061..cae1159 100644 --- a/components/lox-error-format/src/format.rs +++ b/components/lox-error-format/src/format.rs @@ -8,7 +8,7 @@ use lox_ir::input_file::InputFile; pub struct FormatOptions { /// Whether or not errors should use rich formatting with colors. This is generally turned on, /// except in tests, where the escape codes obscure the error messages. - with_color: bool, + pub with_color: bool, } impl FormatOptions { diff --git a/components/lox-execute/src/execute.rs b/components/lox-execute/src/execute.rs index b3803cc..439e42c 100644 --- a/components/lox-execute/src/execute.rs +++ b/components/lox-execute/src/execute.rs @@ -16,10 +16,11 @@ pub fn execute_file( db: &impl crate::Db, input_file: InputFile, kernel: &mut impl Kernel, + diagnostic_with_color: bool, step_inspect: Option, &VM) + Clone>, ) { let main = main_function(db, input_file); - let mut vm = VM::new(main, db); + let mut vm = VM::new(db, main, diagnostic_with_color); while let ControlFlow::Next = vm.step(db, kernel, step_inspect.clone()) {} } diff --git a/components/lox-execute/src/vm.rs b/components/lox-execute/src/vm.rs index 815a10f..6b65ed4 100644 --- a/components/lox-execute/src/vm.rs +++ b/components/lox-execute/src/vm.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use lox_compile::compile_fn; +use lox_error_format::FormatOptions; use lox_ir::{ bytecode::{self, CompiledFunction}, diagnostic::Diagnostics, @@ -183,12 +184,14 @@ pub struct VM { pub stack: Vec, + diagnostic_with_color: bool, + // global variables globals: HashMap, } impl VM { - pub fn new(main: Function, db: &dyn crate::Db) -> Self { + pub fn new(db: &dyn crate::Db, main: Function, diagnostic_with_color: bool) -> Self { let function = compile_fn(db, main); let frame = CallFrame { function, @@ -206,6 +209,7 @@ impl VM { frames: vec![frame], heap, stack, + diagnostic_with_color, globals: HashMap::new(), } } @@ -428,8 +432,14 @@ impl VM { if diagnostics.is_empty() { self.push_frame(compiled_function); } else { - let output = - lox_error_format::format_diagnostics(db, &diagnostics).unwrap(); + let output = lox_error_format::format_diagnostics_with_options( + db, + &diagnostics, + FormatOptions { + with_color: self.diagnostic_with_color, + }, + ) + .unwrap(); kernel.print(&output); } } diff --git a/components/lox-web/src/lib.rs b/components/lox-web/src/lib.rs index 3447317..37a7d9d 100644 --- a/components/lox-web/src/lib.rs +++ b/components/lox-web/src/lib.rs @@ -43,6 +43,7 @@ impl Compiler { &self.db, self.input_file, &mut kernel, + true, None::, ); kernel.take_buffer() diff --git a/lox_tests/diagnostics/error_in_function/output b/lox_tests/diagnostics/error_in_function/output index 4ee6e35..347c4d5 100644 --- a/lox_tests/diagnostics/error_in_function/output +++ b/lox_tests/diagnostics/error_in_function/output @@ -1,8 +1,8 @@ -Error: expected `;` - ╭─[/home/zhoufan/workspace/rust/lox/lox_tests/diagnostics/error_in_function.lox:2:18] - │ - 2 │     print "hello" -  │ ┬ -  │ ╰── here -───╯ +Error: expected `;` + ╭─[/home/zhoufan/workspace/rust/lox/lox_tests/diagnostics/error_in_function.lox:2:18] + │ + 2 │ print "hello" + │ ┬ + │ ╰── here +───╯ diff --git a/src/main.rs b/src/main.rs index 36832e2..54bf31b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,7 +6,6 @@ use std::{ use clap::{Parser, Subcommand}; use expect_test::expect_file; use lox_db::Database; -use lox_error_format::FormatOptions; use lox_execute::kernel::{BufferKernel, StdoutKernel}; use lox_ir::{diagnostic::Diagnostics, input_file::InputFile, word::Word}; use salsa::DebugWithDb; @@ -20,7 +19,6 @@ struct TestCase { syntax: PathBuf, bytecode: PathBuf, execute: PathBuf, - diagnostic: PathBuf, stdout: PathBuf, text: String, } @@ -44,7 +42,6 @@ impl TestCase { let syntax = lox_dir.join("syntax"); let bytecode = lox_dir.join("bytecode"); let execute = lox_dir.join("execute"); - let diagnostic = lox_dir.join("diagnostic"); let output = lox_dir.join("output"); let text = fs::read_to_string(&lox).unwrap(); TestCase { @@ -53,7 +50,6 @@ impl TestCase { syntax, bytecode, execute, - diagnostic, stdout: output, text, } @@ -116,17 +112,6 @@ impl TestCase { } expect_file![self.syntax].assert_eq(&buf); - // test diagnostics - let diagnostics = lox_compile::compile_file::accumulated::(db, input_file); - let output = lox_error_format::format_diagnostics_with_options( - db, - &diagnostics, - FormatOptions::no_color(), - ); - if let Ok(output) = output { - expect_file![self.diagnostic].assert_eq(&output); - } - // test bytecode let chunk = lox_compile::compile_file(db, input_file); expect_file![self.bytecode].assert_eq(&format!("{:#?}", chunk)); @@ -145,7 +130,13 @@ impl TestCase { // test stdout let mut kernel = BufferKernel::new(); - lox_execute::execute_file(db, input_file, &mut kernel, None::); + lox_execute::execute_file( + db, + input_file, + &mut kernel, + false, + None::, + ); expect_file![self.stdout].assert_eq(kernel.buffer()); println!("ok"); @@ -226,6 +217,7 @@ fn main() { &db, input_file, &mut StdoutKernel {}, + true, None::, ); }