Merge pull request #11 from xffxff/dignositics

print dignositics with correct spans

components/lox-error-format/src/format.rs

@@ -8,7 +8,7 @@ use lox_ir::input_file::InputFile;
 pub struct FormatOptions {
     /// Whether or not errors should use rich formatting with colors. This is generally turned on,
     /// except in tests, where the escape codes obscure the error messages.
-    with_color: bool,
+    pub with_color: bool,
 }
 
 impl FormatOptions {

components/lox-execute/src/execute.rs

@@ -16,10 +16,11 @@ pub fn execute_file(
     db: &impl crate::Db,
     input_file: InputFile,
     kernel: &mut impl Kernel,
+    diagnostic_with_color: bool,
     step_inspect: Option<impl FnMut(Option<bytecode::Code>, &VM) + Clone>,
 ) {
     let main = main_function(db, input_file);
-    let mut vm = VM::new(main, db);
+    let mut vm = VM::new(db, main, diagnostic_with_color);
 
     while let ControlFlow::Next = vm.step(db, kernel, step_inspect.clone()) {}
 }

components/lox-execute/src/vm.rs

@@ -1,6 +1,7 @@
 use std::collections::HashMap;
 
 use lox_compile::compile_fn;
+use lox_error_format::FormatOptions;
 use lox_ir::{
     bytecode::{self, CompiledFunction},
     diagnostic::Diagnostics,
@@ -183,12 +184,14 @@ pub struct VM {
 
     pub stack: Vec<generational_arena::Index>,
 
+    diagnostic_with_color: bool,
+
     // global variables
     globals: HashMap<String, Value>,
 }
 
 impl VM {
-    pub fn new(main: Function, db: &dyn crate::Db) -> Self {
+    pub fn new(db: &dyn crate::Db, main: Function, diagnostic_with_color: bool) -> Self {
         let function = compile_fn(db, main);
         let frame = CallFrame {
             function,
@@ -206,6 +209,7 @@ impl VM {
             frames: vec![frame],
             heap,
             stack,
+            diagnostic_with_color,
             globals: HashMap::new(),
         }
     }
@@ -428,8 +432,14 @@ impl VM {
                         if diagnostics.is_empty() {
                             self.push_frame(compiled_function);
                         } else {
-                            let output =
-                                lox_error_format::format_diagnostics(db, &diagnostics).unwrap();
+                            let output = lox_error_format::format_diagnostics_with_options(
+                                db,
+                                &diagnostics,
+                                FormatOptions {
+                                    with_color: self.diagnostic_with_color,
+                                },
+                            )
+                            .unwrap();
                             kernel.print(&output);
                         }
                     }

components/lox-ir/src/token_tree.rs

@@ -22,6 +22,7 @@ impl<'db> DebugWithDb<dyn crate::Db + 'db> for TokenTree {
     ) -> std::fmt::Result {
         f.debug_struct("TokenTree")
             .field("source text", &self.input_file(db).source_text(db))
+            .field("span", &self.span(db))
             .field("tokens", &self.tokens(db).debug(db))
             .finish()
     }

components/lox-lex/src/lex.rs

@@ -46,8 +46,10 @@ where
         };
 
         let mut end_pos = 0;
+        let mut start_pos = self.input_file.source_text(self.db).len();
         while let Some((pos, ch)) = self.chars.peek().cloned() {
             end_pos = end_pos.max(pos);
+            start_pos = start_pos.min(pos);
 
             if Some(ch) == end_ch {
                 break;
@@ -104,7 +106,12 @@ where
             }
         }
 
-        TokenTree::new(self.db, self.input_file, Span::from(0u32, end_pos), tokens)
+        TokenTree::new(
+            self.db,
+            self.input_file,
+            Span::from(start_pos, end_pos),
+            tokens,
+        )
     }
 
     /// Accumulate `ch0` and following characters while `matches` returns true

components/lox-parse/src/parser.rs

@@ -30,6 +30,7 @@ impl<'me> Parser<'me> {
     }
 
     pub(crate) fn parse(&mut self) -> Vec<Stmt> {
+        tracing::debug!("parsing {:?}", self.tokens.tokens);
         let mut stmts = vec![];
         while let Some(stmt) = self.declaration() {
             stmts.push(stmt);
@@ -42,6 +43,7 @@ impl<'me> Parser<'me> {
         stmts
     }
 
+    #[tracing::instrument(skip(self))]
     fn declaration(&mut self) -> Option<Stmt> {
         if self.eat(Keyword::Var).is_some() {
             self.var_declaration()
@@ -52,6 +54,7 @@ impl<'me> Parser<'me> {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     fn func_declaration(&mut self) -> Option<Stmt> {
         let name = self.eat(Identifier)?.1;
         let parameters_tree = self.delimited('(')?.1;
@@ -71,6 +74,7 @@ impl<'me> Parser<'me> {
     }
 
     // "var" IDENTIFIER ( "=" expression )? ";" ;
+    #[tracing::instrument(skip(self))]
     fn var_declaration(&mut self) -> Option<Stmt> {
         if let Some((_, id)) = self.eat(Identifier) {
             let initializer = if self.eat_op(Op::Equal).is_some() {
@@ -90,6 +94,7 @@ impl<'me> Parser<'me> {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     fn stmt(&mut self) -> Option<Stmt> {
         if self.eat(Keyword::Print).is_some() {
             return self.print_stmt();
@@ -110,6 +115,7 @@ impl<'me> Parser<'me> {
         self.expr_stmt()
     }
 
+    #[tracing::instrument(skip(self))]
     fn return_stmt(&mut self) -> Option<Stmt> {
         if self.eat(Token::Semicolon).is_some() {
             return Some(Stmt::Return(None));
@@ -123,6 +129,7 @@ impl<'me> Parser<'me> {
     // forStmt        → "for" "(" ( varDecl | exprStmt | ";" )
     //              expression? ";"
     //              expression? ")" statement ;
+    #[tracing::instrument(skip(self))]
     fn for_stmt(&mut self) -> Option<Stmt> {
         let (_, token_tree) = self.delimited('(')?;
         let mut sub_parser = Parser::new(self.db, token_tree);
@@ -162,6 +169,7 @@ impl<'me> Parser<'me> {
         })
     }
 
+    #[tracing::instrument(skip(self))]
     fn while_stmt(&mut self) -> Option<Stmt> {
         let (_, token_tree) = self.delimited('(')?;
         let condition = Parser::new(self.db, token_tree).parse_expr()?;
@@ -172,6 +180,7 @@ impl<'me> Parser<'me> {
         })
     }
 
+    #[tracing::instrument(skip(self))]
     fn if_stmt(&mut self) -> Option<Stmt> {
         let (_, token_tree) = self.delimited('(')?;
         let condition = Parser::new(self.db, token_tree).parse_expr()?;
@@ -189,13 +198,15 @@ impl<'me> Parser<'me> {
     }
 
     //  "print" expression ";" ;
+    #[tracing::instrument(skip(self))]
     fn print_stmt(&mut self) -> Option<Stmt> {
         let expr = self.parse_expr()?;
         self.eat(Token::Semicolon)
             .or_report_error(self, || "expected `;`");
         Some(Stmt::Print(expr))
     }
 
+    #[tracing::instrument(skip(self))]
     fn expr_stmt(&mut self) -> Option<Stmt> {
         let expr = self.parse_expr()?;
         self.eat(Token::Semicolon)
@@ -219,6 +230,7 @@ impl<'me> Parser<'me> {
 
     // assignment     -> IDENTIFIER "=" assignment | logic_or ;
     // assignment is not a statement, it is an expression
+    #[tracing::instrument(skip(self))]
     fn assignment(&mut self) -> Option<Expr> {
         let expr = self.logic_or()?;
         if self.eat_op(Op::Equal).is_some() {
@@ -238,6 +250,7 @@ impl<'me> Parser<'me> {
     }
 
     // logic_or       -> logic_and ( "or" logic_and )* ;
+    #[tracing::instrument(skip(self))]
     fn logic_or(&mut self) -> Option<Expr> {
         let mut left = self.logic_and()?;
 
@@ -253,6 +266,7 @@ impl<'me> Parser<'me> {
     }
 
     // logic_and       -> equality ( "and" equality )* ;
+    #[tracing::instrument(skip(self))]
     fn logic_and(&mut self) -> Option<Expr> {
         let mut left = self.equality()?;
 
@@ -267,6 +281,7 @@ impl<'me> Parser<'me> {
         Some(left)
     }
 
+    #[tracing::instrument(skip(self))]
     fn equality(&mut self) -> Option<Expr> {
         let mut left = self.comparison()?;
 
@@ -284,6 +299,7 @@ impl<'me> Parser<'me> {
         Some(left)
     }
 
+    #[tracing::instrument(skip(self))]
     fn comparison(&mut self) -> Option<Expr> {
         let mut left = self.term()?;
 
@@ -301,6 +317,7 @@ impl<'me> Parser<'me> {
         Some(left)
     }
 
+    #[tracing::instrument(skip(self))]
     fn term(&mut self) -> Option<Expr> {
         let mut left = self.factor()?;
 
@@ -316,6 +333,7 @@ impl<'me> Parser<'me> {
         Some(left)
     }
 
+    #[tracing::instrument(skip(self))]
     fn factor(&mut self) -> Option<Expr> {
         let mut left = self.unary()?;
 
@@ -347,6 +365,7 @@ impl<'me> Parser<'me> {
         None
     }
 
+    #[tracing::instrument(skip(self))]
     fn unary(&mut self) -> Option<Expr> {
         for op in &[Op::Minus, Op::Bang] {
             if self.eat_op(*op).is_some() {
@@ -357,6 +376,7 @@ impl<'me> Parser<'me> {
         self.call()
     }
 
+    #[tracing::instrument(skip(self))]
     fn call(&mut self) -> Option<Expr> {
         let mut expr = self.primary()?;
         loop {
@@ -382,6 +402,7 @@ impl<'me> Parser<'me> {
         Some(expr)
     }
 
+    #[tracing::instrument(skip(self))]
     fn primary(&mut self) -> Option<Expr> {
         if self.eat(Keyword::True).is_some() {
             Some(Expr::BooleanLiteral(true))
@@ -415,9 +436,11 @@ impl<'me> Parser<'me> {
     /// returns the span + narrowed view. Otherwise returns None
     /// and has no effect. Returns None if there is no pending token.
     fn eat<TT: TokenTest>(&mut self, test: TT) -> Option<(Span, TT::Narrow)> {
+        tracing::debug!("trying to eat {:?}", test);
         let span = self.tokens.peek_span();
         let narrow = self.peek(test)?;
         self.tokens.consume();
+        tracing::debug!("ate {:?}", narrow);
         Some((span, narrow))
     }
 
@@ -469,6 +492,7 @@ impl<'me> Parser<'me> {
 
     /// If the next tokens match the given operator, consume it and
     /// return.
+    #[tracing::instrument(skip(self))]
     fn eat_op(&mut self, op: Op) -> Option<Span> {
         let (span, tokens) = self.test_op(op)?;
         self.tokens = tokens;

components/lox-parse/src/tokens.rs

@@ -1,4 +1,5 @@
 use lox_ir::{span::Span, token::Token, token_tree::TokenTree};
+use salsa::DebugWithDb;
 
 #[derive(Copy, Clone)]
 pub(crate) struct Tokens<'me> {
@@ -11,7 +12,7 @@ pub(crate) struct Tokens<'me> {
     last_not_skipped_span: Span,
 
     skipped: Skipped,
-    tokens: &'me [Token],
+    pub(crate) tokens: &'me [Token],
 }
 
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug)]
@@ -24,6 +25,7 @@ enum Skipped {
 impl<'me> Tokens<'me> {
     pub(crate) fn new(db: &'me dyn crate::Db, token_tree: TokenTree) -> Self {
         let start_span = token_tree.span(db).span_at_start();
+        tracing::info!("start_span={:?}", start_span);
         let tokens = token_tree.tokens(db);
         let mut this = Tokens {
             db,
@@ -84,12 +86,17 @@ impl<'me> Tokens<'me> {
     }
 
     /// Advance by one token and return the span + token just consumed (if any).
+    #[tracing::instrument(skip(self))]
     pub(crate) fn consume(&mut self) -> Option<Token> {
         let token = self.next_token(false)?;
 
         self.skip_tokens();
 
-        tracing::debug!("consumed {token:?}, lookahead={:?}", self.peek());
+        tracing::debug!(
+            "consumed {:?}, lookahead={:?}",
+            token.debug(self.db),
+            self.peek()
+        );
         Some(token)
     }
 

components/lox-web/src/lib.rs

@@ -43,6 +43,7 @@ impl Compiler {
                 &self.db,
                 self.input_file,
                 &mut kernel,
+                true,
                 None::<fn(_, &lox_execute::VM)>,
             );
             kernel.take_buffer()

lox_tests/and/token

@@ -1,5 +1,13 @@
 TokenTree {
     source text: "false and true;",
+    span: Span {
+        start: Offset(
+            0,
+        ),
+        end: Offset(
+            14,
+        ),
+    },
     tokens: [
         Alphabetic(false),
         Whitespace(' '),

lox_tests/arithmetic/token

@@ -1,5 +1,13 @@
 TokenTree {
     source text: "1 + 2;\n\n1 + 2 * 3;\n\n1 - 2 / 3;\n\n1 + 2 * -3;",
+    span: Span {
+        start: Offset(
+            0,
+        ),
+        end: Offset(
+            42,
+        ),
+    },
     tokens: [
         Number(1),
         Whitespace(' '),

lox_tests/assignment/token

@@ -1,5 +1,13 @@
 TokenTree {
     source text: "var a = 1;\na = 2;\nprint a;\nprint a = 3;\nvar b = a;\nvar c;\nc = b;\nprint c;",
+    span: Span {
+        start: Offset(
+            0,
+        ),
+        end: Offset(
+            72,
+        ),
+    },
     tokens: [
         Alphabetic(var),
         Whitespace(' '),

lox_tests/boolean/token

@@ -1,5 +1,13 @@
 TokenTree {
     source text: "true;\nfalse;\n!true;\n!false;",
+    span: Span {
+        start: Offset(
+            0,
+        ),
+        end: Offset(
+            26,
+        ),
+    },
     tokens: [
         Alphabetic(true),
         Semicolon,