Better parser errors using codespan_reporting

Cargo.lock

@@ -197,6 +197,16 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "codespan-reporting"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
+dependencies = [
+ "termcolor",
+ "unicode-width",
+]
+
 [[package]]
 name = "colorchoice"
 version = "1.0.0"
@@ -487,6 +497,7 @@ name = "numbat"
 version = "0.1.0"
 dependencies = [
  "approx",
+ "codespan-reporting",
  "itertools",
  "num-rational",
  "num-traits",
@@ -718,6 +729,15 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "termcolor"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"
+dependencies = [
+ "winapi-util",
+]
+
 [[package]]
 name = "terminal_size"
 version = "0.2.6"
@@ -809,6 +829,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
+[[package]]
+name = "winapi-util"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"

numbat-cli/src/main.rs

@@ -9,7 +9,7 @@ use highlighter::NumbatHighlighter;
 use numbat::markup;
 use numbat::pretty_print::PrettyPrint;
 use numbat::resolver::{CodeSource, FileSystemImporter};
-use numbat::{Context, ExitStatus, InterpreterResult, NumbatError, ParseError};
+use numbat::{Context, ExitStatus, InterpreterResult, NumbatError};
 
 use anyhow::{bail, Context as AnyhowContext, Result};
 use clap::Parser;
@@ -329,35 +329,18 @@ impl Cli {
                     InterpreterResult::Exit(exit_status) => ControlFlow::Break(exit_status),
                 }
             }
-            Err(NumbatError::ParseError {
-                inner: ref e @ ParseError { ref span, .. },
-                code_source,
-                line,
-            }) => {
-                let code_source_text = match code_source {
-                    CodeSource::Text => "<input>".to_string(),
-                    CodeSource::File(path) => format!("File {}", path.to_string_lossy()),
-                    CodeSource::Module(module_path, path) => format!(
-                        "Module '{module_path}', File {path}",
-                        module_path = itertools::join(module_path.0.iter(), "::"),
-                        path = path
-                            .map(|p| p.to_string_lossy().to_string())
-                            .unwrap_or("?".into()),
-                    ),
-                };
-                eprintln!(
-                    "{code_source_text}:{line_number}:{position}",
-                    line_number = span.line,
-                    position = span.position
-                );
-                eprintln!("    {line}");
-                eprintln!("    {offset}^", offset = " ".repeat(span.position - 1));
-                eprintln!("{}", e);
-
-                execution_mode.exit_status_in_case_of_error()
-            }
             Err(NumbatError::ResolverError(e)) => {
-                eprintln!("Module resolver error: {:#}", e);
+                match e {
+                    numbat::resolver::ResolverError::UnknownModule(_) => {
+                        eprintln!("Module resolver error: {:#}", e);
+                    }
+                    numbat::resolver::ResolverError::ParseError {
+                        inner: _,
+                        diagnostic,
+                    } => {
+                        self.context.lock().unwrap().print_diagnostic(&diagnostic);
+                    }
+                }
                 execution_mode.exit_status_in_case_of_error()
             }
             Err(NumbatError::NameResolutionError(e)) => {

numbat/Cargo.toml

@@ -10,6 +10,7 @@ thiserror = "1"
 itertools = "0.10"
 num-rational = "0.4"
 num-traits = "0.2"
+codespan-reporting = "0.11"
 
 [dev-dependencies]
 approx = "0.5"

numbat/src/lib.rs

@@ -26,6 +26,7 @@ mod unit_registry;
 mod vm;
 
 use bytecode_interpreter::BytecodeInterpreter;
+use codespan_reporting::diagnostic::Diagnostic;
 use interpreter::{Interpreter, RuntimeError};
 use name_resolution::NameResolutionError;
 use prefix_transformer::Transformer;
@@ -44,12 +45,6 @@ pub use parser::ParseError;
 
 #[derive(Debug, Error)]
 pub enum NumbatError {
-    #[error("{inner}")]
-    ParseError {
-        inner: ParseError,
-        code_source: CodeSource,
-        line: String,
-    },
     #[error("{0}")]
     ResolverError(ResolverError),
     #[error("{0}")]
@@ -66,7 +61,7 @@ pub struct Context {
     prefix_transformer: Transformer,
     typechecker: TypeChecker,
     interpreter: BytecodeInterpreter,
-    module_importer: Box<dyn ModuleImporter>,
+    resolver: Resolver,
 }
 
 impl Context {
@@ -75,7 +70,7 @@ impl Context {
             prefix_transformer: Transformer::new(),
             typechecker: TypeChecker::default(),
             interpreter: BytecodeInterpreter::new(),
-            module_importer: Box::new(module_importer),
+            resolver: Resolver::new(module_importer),
         }
     }
 
@@ -108,20 +103,10 @@ impl Context {
         code: &str,
         code_source: CodeSource,
     ) -> Result<(Vec<Statement>, InterpreterResult)> {
-        let resolver = Resolver::new(self.module_importer.as_ref());
-
-        let statements = resolver.resolve(code, code_source).map_err(|e| match e {
-            ResolverError::ParseError {
-                inner,
-                code_source,
-                line,
-            } => NumbatError::ParseError {
-                inner,
-                code_source,
-                line,
-            },
-            e => NumbatError::ResolverError(e),
-        })?;
+        let statements = self
+            .resolver
+            .resolve(code, code_source)
+            .map_err(NumbatError::ResolverError)?;
 
         let prefix_transformer_old = self.prefix_transformer.clone();
 
@@ -157,4 +142,23 @@ impl Context {
 
         Ok((transformed_statements, result))
     }
+
+    pub fn print_diagnostic(&self, diagnostic: &Diagnostic<usize>) {
+        use codespan_reporting::term::{
+            self,
+            termcolor::{ColorChoice, StandardStream},
+            Config,
+        };
+
+        let writer = StandardStream::stderr(ColorChoice::Always);
+        let config = Config::default();
+
+        term::emit(
+            &mut writer.lock(),
+            &config,
+            &self.resolver.files,
+            &diagnostic,
+        )
+        .unwrap();
+    }
 }

numbat/src/parser.rs

@@ -120,9 +120,9 @@ pub enum ParseErrorKind {
 }
 
 #[derive(Debug, Error)]
-#[error("Parse error in {span}: {kind}")]
+#[error("{kind}")]
 pub struct ParseError {
-    kind: ParseErrorKind,
+    pub kind: ParseErrorKind,
     pub span: Span,
 }
 
@@ -218,7 +218,7 @@ impl<'a> Parser<'a> {
         if self.match_exact(TokenKind::RightParen).is_none() {
             return Err(ParseError::new(
                 ParseErrorKind::MissingClosingParen,
-                self.next().span.clone(),
+                self.peek().span.clone(),
             ));
         }
 
@@ -676,7 +676,7 @@ impl<'a> Parser<'a> {
         if self.match_exact(TokenKind::RightParen).is_none() {
             return Err(ParseError::new(
                 ParseErrorKind::MissingClosingParen,
-                self.next().span.clone(),
+                self.peek().span.clone(),
             ));
         }
 
@@ -710,7 +710,7 @@ impl<'a> Parser<'a> {
             if self.match_exact(TokenKind::RightParen).is_none() {
                 return Err(ParseError::new(
                     ParseErrorKind::MissingClosingParen,
-                    self.next().span.clone(),
+                    self.peek().span.clone(),
                 ));
             }
 
@@ -827,7 +827,10 @@ impl<'a> Parser<'a> {
         } else if self.match_exact(TokenKind::LeftParen).is_some() {
             let dexpr = self.dimension_expression()?;
             if self.match_exact(TokenKind::RightParen).is_none() {
-                todo!("Parse error: expected ')'");
+                return Err(ParseError::new(
+                    ParseErrorKind::MissingClosingParen,
+                    self.peek().span.clone(),
+                ));
             }
             Ok(dexpr)
         } else {
@@ -868,14 +871,6 @@ impl<'a> Parser<'a> {
         self.tokens.get(self.current - 1)
     }
 
-    fn next(&self) -> &'a Token {
-        if self.is_at_end() {
-            self.peek()
-        } else {
-            &self.tokens[self.current + 1]
-        }
-    }
-
     pub fn is_at_end(&self) -> bool {
         self.peek().kind == TokenKind::Eof
     }

numbat/src/resolver.rs

@@ -5,6 +5,8 @@ use std::{
 
 use crate::{ast::Statement, parser::parse, ParseError};
 
+use codespan_reporting::diagnostic::{Diagnostic, Label};
+use codespan_reporting::files::SimpleFiles;
 use thiserror::Error;
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -36,34 +38,61 @@ pub enum ResolverError {
     #[error("{inner}")]
     ParseError {
         inner: ParseError,
-        code_source: CodeSource,
-        line: String,
+        diagnostic: Diagnostic<usize>,
     },
 }
 
 type Result<T> = std::result::Result<T, ResolverError>;
 
-pub(crate) struct Resolver<'a> {
-    importer: &'a dyn ModuleImporter,
+pub(crate) struct Resolver {
+    importer: Box<dyn ModuleImporter>,
+    code_sources: Vec<CodeSource>,
+    pub files: SimpleFiles<String, String>,
 }
 
-impl<'a> Resolver<'a> {
-    pub(crate) fn new(importer: &'a dyn ModuleImporter) -> Self {
-        Self { importer }
+impl Resolver {
+    pub(crate) fn new(importer: impl ModuleImporter + 'static) -> Self {
+        Self {
+            importer: Box::new(importer),
+            code_sources: vec![],
+            files: SimpleFiles::new(),
+        }
+    }
+
+    fn add_code_source(&mut self, code_source: CodeSource, content: &str) -> usize {
+        let code_source_text = match &code_source {
+            CodeSource::Text => "<input>".to_string(),
+            CodeSource::File(path) => format!("File {}", path.to_string_lossy()),
+            CodeSource::Module(module_path, path) => format!(
+                "Module '{module_path}', File {path}",
+                module_path = itertools::join(module_path.0.iter(), "::"),
+                path = path
+                    .as_ref()
+                    .map(|p| p.to_string_lossy().to_string())
+                    .unwrap_or("?".into()),
+            ),
+        };
+
+        let _file_id = self.files.add(code_source_text, content.to_string()); // TODO: it's a shame we need to clone the full source code here
+
+        self.code_sources.push(code_source);
+        self.code_sources.len() - 1
     }
 
-    fn parse(&self, code: &str, code_source: CodeSource) -> Result<Vec<Statement>> {
+    fn parse(&self, code: &str, code_source_index: usize) -> Result<Vec<Statement>> {
         parse(code).map_err(|inner| {
-            let line = code.lines().nth(inner.span.line - 1).unwrap().to_string();
-            ResolverError::ParseError {
-                inner,
-                code_source,
-                line,
-            }
+            let diagnostic = Diagnostic::error()
+                .with_message("Parse error")
+                .with_labels(vec![Label::primary(
+                    code_source_index,
+                    (inner.span.position.byte)..(inner.span.position.byte + 1),
+                )
+                .with_message(inner.kind.to_string())]);
+            ResolverError::ParseError { inner, diagnostic }
         })
     }
 
-    fn inlining_pass(&self, program: &[Statement]) -> Result<(Vec<Statement>, bool)> {
+    fn inlining_pass(&mut self, program: &[Statement]) -> Result<(Vec<Statement>, bool)> {
         let mut new_program = vec![];
         let mut performed_imports = false;
 
@@ -71,10 +100,11 @@ impl<'a> Resolver<'a> {
             match statement {
                 Statement::ModuleImport(module_path) => {
                     if let Some((code, filesystem_path)) = self.importer.import(module_path) {
-                        for statement in self.parse(
-                            &code,
+                        let index = self.add_code_source(
                             CodeSource::Module(module_path.clone(), filesystem_path),
-                        )? {
+                            &code,
+                        );
+                        for statement in self.parse(&code, index)? {
                             new_program.push(statement);
                         }
                         performed_imports = true;
@@ -89,10 +119,11 @@ impl<'a> Resolver<'a> {
         Ok((new_program, performed_imports))
     }
 
-    pub fn resolve(&self, code: &str, code_source: CodeSource) -> Result<Vec<Statement>> {
+    pub fn resolve(&mut self, code: &str, code_source: CodeSource) -> Result<Vec<Statement>> {
         // TODO: handle cyclic dependencies & infinite loops
 
-        let mut statements = self.parse(code, code_source)?;
+        let index = self.add_code_source(code_source, &code);
+        let mut statements = self.parse(code, index)?;
 
         loop {
             let result = self.inlining_pass(&statements)?;
@@ -185,7 +216,7 @@ mod tests {
 
         let importer = TestImporter {};
 
-        let resolver = Resolver::new(&importer);
+        let mut resolver = Resolver::new(importer);
         let program_inlined = resolver.resolve(program, CodeSource::Text).unwrap();
 
         assert_eq!(

numbat/src/span.rs

@@ -1,9 +1,29 @@
-use thiserror::Error;
-
-#[derive(Debug, Error, Clone, PartialEq, Eq)]
-#[error("line {line}, position {position}")]
-pub struct Span {
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SourceCodePositition {
+    pub byte: usize,
+    pub index: usize,
     pub line: usize,
     pub position: usize,
-    pub index: usize,
+}
+
+impl SourceCodePositition {
+    pub fn start() -> Self {
+        Self {
+            byte: 0,
+            index: 0,
+            line: 1,
+            position: 1,
+        }
+    }
+
+    pub fn to_single_character_span(&self) -> Span {
+        Span {
+            position: self.clone(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Span {
+    pub position: SourceCodePositition,
 }

numbat/src/tokenizer.rs

@@ -1,4 +1,4 @@
-use crate::span::Span;
+use crate::span::{SourceCodePositition, Span};
 
 use std::collections::HashMap;
 use std::sync::OnceLock;
@@ -15,9 +15,6 @@ pub enum TokenizerErrorKind {
     #[error("Unexpected character in number literal: '{0}'")]
     UnexpectedCharacterInNumberLiteral(char),
 
-    #[error("Unexpected character in scientific notation: '{0}'")]
-    UnexpectedCharacterInScientificNotation(char),
-
     #[error("Unexpected character in identifier: '{0}'")]
     UnexpectedCharacterInIdentifier(char),
 
@@ -105,13 +102,8 @@ pub struct Token {
 
 struct Tokenizer {
     input: Vec<char>,
-
-    token_start_index: usize,
-    token_start_position: usize,
-
-    current_index: usize,
-    current_line: usize,
-    current_position: usize,
+    current: SourceCodePositition,
+    token_start: SourceCodePositition,
 }
 
 fn is_exponent_char(c: char) -> bool {
@@ -133,19 +125,15 @@ impl Tokenizer {
     fn new(input: &str) -> Self {
         Tokenizer {
             input: input.chars().collect(),
-            token_start_index: 0,
-            token_start_position: 0,
-            current_index: 0,
-            current_position: 1,
-            current_line: 1,
+            current: SourceCodePositition::start(),
+            token_start: SourceCodePositition::start(),
         }
     }
 
     fn scan(&mut self) -> Result<Vec<Token>> {
         let mut tokens = vec![];
         while !self.at_end() {
-            self.token_start_index = self.current_index;
-            self.token_start_position = self.current_position;
+            self.token_start = self.current.clone();
             if let Some(token) = self.scan_single_token()? {
                 tokens.push(token);
             }
@@ -154,11 +142,7 @@ impl Tokenizer {
         tokens.push(Token {
             kind: TokenKind::Eof,
             lexeme: "".into(),
-            span: Span {
-                line: self.current_line,
-                position: self.current_position,
-                index: self.current_index,
-            },
+            span: self.current.to_single_character_span(),
         });
 
         Ok(tokens)
@@ -174,11 +158,7 @@ impl Tokenizer {
                 kind: TokenizerErrorKind::ExpectedDigit {
                     character: self.peek(),
                 },
-                span: Span {
-                    line: self.current_line,
-                    position: self.current_position,
-                    index: self.token_start_index,
-                },
+                span: self.current.to_single_character_span(),
             });
         }
 
@@ -194,11 +174,7 @@ impl Tokenizer {
         {
             return Err(TokenizerError {
                 kind: TokenizerErrorKind::UnexpectedCharacterInNumberLiteral(self.peek().unwrap()),
-                span: Span {
-                    line: self.current_line,
-                    position: self.current_position,
-                    index: self.current_index,
-                },
+                span: self.current.to_single_character_span(),
             });
         }
 
@@ -215,23 +191,6 @@ impl Tokenizer {
             let _ = self.match_char('+') || self.match_char('-');
 
             self.consume_stream_of_digits(true, true)?;
-
-            if self
-                .peek()
-                .map(|c| c.is_ascii_digit() || c == '.')
-                .unwrap_or(false)
-            {
-                return Err(TokenizerError {
-                    kind: TokenizerErrorKind::UnexpectedCharacterInScientificNotation(
-                        self.peek().unwrap(),
-                    ),
-                    span: Span {
-                        line: self.current_line,
-                        position: self.current_position,
-                        index: self.current_index,
-                    },
-                });
-            }
         }
 
         Ok(())
@@ -272,14 +231,10 @@ impl Tokenizer {
 
         let current_char = self.advance();
 
-        let tokenizer_error = |line, position, index, kind| -> Result<Option<Token>> {
+        let tokenizer_error = |position: &SourceCodePositition, kind| -> Result<Option<Token>> {
             Err(TokenizerError {
                 kind,
-                span: Span {
-                    line,
-                    position,
-                    index,
-                },
+                span: position.to_single_character_span(),
             })
         };
 
@@ -311,9 +266,7 @@ impl Tokenizer {
 
                 if !has_advanced || self.peek().map(is_identifier_char).unwrap_or(false) {
                     return tokenizer_error(
-                        self.current_line,
-                        self.current_position,
-                        self.token_start_index,
+                        &self.current,
                         TokenizerErrorKind::ExpectedDigitInBase {
                             base,
                             character: self.peek(),
@@ -368,9 +321,7 @@ impl Tokenizer {
                     TokenKind::UnicodeExponent
                 } else {
                     return tokenizer_error(
-                        self.current_line,
-                        self.current_position,
-                        self.current_index,
+                        &self.current,
                         TokenizerErrorKind::UnexpectedCharacterInNegativeExponent { character: c },
                     );
                 }
@@ -386,9 +337,7 @@ impl Tokenizer {
                     TokenKind::String
                 } else {
                     return tokenizer_error(
-                        self.current_line,
-                        self.current_position,
-                        self.current_index,
+                        &self.token_start,
                         TokenizerErrorKind::UnterminatedString,
                     );
                 }
@@ -401,9 +350,7 @@ impl Tokenizer {
 
                 if self.peek().map(|c| c == '.').unwrap_or(false) {
                     return tokenizer_error(
-                        self.current_line,
-                        self.current_position,
-                        self.current_index,
+                        &self.current,
                         TokenizerErrorKind::UnexpectedCharacterInIdentifier(self.peek().unwrap()),
                     );
                 }
@@ -416,9 +363,7 @@ impl Tokenizer {
             }
             c => {
                 return tokenizer_error(
-                    self.current_line,
-                    self.current_position - 1,
-                    self.current_index - 1,
+                    &self.token_start,
                     TokenizerErrorKind::UnexpectedCharacter { character: c },
                 );
             }
@@ -427,40 +372,37 @@ impl Tokenizer {
         let token = Some(Token {
             kind,
             lexeme: self.lexeme(),
-            span: Span {
-                line: self.current_line,
-                position: self.token_start_position,
-                index: self.token_start_index,
-            },
+            span: self.token_start.to_single_character_span(),
         });
 
         if kind == TokenKind::Newline {
-            self.current_line += 1;
-            self.current_position = 1;
+            self.current.line += 1;
+            self.current.position = 1;
         }
 
         Ok(token)
     }
 
     fn lexeme(&self) -> String {
-        self.input[self.token_start_index..self.current_index]
+        self.input[self.token_start.index..self.current.index]
             .iter()
             .collect()
     }
 
     fn advance(&mut self) -> char {
-        let c = self.input[self.current_index];
-        self.current_index += 1;
-        self.current_position += 1;
+        let c = self.input[self.current.index];
+        self.current.index += 1;
+        self.current.byte += c.len_utf8();
+        self.current.position += 1;
         c
     }
 
     fn peek(&self) -> Option<char> {
-        self.input.get(self.current_index).copied()
+        self.input.get(self.current.index).copied()
     }
 
     fn peek2(&self) -> Option<char> {
-        self.input.get(self.current_index + 1).copied()
+        self.input.get(self.current.index + 1).copied()
     }
 
     fn match_char(&mut self, c: char) -> bool {
@@ -473,7 +415,7 @@ impl Tokenizer {
     }
 
     fn at_end(&self) -> bool {
-        self.current_index >= self.input.len()
+        self.current.index >= self.input.len()
     }
 }
 
@@ -483,17 +425,16 @@ pub fn tokenize(input: &str) -> Result<Vec<Token>> {
 }
 
 #[cfg(test)]
-fn token_stream(input: &[(&str, TokenKind, (usize, usize, usize))]) -> Vec<Token> {
-    input
+fn tokenize_reduced(input: &str) -> Vec<(String, TokenKind, (usize, usize))> {
+    tokenize(input)
+        .unwrap()
         .iter()
-        .map(|(lexeme, kind, (line, position, index))| Token {
-            kind: *kind,
-            lexeme: lexeme.to_string(),
-            span: Span {
-                line: *line,
-                position: *position,
-                index: *index,
-            },
+        .map(|token| {
+            (
+                token.lexeme.to_string(),
+                token.kind,
+                (token.span.position.line, token.span.position.position),
+            )
         })
         .collect()
 }
@@ -503,82 +444,85 @@ fn test_tokenize_basic() {
     use TokenKind::*;
 
     assert_eq!(
-        tokenize("  12 + 34  ").unwrap(),
-        token_stream(&[
-            ("12", Number, (1, 3, 2)),
-            ("+", Plus, (1, 6, 5)),
-            ("34", Number, (1, 8, 7)),
-            ("", Eof, (1, 12, 11))
-        ])
+        tokenize_reduced("  12 + 34  "),
+        [
+            ("12".to_string(), Number, (1, 3)),
+            ("+".to_string(), Plus, (1, 6)),
+            ("34".to_string(), Number, (1, 8)),
+            ("".to_string(), Eof, (1, 12))
+        ]
     );
 
     assert_eq!(
-        tokenize("1 2").unwrap(),
-        token_stream(&[
-            ("1", Number, (1, 1, 0)),
-            ("2", Number, (1, 3, 2)),
-            ("", Eof, (1, 4, 3))
-        ])
+        tokenize_reduced("1 2"),
+        [
+            ("1".to_string(), Number, (1, 1)),
+            ("2".to_string(), Number, (1, 3)),
+            ("".to_string(), Eof, (1, 4))
+        ]
     );
 
     assert_eq!(
-        tokenize("12 × (3 - 4)").unwrap(),
-        token_stream(&[
-            ("12", Number, (1, 1, 0)),
-            ("×", Multiply, (1, 4, 3)),
-            ("(", LeftParen, (1, 6, 5)),
-            ("3", Number, (1, 7, 6)),
-            ("-", Minus, (1, 9, 8)),
-            ("4", Number, (1, 11, 10)),
-            (")", RightParen, (1, 12, 11)),
-            ("", Eof, (1, 13, 12))
-        ])
+        tokenize_reduced("12 × (3 - 4)"),
+        [
+            ("12".to_string(), Number, (1, 1)),
+            ("×".to_string(), Multiply, (1, 4)),
+            ("(".to_string(), LeftParen, (1, 6)),
+            ("3".to_string(), Number, (1, 7)),
+            ("-".to_string(), Minus, (1, 9)),
+            ("4".to_string(), Number, (1, 11)),
+            (")".to_string(), RightParen, (1, 12)),
+            ("".to_string(), Eof, (1, 13))
+        ]
     );
 
     assert_eq!(
-        tokenize("foo to bar").unwrap(),
-        token_stream(&[
-            ("foo", Identifier, (1, 1, 0)),
-            ("to", Arrow, (1, 5, 4)),
-            ("bar", Identifier, (1, 8, 7)),
-            ("", Eof, (1, 11, 10))
-        ])
+        tokenize_reduced("foo to bar"),
+        [
+            ("foo".to_string(), Identifier, (1, 1)),
+            ("to".to_string(), Arrow, (1, 5)),
+            ("bar".to_string(), Identifier, (1, 8)),
+            ("".to_string(), Eof, (1, 11))
+        ]
     );
 
     assert_eq!(
-        tokenize("1 -> 2").unwrap(),
-        token_stream(&[
-            ("1", Number, (1, 1, 0)),
-            ("->", Arrow, (1, 3, 2)),
-            ("2", Number, (1, 6, 5)),
-            ("", Eof, (1, 7, 6))
-        ])
+        tokenize_reduced("1 -> 2"),
+        [
+            ("1".to_string(), Number, (1, 1)),
+            ("->".to_string(), Arrow, (1, 3)),
+            ("2".to_string(), Number, (1, 6)),
+            ("".to_string(), Eof, (1, 7))
+        ]
     );
 
     assert_eq!(
-        tokenize("45°").unwrap(),
-        token_stream(&[
-            ("45", Number, (1, 1, 0)),
-            ("°", Identifier, (1, 3, 2)),
-            ("", Eof, (1, 4, 3))
-        ])
+        tokenize_reduced("45°"),
+        [
+            ("45".to_string(), Number, (1, 1)),
+            ("°".to_string(), Identifier, (1, 3)),
+            ("".to_string(), Eof, (1, 4))
+        ]
     );
 
     assert_eq!(
-        tokenize("1+2\n42").unwrap(),
-        token_stream(&[
-            ("1", Number, (1, 1, 0)),
-            ("+", Plus, (1, 2, 1)),
-            ("2", Number, (1, 3, 2)),
-            ("\n", Newline, (1, 4, 3)),
-            ("42", Number, (2, 1, 4)),
-            ("", Eof, (2, 3, 6))
-        ])
+        tokenize_reduced("1+2\n42"),
+        [
+            ("1".to_string(), Number, (1, 1)),
+            ("+".to_string(), Plus, (1, 2)),
+            ("2".to_string(), Number, (1, 3)),
+            ("\n".to_string(), Newline, (1, 4)),
+            ("42".to_string(), Number, (2, 1)),
+            ("".to_string(), Eof, (2, 3))
+        ]
     );
 
     assert_eq!(
-        tokenize("\"foo\"").unwrap(),
-        token_stream(&[("\"foo\"", String, (1, 1, 0)), ("", Eof, (1, 6, 5))])
+        tokenize_reduced("\"foo\""),
+        [
+            ("\"foo\"".to_string(), String, (1, 1)),
+            ("".to_string(), Eof, (1, 6))
+        ]
     );
 
     assert!(tokenize("~").is_err());

numbat/tests/prelude_and_examples.rs

@@ -2,7 +2,7 @@ mod common;
 
 use common::get_test_context;
 
-use numbat::resolver::CodeSource;
+use numbat::resolver::{CodeSource, ResolverError};
 use numbat::{InterpreterResult, NumbatError};
 
 use std::ffi::OsStr;
@@ -20,7 +20,7 @@ fn assert_typechecks_and_runs(code: &str) {
 fn assert_parse_error(code: &str) {
     assert!(matches!(
         get_test_context().interpret(code, CodeSource::Text),
-        Err(NumbatError::ParseError { .. })
+        Err(NumbatError::ResolverError(ResolverError::ParseError { .. }))
     ));
 }