semicolon as seperator

numbat/src/parser.rs

@@ -2,6 +2,7 @@
 //!
 //! Grammar:
 //! ```txt
+//! prog            ::=   statement ((";" | "\n"+) statement)*
 //! statement       ::=   variable_decl | struct_decl | function_decl | dimension_decl | unit_decl | module_import | procedure_call | expression
 //!
 //! variable_decl   ::=   "let" identifier ( ":" type_annotation ) ? "=" expression
@@ -315,6 +316,9 @@ impl<'a> Parser<'a> {
                     // Skip over empty lines
                     self.skip_empty_lines(tokens);
                 }
+                TokenKind::Semicolon => {
+                    self.advance(tokens);
+                }
                 TokenKind::Eof => {
                     break;
                 }
@@ -359,7 +363,7 @@ impl<'a> Parser<'a> {
     /// Must be called after encountering an error.
     fn recover_from_error(&mut self, tokens: &[Token]) {
         // Skip all the tokens until we encounter a newline or EoF.
-        while !matches!(self.peek(tokens).kind, TokenKind::Newline | TokenKind::Eof) {
+        while !matches!(self.peek(tokens).kind, TokenKind::Newline | TokenKind::Semicolon | TokenKind::Eof) {
             self.advance(tokens)
         }
     }
@@ -1951,7 +1955,7 @@ impl<'a> Parser<'a> {
     fn look_ahead_beyond_linebreak(&self, tokens: &[Token], token_kind: TokenKind) -> bool {
         let mut i = self.current;
         while i < tokens.len() {
-            if tokens[i].kind != TokenKind::Newline {
+            if !matches!(tokens[i].kind, TokenKind::Newline | TokenKind::Semicolon) {
                 return tokens[i].kind == token_kind;
             }
             i += 1;
@@ -2004,7 +2008,7 @@ impl<'a> Parser<'a> {
     }
 
     pub fn is_end_of_statement(&self, tokens: &[Token]) -> bool {
-        self.peek(tokens).kind == TokenKind::Newline || self.is_at_end(tokens)
+        matches!( self.peek(tokens).kind, TokenKind::Newline | TokenKind::Semicolon | TokenKind::Eof)
     }
 
     pub fn is_at_end(&self, tokens: &[Token]) -> bool {

numbat/src/tokenizer.rs

@@ -159,6 +159,7 @@ pub enum TokenKind {
 
     // Other
     Newline,
+    Semicolon,
     Eof,
 }
 
@@ -639,7 +640,7 @@ impl Tokenizer {
                 return Ok(None);
             }
             '\n' => TokenKind::Newline,
-            ';' => TokenKind::Newline,
+            ';' => TokenKind::Semicolon,
             '&' if self.match_char(input, '&') => TokenKind::LogicalAnd,
             '|' if self.match_char(input, '|') => TokenKind::LogicalOr,
             '|' if self.match_char(input, '>') => TokenKind::PostfixApply,
@@ -988,7 +989,7 @@ fn test_tokenize_basic() {
         tokenize_reduced("1;42").unwrap(),
         [
             ("1", Number, ByteIndex(0)),
-            (";", Newline, ByteIndex(1)),
+            (";", Semicolon, ByteIndex(1)),
             ("42", Number, ByteIndex(2)),
             ("", Eof, ByteIndex(4))
         ]