Quellcode durchsuchen

treat semicolon as newline

Nils Boettcher vor 2 Monaten
Ursprung
Commit
73691727c2
2 geänderte Dateien mit 16 neuen und 0 gelöschten Zeilen
  1. 5 0
      examples/numbat_syntax.nbt
  2. 11 0
      numbat/src/tokenizer.rs

+ 5 - 0
examples/numbat_syntax.nbt

@@ -1,6 +1,11 @@
 # This is a line comment. It can span over
 # multiple lines
 
+# statements can be separated by newlines or semicolons
+1
+2
+1;2
+
 # 1. Imports
 
 use prelude        # This is not necessary. The 'prelude'

+ 11 - 0
numbat/src/tokenizer.rs

@@ -639,6 +639,7 @@ impl Tokenizer {
                 return Ok(None);
             }
             '\n' => TokenKind::Newline,
+            ';' => TokenKind::Newline,
             '&' if self.match_char(input, '&') => TokenKind::LogicalAnd,
             '|' if self.match_char(input, '|') => TokenKind::LogicalOr,
             '|' if self.match_char(input, '>') => TokenKind::PostfixApply,
@@ -983,6 +984,16 @@ fn test_tokenize_basic() {
         ]
     );
 
+    assert_eq!(
+        tokenize_reduced("1;42").unwrap(),
+        [
+            ("1", Number, ByteIndex(0)),
+            (";", Newline, ByteIndex(1)),
+            ("42", Number, ByteIndex(2)),
+            ("", Eof, ByteIndex(4))
+        ]
+    );
+
     assert_eq!(
         tokenize_reduced("…").unwrap(),
         [