Token-related definitions

src/main.rs

@@ -1,10 +1,16 @@
-use std::{env, fs, io::{self, Write}, process::ExitCode};
+mod tokens;
+
+use std::{
+    env, fs,
+    io::{self, Write},
+    process::ExitCode,
+};
 
 /// Error handler. Can be used to print error messages; will also retain the
 /// current error status.
 #[derive(Default, Debug)]
 struct ErrorHandler {
-    had_error: bool
+    had_error: bool,
 }
 
 impl ErrorHandler {
@@ -45,7 +51,9 @@ fn run_prompt() {
     loop {
         print!("slox> ");
         stdout.flush().unwrap();
-        let n_read = stdin.read_line(&mut buffer).expect("Failed to read from stdin");
+        let n_read = stdin
+            .read_line(&mut buffer)
+            .expect("Failed to read from stdin");
         let _ = match n_read {
             0 => return,
             _ => run(buffer.clone()),

src/tokens.rs

@@ -0,0 +1,55 @@
+/// The type of a token. May also contain its litteral value.
+#[derive(Clone, PartialEq, Debug)]
+pub enum TokenType {
+    LeftParen,
+    RightParen,
+    LeftBrace,
+    RightBrace,
+    Comma,
+    Dot,
+    Minus,
+    Plus,
+    Semicolon,
+    Slash,
+    Star,
+
+    Bang,
+    BangEqual,
+    Equal,
+    EqualEqual,
+    Greater,
+    GreaterEqual,
+    Less,
+    LessEqual,
+
+    Identifier(String),
+    String(String),
+    Number(f64),
+
+    And,
+    Class,
+    Else,
+    False,
+    Fun,
+    For,
+    If,
+    Nil,
+    Or,
+    Print,
+    Return,
+    Super,
+    This,
+    True,
+    Var,
+    While,
+
+    EOF,
+}
+
+/// Full information about a token.
+#[derive(Clone, Debug)]
+pub struct Token {
+    pub token_type: TokenType,
+    pub lexeme: String,
+    pub line: usize,
+}