wip: lexer + parser, basic ast

Cargo.toml

@@ -9,6 +9,10 @@ exclude = ["default.nix", "flake.nix", "shell.nix", "example/*"]
 license = "GPL-2.0"
 edition = "2021"
 
+[workspace]
+
+members = ["matugen-parser"]
+
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [target.'cfg(windows)'.dependencies]
@@ -32,7 +36,9 @@ harness = false
 [dependencies]
 # logging
 log = "0.4.17"
-paris-log = { version = "1.0.2", features = ["icons"] } # TODO replace with using paris directly?
+paris-log = { version = "1.0.2", features = [
+  "icons",
+] } # TODO replace with using paris directly?
 pretty_env_logger = "0.5.0"
 
 # terminal output
@@ -42,13 +48,19 @@ colorsys = { default-features = false, version = "0.6.7" }
 owo-colors = "4.0.0"
 
 # templating engine
-upon = { features = ["filters", "serde"], default-features = false, version = "0.8.0" }
+upon = { features = [
+  "filters",
+  "serde",
+], default-features = false, version = "0.8.0" }
 
 # config
 serde = { version = "1.0.160", features = ["serde_derive", "std"], default-features = false }
 toml = { features = ["parse"], default-features = false, version = "0.8.8" }
 
-clap = { version = "4.2.4", features = ["derive", "std"], default-features = false }
+clap = { version = "4.2.4", features = [
+  "derive",
+  "std",
+], default-features = false }
 material-colors = { version = "0.4.0", features = ["image"] }
 image = { default-features = false, version = "0.25.2" }
 directories = "5.0"

matugen-parser/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "matugen-parser"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+nom = "7.1.3"
+serde = "1.0.209"
+string_cache = "0.8.7"

matugen-parser/src/lexer.rs

@@ -0,0 +1,135 @@
+use std::str::Chars;
+use string_cache::DefaultAtom as Atom;
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum Kind {
+    String,
+    Number,
+    Lbracket,
+    RBracket,
+    Dot,
+    Bar,
+    Space,
+    Colon,
+    Sof,
+    Eof,
+    Identifier,
+}
+
+#[derive(Debug, Clone)]
+pub struct Token {
+    pub kind: Kind,
+    pub start: usize,
+    pub end: usize,
+    pub value: TokenValue,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum TokenValue {
+    None,
+    Number(i32),
+    String(Atom),
+}
+
+#[derive(Debug)]
+pub struct Lexer<'a> {
+    source: &'a str,
+    chars: Chars<'a>,
+}
+
+impl<'a> Lexer<'a> {
+    pub fn new(input: &'a str) -> Self {
+        Lexer {
+            source: &input,
+            chars: input.chars(),
+        }
+    }
+
+    fn offset(&self) -> usize {
+        self.source.len() - self.chars.as_str().len()
+    }
+
+    // pub fn tokenize(&mut self) -> Vec<Token> {
+    //     let mut tokens = Vec::new();
+    //     while let token = self.next_token() {
+    //         if token.kind == Kind::Space {
+    //             continue;
+    //         }
+    //         tokens.push(token);
+    //     }
+    //     tokens.push(Token {
+    //         kind: Kind::Eof,
+    //         start: self.source.len(),
+    //         end: self.source.len(),
+    //         value: TokenValue::None,
+    //     });
+    //     tokens
+    // }
+
+    pub fn peek(&self) -> Option<char> {
+        self.chars.clone().next()
+    }
+
+    pub fn next_token(&mut self) -> Token {
+        let start = self.offset();
+        let res = self.read_next_kind();
+        let end = self.offset();
+
+        Token {
+            kind: res.0,
+            start,
+            end,
+            value: res.1,
+        }
+    }
+
+    fn read_next_kind(&mut self) -> (Kind, TokenValue) {
+        let next_char = self.chars.next();
+
+        if next_char.is_none() {
+            return (Kind::Eof, TokenValue::None);
+        }
+
+        match next_char.unwrap() {
+            '{' => (Kind::Lbracket, TokenValue::None),
+            '}' => (Kind::RBracket, TokenValue::None),
+            '|' => (Kind::Bar, TokenValue::None),
+            '.' => (Kind::Dot, TokenValue::None),
+            ' ' => (Kind::Space, TokenValue::None),
+            ':' => (Kind::Colon, TokenValue::None),
+            '0'..='9' => {
+                let mut number = next_char.unwrap().to_digit(10).unwrap() as i32;
+                while let Some(next_char) = self.peek() {
+                    if let Some(digit) = next_char.to_digit(10) {
+                        number = number * 10 + digit as i32;
+                        self.chars.next();
+                    } else {
+                        break;
+                    }
+                }
+                (Kind::Number, TokenValue::Number(number))
+            }
+            _ => {
+                if !next_char.unwrap().is_alphanumeric() || next_char.unwrap() == '_' {
+                    (Kind::Identifier, TokenValue::None)
+                } else {
+                    let start = self.offset() - 1;
+                    let mut string = String::from(next_char.unwrap());
+                    while let Some(next_char) = self.peek() {
+                        if next_char.is_alphanumeric() || next_char == '_' {
+                            string.push(next_char);
+                            self.chars.next();
+                        } else {
+                            break;
+                        }
+                    }
+                    let end = self.offset();
+                    (
+                        Kind::String,
+                        TokenValue::String(self.source[start..end].to_string().into()),
+                    )
+                }
+            }
+        }
+    }
+}

matugen-parser/src/main.rs

@@ -0,0 +1,21 @@
+use lexer::Lexer;
+use parser::Parser;
+
+mod lexer;
+mod node;
+mod parser;
+
+fn main() {
+    // let source = ".}|aa {{    colors.source_color.default.hex }} {{ aaa }} {{ aaa }} {{ aaaa | set_alpha: 100 }} {{ colors..aaaaaa....a }}";
+    let source = ".}|aa {{    colors.source_color.default.hex }} {{ a1.aaa }} {{ a2 }}";
+
+    let mut parser = Parser::new(source);
+
+    // parser.get_keywords();
+
+    println!("----------------------\n");
+
+    let parsed = parser.parse();
+
+    println!("parsed: \n{:#?}", parsed);
+}

matugen-parser/src/node.rs

@@ -0,0 +1,34 @@
+use crate::lexer::{Token, TokenValue};
+
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
+pub struct Node {
+    /// Start offset in source
+    pub start: usize,
+
+    /// End offset in source
+    pub end: usize,
+}
+
+impl Node {
+    pub fn new(start: usize, end: usize) -> Self {
+        Self { start, end }
+    }
+}
+
+#[derive(Debug)]
+pub struct Program {
+    pub node: Node,
+    pub body: Vec<Statement>,
+}
+
+#[derive(Debug)]
+pub enum Statement {
+    KeywordDefinition(Box<KeywordDefinition>),
+}
+
+#[derive(Debug)]
+pub struct KeywordDefinition {
+    pub node: Node,
+    pub keywords: Vec<TokenValue>,
+    pub filters: Option<Vec<String>>,
+}