fix: refactor

Author: psteinroe

Cargo.toml

@@ -3,3 +3,8 @@ members = [
     "crates/*",
 ]
 
+[workspace.dependencies]
+parser = { path = "./crates/parser", version = "0.0.0" }
+triomphe = { version = "0.1.8", default-features = false, features = ["std"] }
+
+

crates/parser/Cargo.toml

@@ -1,10 +1,13 @@
 [package]
 name = "parser"
-version = "0.1.0"
+version = "0.0.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+cstree = { version = "0.12.0", features = ["derive"] }
 pg_query = "0.7"
 logos = "0.13.0"
+serde_json = "1.0"
+serde = { version = "1.0", features = ["derive"] }

crates/parser/src/ast_node.rs

@@ -0,0 +1,10 @@
+use cstree::text::TextRange;
+use pg_query::NodeEnum;
+
+// TODO: implement serde for node: https://serde.rs/remote-derive.html
+
+#[derive(Debug)]
+pub struct RawStmt {
+    pub stmt: NodeEnum,
+    pub range: TextRange,
+}

crates/parser/src/event_buffer.rs

@@ -54,12 +54,16 @@
 // so we will have a `parse_statement` and a `parse_source_file` function
 // the tree always covers all text since we use the scantokens and, if failing, the StatementTokens
 // errors are added to a list, and are not part of the tree
-//
-mod event_buffer;
-mod event_sink;
+
+mod ast_node;
+mod parser;
 mod pg_query_utils;
-mod source_file_lexer;
-mod source_file_parsing;
-mod statement_lexer;
-mod statement_parsing;
+mod source_file;
+mod statement;
+mod syntax_error;
 mod syntax_kind;
+mod syntax_node;
+
+pub use crate::parser::{Parse, Parser};
+pub use crate::syntax_kind::SyntaxKind;
+pub use crate::syntax_node::{SyntaxElement, SyntaxNode, SyntaxToken};

crates/parser/src/event_sink.rs

@@ -1,26 +1,33 @@
-use cstree::testing::GreenNodeBuilder;
+use cstree::syntax::ResolvedNode;
+use cstree::{build::GreenNodeBuilder, text::TextRange};
+use pg_query::NodeEnum;
 
+use crate::ast_node::RawStmt;
+use crate::syntax_error::SyntaxError;
 use crate::syntax_kind::{SyntaxKind, SyntaxKindType};
+use crate::syntax_node::SyntaxNode;
 
-pub struct StatementBuilder<'builder> {
-    builder: &'builder mut GreenNodeBuilder<'static, 'static, SyntaxKind>,
+#[derive(Default, Debug)]
+pub struct Parser {
+    inner: GreenNodeBuilder<'static, 'static, SyntaxKind>,
     token_buffer: Vec<(SyntaxKind, String)>,
     curr_depth: i32,
+    errors: Vec<SyntaxError>,
+    stmts: Vec<RawStmt>,
 }
 
-/// Wrapper around GreenNodeBuilder to simplify integration with SyntaxKind
-impl<'builder> StatementBuilder<'builder> {
-    pub fn new(builder: &'builder mut GreenNodeBuilder<'static, 'static, SyntaxKind>) -> Self {
-        return Self {
-            builder,
-            token_buffer: Vec::new(),
-            curr_depth: 0,
-        };
-    }
+#[derive(Debug)]
+pub struct Parse {
+    pub cst: ResolvedNode<SyntaxKind>,
+    pub errors: Vec<SyntaxError>,
+    pub stmts: Vec<RawStmt>,
+}
 
+/// Main parser that controls the cst building process, and collects errors and statements
+impl Parser {
     pub fn close_until_depth(&mut self, depth: i32) {
         while self.curr_depth >= depth {
-            self.builder.finish_node();
+            self.inner.finish_node();
             self.curr_depth -= 1;
         }
     }
@@ -35,17 +42,17 @@ impl<'builder> StatementBuilder<'builder> {
         self.consume_token_buffer();
 
         self.curr_depth = *depth;
-        self.builder.start_node(kind);
+        self.inner.start_node(kind);
     }
 
     pub fn finish_node(&mut self) {
-        self.builder.finish_node();
+        self.inner.finish_node();
     }
 
     /// Drains the token buffer and applies all tokens
     pub fn consume_token_buffer(&mut self) {
         for (kind, text) in self.token_buffer.drain(..) {
-            self.builder.token(kind, &text);
+            self.inner.token(kind, &text);
         }
     }
 
@@ -60,15 +67,32 @@ impl<'builder> StatementBuilder<'builder> {
                 // move up to depth 2 and consume buffered tokens before applying closing token
                 self.close_until_depth(2);
                 self.consume_token_buffer();
-                self.builder.token(kind, text);
+                self.inner.token(kind, text);
             }
             Some(SyntaxKindType::Follow) => {
                 // wait until next node, and apply token at same depth
                 self.token_buffer.push((kind, text.to_string()));
             }
             _ => {
-                self.builder.token(kind, text);
+                self.inner.token(kind, text);
             }
         }
     }
+
+    pub fn error(&mut self, error: String, range: TextRange) {
+        self.errors.push(SyntaxError::new(error, range));
+    }
+
+    pub fn stmt(&mut self, stmt: NodeEnum, range: TextRange) {
+        self.stmts.push(RawStmt { stmt, range });
+    }
+
+    pub fn finish(self) -> Parse {
+        let (tree, cache) = self.inner.finish();
+        Parse {
+            cst: SyntaxNode::new_root_with_resolver(tree, cache.unwrap().into_interner().unwrap()),
+            stmts: self.stmts,
+            errors: self.errors,
+        }
+    }
 }

crates/parser/src/lib.rs

@@ -7,6 +7,8 @@
 /// This lexer does the split.
 use logos::Logos;
 
+use crate::{parser::Parser, syntax_kind::SyntaxKind};
+
 #[derive(Logos, Debug, PartialEq)]
 #[logos(skip r"[ \t\f]+")] // Ignore this regex pattern between tokens
 pub enum SourceFileToken {
@@ -20,12 +22,39 @@ pub enum SourceFileToken {
     Comment,
 }
 
+impl Parser {
+    pub fn parse_source_file(&mut self, text: &str) {
+        let mut lexer = SourceFileToken::lexer(text);
+
+        self.start_node(SyntaxKind::SourceFile, &0);
+        while let Some(token) = lexer.next() {
+            match token {
+                Ok(token) => {
+                    match token {
+                        SourceFileToken::Comment => {
+                            self.token(SyntaxKind::Comment, lexer.slice());
+                        }
+                        SourceFileToken::Newline => {
+                            self.token(SyntaxKind::Newline, lexer.slice());
+                        }
+                        SourceFileToken::Statement => {
+                            self.parse_statement(lexer.slice(), Some(lexer.span().start as u32));
+                        }
+                    };
+                }
+                Err(_) => panic!("Unknown SourceFileToken: {:?}", lexer.span()),
+            }
+        }
+        self.finish_node();
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
     #[test]
-    fn test_expr_lexer() {
+    fn test_source_file_lexer() {
         let input = "select * from contact where id = '123';\n\n-- test comment\n\nselect wrong statement;\n\nselect id,username from contact\n\nselect id,name\nfrom contact -- test inline comment\nwhere id = '123';\n\n";
 
         let mut lex = SourceFileToken::lexer(&input);