pawarser, json-pawarser: get first debug print working!

This commit is contained in:
Schrottkatze 2024-10-21 18:29:46 +02:00
parent af6886214b
commit b8720b2df9
No known key found for this signature in database
7 changed files with 140 additions and 23 deletions

View file

@ -4,8 +4,8 @@ use crate::{syntax_error::SyntaxError, syntax_kind::SyntaxKind};
use self::object::object; use self::object::object;
type Parser<'src> = pawarser::Parser<'src, SyntaxKind, SyntaxError>; pub(crate) type Parser<'src> = pawarser::Parser<'src, SyntaxKind, SyntaxError>;
type CompletedMarker = pawarser::CompletedMarker<SyntaxKind, SyntaxError>; pub(crate) type CompletedMarker = pawarser::CompletedMarker<SyntaxKind, SyntaxError>;
const BASIC_VALUE_TOKENS: EnumSet<SyntaxKind> = const BASIC_VALUE_TOKENS: EnumSet<SyntaxKind> =
enum_set!(SyntaxKind::BOOL | SyntaxKind::NULL | SyntaxKind::NUMBER | SyntaxKind::STRING); enum_set!(SyntaxKind::BOOL | SyntaxKind::NULL | SyntaxKind::NUMBER | SyntaxKind::STRING);
@ -27,12 +27,15 @@ mod object {
pub(super) fn object(p: &mut Parser) -> Option<CompletedMarker> { pub(super) fn object(p: &mut Parser) -> Option<CompletedMarker> {
let obj_start = p.start("object"); let obj_start = p.start("object");
if !p.at(SyntaxKind::BRACE_OPEN) { if !p.eat(SyntaxKind::BRACE_OPEN) {
obj_start.abandon(p); obj_start.abandon(p);
return None; return None;
} }
todo!() member(p);
p.eat(SyntaxKind::BRACE_CLOSE);
Some(obj_start.complete(p, SyntaxKind::OBJECT))
} }
fn member(p: &mut Parser) -> Option<CompletedMarker> { fn member(p: &mut Parser) -> Option<CompletedMarker> {
@ -46,7 +49,7 @@ mod object {
p.eat(SyntaxKind::STRING); p.eat(SyntaxKind::STRING);
member_name_start.complete(p, SyntaxKind::MEMBER_NAME); member_name_start.complete(p, SyntaxKind::MEMBER_NAME);
} else { } else {
return todo!("handle other tokens"); return todo!("handle other tokens: {:?}", p.current());
} }
if !p.eat(SyntaxKind::COLON) { if !p.eat(SyntaxKind::COLON) {

View file

@ -1,3 +1,29 @@
mod grammar; mod grammar;
mod syntax_error; mod syntax_error;
mod syntax_kind; mod syntax_kind;
#[cfg(test)]
mod test {
use pawarser::parser::ParserBuilder;
use crate::{
grammar::{value, Parser},
syntax_kind::{lex, SyntaxKind},
};
#[test]
fn test() {
const TEST_DATA: &str = r#"{"hello_world": "meow"}"#;
let toks = lex(TEST_DATA);
let mut p: Parser = ParserBuilder::new(toks)
.add_meaningless(SyntaxKind::WHITESPACE)
.add_meaningless(SyntaxKind::NEWLINE)
.build();
value(&mut p);
let out = p.finish();
assert_eq!("", format!("{:#?}", out))
}
}

View file

@ -1,6 +1,6 @@
use crate::syntax_kind::SyntaxKind; use crate::syntax_kind::SyntaxKind;
#[derive(Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
pub enum SyntaxError { pub enum SyntaxError {
DisallowedKeyType(SyntaxKind), DisallowedKeyType(SyntaxKind),
MemberMissingValue, MemberMissingValue,

View file

@ -16,14 +16,6 @@ pub fn lex(src: &str) -> Vec<(SyntaxKind, &str)> {
#[enumset(no_super_impls)] #[enumset(no_super_impls)]
#[allow(non_camel_case_types)] #[allow(non_camel_case_types)]
pub enum SyntaxKind { pub enum SyntaxKind {
// Error SyntaxKinds
LEX_ERR,
PARSE_ERR,
// Meta SyntaxKinds
TOMBSTONE,
EOF,
OBJECT, OBJECT,
MEMBER, MEMBER,
MEMBER_NAME, MEMBER_NAME,
@ -61,6 +53,13 @@ pub enum SyntaxKind {
WHITESPACE, WHITESPACE,
#[token("\n")] #[token("\n")]
NEWLINE, NEWLINE,
// Error SyntaxKinds
LEX_ERR,
PARSE_ERR,
// Meta SyntaxKinds
EOF,
} }
impl pawarser::parser::SyntaxElement for SyntaxKind { impl pawarser::parser::SyntaxElement for SyntaxKind {
@ -75,6 +74,16 @@ impl From<SyntaxKind> for rowan::SyntaxKind {
} }
} }
impl From<rowan::SyntaxKind> for SyntaxKind {
fn from(raw: rowan::SyntaxKind) -> Self {
assert!(raw.0 <= SyntaxKind::EOF as u16);
#[allow(unsafe_code, reason = "The transmute is necessary here")]
unsafe {
std::mem::transmute::<u16, SyntaxKind>(raw.0)
}
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::syntax_kind::{lex, SyntaxKind}; use crate::syntax_kind::{lex, SyntaxKind};

View file

@ -1,4 +1,4 @@
use std::{cell::Cell, marker::PhantomData, mem}; use std::{cell::Cell, fmt, marker::PhantomData, mem};
use enumset::{EnumSet, EnumSetType}; use enumset::{EnumSet, EnumSetType};
use rowan::{GreenNode, GreenNodeBuilder}; use rowan::{GreenNode, GreenNodeBuilder};
@ -6,17 +6,24 @@ use rowan::{GreenNode, GreenNodeBuilder};
use crate::parser::event::NodeKind; use crate::parser::event::NodeKind;
use self::{event::Event, input::Input, marker::Marker}; use self::{event::Event, input::Input, marker::Marker};
pub use error::SyntaxError; pub use {error::SyntaxError, output::ParserOutput};
pub mod error; pub mod error;
mod event; mod event;
mod input; mod input;
pub mod marker; pub mod marker;
pub mod output;
/// this is used to define some required SyntaxKinds like an EOF token or an error token /// this is used to define some required SyntaxKinds like an EOF token or an error token
pub trait SyntaxElement pub trait SyntaxElement
where where
Self: EnumSetType + Into<rowan::SyntaxKind> + Clone + PartialEq + Eq, Self: EnumSetType
+ Into<rowan::SyntaxKind>
+ From<rowan::SyntaxKind>
+ fmt::Debug
+ Clone
+ PartialEq
+ Eq,
{ {
/// EOF value. This will be used by the rest of the parser library to represent an EOF. /// EOF value. This will be used by the rest of the parser library to represent an EOF.
const EOF: Self; const EOF: Self;
@ -97,15 +104,14 @@ impl<'src, 'toks, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError>
self.steps.set(steps + 1); self.steps.set(steps + 1);
} }
pub fn finish( pub fn finish(self) -> ParserOutput<SyntaxKind, SyntaxErr> {
Self { let Self {
input, input,
pos, pos,
mut events, mut events,
step_limit, step_limit,
steps, steps,
}: Self, } = self;
) -> ParserOutput<SyntaxKind, SyntaxErr> {
let (mut raw_toks, meaningless_tokens) = input.dissolve(); let (mut raw_toks, meaningless_tokens) = input.dissolve();
let mut builder = GreenNodeBuilder::new(); let mut builder = GreenNodeBuilder::new();
// TODO: document what the hell a forward parent is // TODO: document what the hell a forward parent is
@ -178,7 +184,11 @@ impl<'src, 'toks, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError>
} }
} }
todo!() ParserOutput {
green_node: builder.finish(),
errors,
_syntax_kind: PhantomData::<SyntaxKind>,
}
} }
} }

View file

@ -1,7 +1,9 @@
use std::fmt;
/// A marker trait... for now! /// A marker trait... for now!
// TODO: constrain that conversion to `NodeKind::Error` is enforced to be possible // TODO: constrain that conversion to `NodeKind::Error` is enforced to be possible
pub trait SyntaxError pub trait SyntaxError
where where
Self: Clone + PartialEq + Eq, Self: fmt::Debug + Clone + PartialEq + Eq,
{ {
} }

View file

@ -0,0 +1,67 @@
use std::{fmt, marker::PhantomData};
use rowan::{GreenNode, GreenNodeData, GreenTokenData, NodeOrToken};
use crate::{SyntaxElement, SyntaxError};
pub struct ParserOutput<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> {
pub green_node: GreenNode,
pub errors: Vec<SyntaxErr>,
pub(super) _syntax_kind: PhantomData<SyntaxKind>,
}
impl<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> std::fmt::Debug
for ParserOutput<SyntaxKind, SyntaxErr>
{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let mut errs: Vec<&SyntaxErr> = self.errors.iter().collect();
errs.reverse();
debug_print_output::<SyntaxKind, SyntaxErr>(
NodeOrToken::Node(&self.green_node),
f,
0,
&mut errs,
)
}
}
fn debug_print_output<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError>(
node: NodeOrToken<&GreenNodeData, &GreenTokenData>,
f: &mut std::fmt::Formatter<'_>,
lvl: i32,
errs: &mut Vec<&SyntaxErr>,
) -> std::fmt::Result {
for _ in 0..lvl {
f.write_str(" ")?;
}
match node {
NodeOrToken::Node(n) => {
let kind: SyntaxKind = node.kind().into();
if kind != SyntaxKind::ERROR {
writeln!(f, "{:?} {{", kind)?;
} else {
let err = errs
.pop()
.expect("all error syntax nodes should correspond to an error");
writeln!(f, "{:?}: {err:?} {{", kind)?;
}
for c in n.children() {
debug_print_output::<SyntaxKind, SyntaxErr>(c, f, lvl + 1, errs)?;
}
for _ in 0..lvl {
f.write_str(" ")?;
}
f.write_str("}\n")
}
NodeOrToken::Token(t) => {
writeln!(
f,
"{:?} {:?};",
Into::<SyntaxKind>::into(t.kind()),
t.text()
)
}
}
}