mirror of
https://forge.katzen.cafe/katzen-cafe/iowo.git
synced 2024-11-21 13:14:41 +01:00
pawarser(chore): split up files
This commit is contained in:
parent
ec2ff5778b
commit
34ddaacb58
|
@ -1,166 +1,2 @@
|
||||||
#![feature(iter_collect_into)]
|
#![feature(iter_collect_into)]
|
||||||
pub mod parser {
|
pub mod parser;
|
||||||
use std::cell::Cell;
|
|
||||||
|
|
||||||
use enumset::{EnumSet, EnumSetType};
|
|
||||||
|
|
||||||
use self::{error::SyntaxError, event::Event, input::Input};
|
|
||||||
|
|
||||||
pub mod input {
|
|
||||||
use enumset::{EnumSet, EnumSetType};
|
|
||||||
|
|
||||||
pub struct Input<'src, 'toks, SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>> {
|
|
||||||
raw: &'toks Vec<(SyntaxKind, &'src str)>,
|
|
||||||
// enumset of meaningless tokens
|
|
||||||
semantically_meaningless: EnumSet<SyntaxKind>,
|
|
||||||
// indices of non-meaningless tokens
|
|
||||||
meaningful_toks: Vec<usize>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<'src, 'toks, SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>>
|
|
||||||
Input<'src, 'toks, SyntaxKind>
|
|
||||||
{
|
|
||||||
pub fn new(
|
|
||||||
raw_toks: &'toks Vec<(SyntaxKind, &'src str)>,
|
|
||||||
meaningless: Option<EnumSet<SyntaxKind>>,
|
|
||||||
) -> Self {
|
|
||||||
let mut meaningful_toks = Vec::new();
|
|
||||||
|
|
||||||
if let Some(meaningless) = meaningless {
|
|
||||||
let meaningful_toks = raw_toks
|
|
||||||
.iter()
|
|
||||||
.enumerate()
|
|
||||||
.filter_map(|(i, tok)| (!meaningless.contains(tok.0)).then_some(i))
|
|
||||||
.collect_into(&mut meaningful_toks);
|
|
||||||
}
|
|
||||||
|
|
||||||
Self {
|
|
||||||
raw: raw_toks,
|
|
||||||
semantically_meaningless: meaningless.unwrap_or_default(),
|
|
||||||
meaningful_toks,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
mod event {
|
|
||||||
use enumset::EnumSetType;
|
|
||||||
|
|
||||||
use super::error::SyntaxError;
|
|
||||||
|
|
||||||
pub enum Event<SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>, SyntaxErr: SyntaxError> {
|
|
||||||
Start {
|
|
||||||
kind: NodeKind<SyntaxKind, SyntaxErr>,
|
|
||||||
forward_parent: Option<usize>,
|
|
||||||
},
|
|
||||||
Finish,
|
|
||||||
Eat {
|
|
||||||
count: usize,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>, SyntaxErr: SyntaxError>
|
|
||||||
Event<SyntaxKind, SyntaxErr>
|
|
||||||
{
|
|
||||||
pub fn tombstone() -> Self {
|
|
||||||
Self::Start {
|
|
||||||
kind: NodeKind::Tombstone,
|
|
||||||
forward_parent: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub enum NodeKind<SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>, SyntaxErr: SyntaxError> {
|
|
||||||
Tombstone,
|
|
||||||
Syntax(SyntaxKind),
|
|
||||||
Error(SyntaxErr),
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>, SyntaxErr: SyntaxError>
|
|
||||||
NodeKind<SyntaxKind, SyntaxErr>
|
|
||||||
{
|
|
||||||
pub fn is_tombstone(&self) -> bool {
|
|
||||||
matches!(self, Self::Tombstone)
|
|
||||||
}
|
|
||||||
pub fn is_syntax(&self) -> bool {
|
|
||||||
matches!(self, Self::Syntax(_))
|
|
||||||
}
|
|
||||||
pub fn is_error(&self) -> bool {
|
|
||||||
matches!(self, Self::Error(_))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
mod error {
|
|
||||||
/// A marker trait... for now!
|
|
||||||
// TODO: constrain that conversion to `NodeKind::Error` is enforced to be possible
|
|
||||||
pub trait SyntaxError {}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct Parser<
|
|
||||||
'src,
|
|
||||||
'toks,
|
|
||||||
SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>,
|
|
||||||
SyntaxErr: SyntaxError,
|
|
||||||
> {
|
|
||||||
input: Input<'src, 'toks, SyntaxKind>,
|
|
||||||
pos: usize,
|
|
||||||
events: Vec<Event<SyntaxKind, SyntaxErr>>,
|
|
||||||
step_limit: u32,
|
|
||||||
steps: Cell<u32>,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct ParserBuilder<
|
|
||||||
'src,
|
|
||||||
'toks,
|
|
||||||
SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>,
|
|
||||||
// SyntaxErr: SyntaxError,
|
|
||||||
> {
|
|
||||||
raw_toks: &'toks Vec<(SyntaxKind, &'src str)>,
|
|
||||||
meaningless_token_kinds: EnumSet<SyntaxKind>,
|
|
||||||
step_limit: u32,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<'src, 'toks, SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>>
|
|
||||||
ParserBuilder<'src, 'toks, SyntaxKind>
|
|
||||||
{
|
|
||||||
pub fn new(raw_toks: &'toks Vec<(SyntaxKind, &'src str)>) -> Self {
|
|
||||||
Self {
|
|
||||||
raw_toks,
|
|
||||||
meaningless_token_kinds: EnumSet::new(),
|
|
||||||
step_limit: 4096,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Sets the parser step limit.
|
|
||||||
/// Defaults to 4096
|
|
||||||
pub fn step_limit(mut self, new: u32) -> Self {
|
|
||||||
self.step_limit = new;
|
|
||||||
self
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn add_meaningless(mut self, kind: SyntaxKind) -> Self {
|
|
||||||
self.meaningless_token_kinds.insert(kind);
|
|
||||||
self
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn add_meaningless_many(mut self, kind: Vec<SyntaxKind>) -> Self {
|
|
||||||
self.meaningless_token_kinds
|
|
||||||
.insert_all(kind.into_iter().collect());
|
|
||||||
self
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn build<SyntaxErr: SyntaxError>(self) -> Parser<'src, 'toks, SyntaxKind, SyntaxErr> {
|
|
||||||
let Self {
|
|
||||||
raw_toks,
|
|
||||||
meaningless_token_kinds,
|
|
||||||
step_limit,
|
|
||||||
} = self;
|
|
||||||
Parser {
|
|
||||||
input: Input::new(raw_toks, Some(meaningless_token_kinds)),
|
|
||||||
pos: 0,
|
|
||||||
events: Vec::new(),
|
|
||||||
step_limit,
|
|
||||||
steps: Cell::new(0),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
78
crates/pawarser/src/parser.rs
Normal file
78
crates/pawarser/src/parser.rs
Normal file
|
@ -0,0 +1,78 @@
|
||||||
|
use std::cell::Cell;
|
||||||
|
|
||||||
|
use enumset::{EnumSet, EnumSetType};
|
||||||
|
|
||||||
|
use self::{error::SyntaxError, event::Event, input::Input};
|
||||||
|
|
||||||
|
mod error;
|
||||||
|
mod event;
|
||||||
|
pub mod input;
|
||||||
|
|
||||||
|
pub struct Parser<
|
||||||
|
'src,
|
||||||
|
'toks,
|
||||||
|
SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>,
|
||||||
|
SyntaxErr: SyntaxError,
|
||||||
|
> {
|
||||||
|
input: Input<'src, 'toks, SyntaxKind>,
|
||||||
|
pos: usize,
|
||||||
|
events: Vec<Event<SyntaxKind, SyntaxErr>>,
|
||||||
|
step_limit: u32,
|
||||||
|
steps: Cell<u32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct ParserBuilder<
|
||||||
|
'src,
|
||||||
|
'toks,
|
||||||
|
SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>,
|
||||||
|
// SyntaxErr: SyntaxError,
|
||||||
|
> {
|
||||||
|
raw_toks: &'toks Vec<(SyntaxKind, &'src str)>,
|
||||||
|
meaningless_token_kinds: EnumSet<SyntaxKind>,
|
||||||
|
step_limit: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'src, 'toks, SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>>
|
||||||
|
ParserBuilder<'src, 'toks, SyntaxKind>
|
||||||
|
{
|
||||||
|
pub fn new(raw_toks: &'toks Vec<(SyntaxKind, &'src str)>) -> Self {
|
||||||
|
Self {
|
||||||
|
raw_toks,
|
||||||
|
meaningless_token_kinds: EnumSet::new(),
|
||||||
|
step_limit: 4096,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sets the parser step limit.
|
||||||
|
/// Defaults to 4096
|
||||||
|
pub fn step_limit(mut self, new: u32) -> Self {
|
||||||
|
self.step_limit = new;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn add_meaningless(mut self, kind: SyntaxKind) -> Self {
|
||||||
|
self.meaningless_token_kinds.insert(kind);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn add_meaningless_many(mut self, kind: Vec<SyntaxKind>) -> Self {
|
||||||
|
self.meaningless_token_kinds
|
||||||
|
.insert_all(kind.into_iter().collect());
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn build<SyntaxErr: SyntaxError>(self) -> Parser<'src, 'toks, SyntaxKind, SyntaxErr> {
|
||||||
|
let Self {
|
||||||
|
raw_toks,
|
||||||
|
meaningless_token_kinds,
|
||||||
|
step_limit,
|
||||||
|
} = self;
|
||||||
|
Parser {
|
||||||
|
input: Input::new(raw_toks, Some(meaningless_token_kinds)),
|
||||||
|
pos: 0,
|
||||||
|
events: Vec::new(),
|
||||||
|
step_limit,
|
||||||
|
steps: Cell::new(0),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
3
crates/pawarser/src/parser/error.rs
Normal file
3
crates/pawarser/src/parser/error.rs
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
/// A marker trait... for now!
|
||||||
|
// TODO: constrain that conversion to `NodeKind::Error` is enforced to be possible
|
||||||
|
pub trait SyntaxError {}
|
45
crates/pawarser/src/parser/event.rs
Normal file
45
crates/pawarser/src/parser/event.rs
Normal file
|
@ -0,0 +1,45 @@
|
||||||
|
use enumset::EnumSetType;
|
||||||
|
|
||||||
|
use super::error::SyntaxError;
|
||||||
|
|
||||||
|
pub enum Event<SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>, SyntaxErr: SyntaxError> {
|
||||||
|
Start {
|
||||||
|
kind: NodeKind<SyntaxKind, SyntaxErr>,
|
||||||
|
forward_parent: Option<usize>,
|
||||||
|
},
|
||||||
|
Finish,
|
||||||
|
Eat {
|
||||||
|
count: usize,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>, SyntaxErr: SyntaxError>
|
||||||
|
Event<SyntaxKind, SyntaxErr>
|
||||||
|
{
|
||||||
|
pub fn tombstone() -> Self {
|
||||||
|
Self::Start {
|
||||||
|
kind: NodeKind::Tombstone,
|
||||||
|
forward_parent: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub enum NodeKind<SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>, SyntaxErr: SyntaxError> {
|
||||||
|
Tombstone,
|
||||||
|
Syntax(SyntaxKind),
|
||||||
|
Error(SyntaxErr),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>, SyntaxErr: SyntaxError>
|
||||||
|
NodeKind<SyntaxKind, SyntaxErr>
|
||||||
|
{
|
||||||
|
pub fn is_tombstone(&self) -> bool {
|
||||||
|
matches!(self, Self::Tombstone)
|
||||||
|
}
|
||||||
|
pub fn is_syntax(&self) -> bool {
|
||||||
|
matches!(self, Self::Syntax(_))
|
||||||
|
}
|
||||||
|
pub fn is_error(&self) -> bool {
|
||||||
|
matches!(self, Self::Error(_))
|
||||||
|
}
|
||||||
|
}
|
34
crates/pawarser/src/parser/input.rs
Normal file
34
crates/pawarser/src/parser/input.rs
Normal file
|
@ -0,0 +1,34 @@
|
||||||
|
use enumset::{EnumSet, EnumSetType};
|
||||||
|
|
||||||
|
pub struct Input<'src, 'toks, SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>> {
|
||||||
|
raw: &'toks Vec<(SyntaxKind, &'src str)>,
|
||||||
|
// enumset of meaningless tokens
|
||||||
|
semantically_meaningless: EnumSet<SyntaxKind>,
|
||||||
|
// indices of non-meaningless tokens
|
||||||
|
meaningful_toks: Vec<usize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'src, 'toks, SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>>
|
||||||
|
Input<'src, 'toks, SyntaxKind>
|
||||||
|
{
|
||||||
|
pub fn new(
|
||||||
|
raw_toks: &'toks Vec<(SyntaxKind, &'src str)>,
|
||||||
|
meaningless: Option<EnumSet<SyntaxKind>>,
|
||||||
|
) -> Self {
|
||||||
|
let mut meaningful_toks = Vec::new();
|
||||||
|
|
||||||
|
if let Some(meaningless) = meaningless {
|
||||||
|
let meaningful_toks = raw_toks
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.filter_map(|(i, tok)| (!meaningless.contains(tok.0)).then_some(i))
|
||||||
|
.collect_into(&mut meaningful_toks);
|
||||||
|
}
|
||||||
|
|
||||||
|
Self {
|
||||||
|
raw: raw_toks,
|
||||||
|
semantically_meaningless: meaningless.unwrap_or_default(),
|
||||||
|
meaningful_toks,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in a new issue