pawarser(chore): split up files

This commit is contained in:
Schrottkatze 2024-10-13 16:47:53 +02:00
parent ec2ff5778b
commit 34ddaacb58
No known key found for this signature in database
5 changed files with 161 additions and 165 deletions

View file

@ -1,166 +1,2 @@
#![feature(iter_collect_into)]
pub mod parser {
use std::cell::Cell;
use enumset::{EnumSet, EnumSetType};
use self::{error::SyntaxError, event::Event, input::Input};
pub mod input {
use enumset::{EnumSet, EnumSetType};
pub struct Input<'src, 'toks, SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>> {
raw: &'toks Vec<(SyntaxKind, &'src str)>,
// enumset of meaningless tokens
semantically_meaningless: EnumSet<SyntaxKind>,
// indices of non-meaningless tokens
meaningful_toks: Vec<usize>,
}
impl<'src, 'toks, SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>>
Input<'src, 'toks, SyntaxKind>
{
pub fn new(
raw_toks: &'toks Vec<(SyntaxKind, &'src str)>,
meaningless: Option<EnumSet<SyntaxKind>>,
) -> Self {
let mut meaningful_toks = Vec::new();
if let Some(meaningless) = meaningless {
let meaningful_toks = raw_toks
.iter()
.enumerate()
.filter_map(|(i, tok)| (!meaningless.contains(tok.0)).then_some(i))
.collect_into(&mut meaningful_toks);
}
Self {
raw: raw_toks,
semantically_meaningless: meaningless.unwrap_or_default(),
meaningful_toks,
}
}
}
}
mod event {
use enumset::EnumSetType;
use super::error::SyntaxError;
pub enum Event<SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>, SyntaxErr: SyntaxError> {
Start {
kind: NodeKind<SyntaxKind, SyntaxErr>,
forward_parent: Option<usize>,
},
Finish,
Eat {
count: usize,
},
}
impl<SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>, SyntaxErr: SyntaxError>
Event<SyntaxKind, SyntaxErr>
{
pub fn tombstone() -> Self {
Self::Start {
kind: NodeKind::Tombstone,
forward_parent: None,
}
}
}
pub enum NodeKind<SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>, SyntaxErr: SyntaxError> {
Tombstone,
Syntax(SyntaxKind),
Error(SyntaxErr),
}
impl<SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>, SyntaxErr: SyntaxError>
NodeKind<SyntaxKind, SyntaxErr>
{
pub fn is_tombstone(&self) -> bool {
matches!(self, Self::Tombstone)
}
pub fn is_syntax(&self) -> bool {
matches!(self, Self::Syntax(_))
}
pub fn is_error(&self) -> bool {
matches!(self, Self::Error(_))
}
}
}
mod error {
/// A marker trait... for now!
// TODO: constrain that conversion to `NodeKind::Error` is enforced to be possible
pub trait SyntaxError {}
}
pub struct Parser<
'src,
'toks,
SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>,
SyntaxErr: SyntaxError,
> {
input: Input<'src, 'toks, SyntaxKind>,
pos: usize,
events: Vec<Event<SyntaxKind, SyntaxErr>>,
step_limit: u32,
steps: Cell<u32>,
}
pub struct ParserBuilder<
'src,
'toks,
SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>,
// SyntaxErr: SyntaxError,
> {
raw_toks: &'toks Vec<(SyntaxKind, &'src str)>,
meaningless_token_kinds: EnumSet<SyntaxKind>,
step_limit: u32,
}
impl<'src, 'toks, SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>>
ParserBuilder<'src, 'toks, SyntaxKind>
{
pub fn new(raw_toks: &'toks Vec<(SyntaxKind, &'src str)>) -> Self {
Self {
raw_toks,
meaningless_token_kinds: EnumSet::new(),
step_limit: 4096,
}
}
/// Sets the parser step limit.
/// Defaults to 4096
pub fn step_limit(mut self, new: u32) -> Self {
self.step_limit = new;
self
}
pub fn add_meaningless(mut self, kind: SyntaxKind) -> Self {
self.meaningless_token_kinds.insert(kind);
self
}
pub fn add_meaningless_many(mut self, kind: Vec<SyntaxKind>) -> Self {
self.meaningless_token_kinds
.insert_all(kind.into_iter().collect());
self
}
pub fn build<SyntaxErr: SyntaxError>(self) -> Parser<'src, 'toks, SyntaxKind, SyntaxErr> {
let Self {
raw_toks,
meaningless_token_kinds,
step_limit,
} = self;
Parser {
input: Input::new(raw_toks, Some(meaningless_token_kinds)),
pos: 0,
events: Vec::new(),
step_limit,
steps: Cell::new(0),
}
}
}
}
pub mod parser;

View file

@ -0,0 +1,78 @@
use std::cell::Cell;
use enumset::{EnumSet, EnumSetType};
use self::{error::SyntaxError, event::Event, input::Input};
mod error;
mod event;
pub mod input;
pub struct Parser<
'src,
'toks,
SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>,
SyntaxErr: SyntaxError,
> {
input: Input<'src, 'toks, SyntaxKind>,
pos: usize,
events: Vec<Event<SyntaxKind, SyntaxErr>>,
step_limit: u32,
steps: Cell<u32>,
}
pub struct ParserBuilder<
'src,
'toks,
SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>,
// SyntaxErr: SyntaxError,
> {
raw_toks: &'toks Vec<(SyntaxKind, &'src str)>,
meaningless_token_kinds: EnumSet<SyntaxKind>,
step_limit: u32,
}
impl<'src, 'toks, SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>>
ParserBuilder<'src, 'toks, SyntaxKind>
{
pub fn new(raw_toks: &'toks Vec<(SyntaxKind, &'src str)>) -> Self {
Self {
raw_toks,
meaningless_token_kinds: EnumSet::new(),
step_limit: 4096,
}
}
/// Sets the parser step limit.
/// Defaults to 4096
pub fn step_limit(mut self, new: u32) -> Self {
self.step_limit = new;
self
}
pub fn add_meaningless(mut self, kind: SyntaxKind) -> Self {
self.meaningless_token_kinds.insert(kind);
self
}
pub fn add_meaningless_many(mut self, kind: Vec<SyntaxKind>) -> Self {
self.meaningless_token_kinds
.insert_all(kind.into_iter().collect());
self
}
pub fn build<SyntaxErr: SyntaxError>(self) -> Parser<'src, 'toks, SyntaxKind, SyntaxErr> {
let Self {
raw_toks,
meaningless_token_kinds,
step_limit,
} = self;
Parser {
input: Input::new(raw_toks, Some(meaningless_token_kinds)),
pos: 0,
events: Vec::new(),
step_limit,
steps: Cell::new(0),
}
}
}

View file

@ -0,0 +1,3 @@
/// A marker trait... for now!
// TODO: constrain that conversion to `NodeKind::Error` is enforced to be possible
pub trait SyntaxError {}

View file

@ -0,0 +1,45 @@
use enumset::EnumSetType;
use super::error::SyntaxError;
pub enum Event<SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>, SyntaxErr: SyntaxError> {
Start {
kind: NodeKind<SyntaxKind, SyntaxErr>,
forward_parent: Option<usize>,
},
Finish,
Eat {
count: usize,
},
}
impl<SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>, SyntaxErr: SyntaxError>
Event<SyntaxKind, SyntaxErr>
{
pub fn tombstone() -> Self {
Self::Start {
kind: NodeKind::Tombstone,
forward_parent: None,
}
}
}
pub enum NodeKind<SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>, SyntaxErr: SyntaxError> {
Tombstone,
Syntax(SyntaxKind),
Error(SyntaxErr),
}
impl<SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>, SyntaxErr: SyntaxError>
NodeKind<SyntaxKind, SyntaxErr>
{
pub fn is_tombstone(&self) -> bool {
matches!(self, Self::Tombstone)
}
pub fn is_syntax(&self) -> bool {
matches!(self, Self::Syntax(_))
}
pub fn is_error(&self) -> bool {
matches!(self, Self::Error(_))
}
}

View file

@ -0,0 +1,34 @@
use enumset::{EnumSet, EnumSetType};
pub struct Input<'src, 'toks, SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>> {
raw: &'toks Vec<(SyntaxKind, &'src str)>,
// enumset of meaningless tokens
semantically_meaningless: EnumSet<SyntaxKind>,
// indices of non-meaningless tokens
meaningful_toks: Vec<usize>,
}
impl<'src, 'toks, SyntaxKind: EnumSetType + Into<rowan::SyntaxKind>>
Input<'src, 'toks, SyntaxKind>
{
pub fn new(
raw_toks: &'toks Vec<(SyntaxKind, &'src str)>,
meaningless: Option<EnumSet<SyntaxKind>>,
) -> Self {
let mut meaningful_toks = Vec::new();
if let Some(meaningless) = meaningless {
let meaningful_toks = raw_toks
.iter()
.enumerate()
.filter_map(|(i, tok)| (!meaningless.contains(tok.0)).then_some(i))
.collect_into(&mut meaningful_toks);
}
Self {
raw: raw_toks,
semantically_meaningless: meaningless.unwrap_or_default(),
meaningful_toks,
}
}
}