From 37651a83bc7db9ec5cebf98be1d8c45ee8bab9f0 Mon Sep 17 00:00:00 2001 From: Schrottkatze Date: Mon, 10 Jun 2024 09:47:48 +0200 Subject: [PATCH] lang: current state idk try to parallelize --- Cargo.lock | 91 ++++++++++++++- crates/lang/Cargo.toml | 3 + crates/lang/src/lib.rs | 7 +- crates/lang/src/main.rs | 18 +-- crates/lang/src/world.rs | 47 ++++++++ crates/lang/src/world/error.rs | 17 +++ crates/lang/src/world/files.rs | 116 +++++++++++++++++++ crates/lang/src/world/modules.rs | 38 ++++++ crates/lang/src/world/nodes.rs | 77 ++++++++++++ crates/lang/src/world/registry.rs | 41 +++++++ crates/lang/src/world/world_creation_pool.rs | 82 +++++++++++++ 11 files changed, 522 insertions(+), 15 deletions(-) create mode 100644 crates/lang/src/world.rs create mode 100644 crates/lang/src/world/error.rs create mode 100644 crates/lang/src/world/files.rs create mode 100644 crates/lang/src/world/modules.rs create mode 100644 crates/lang/src/world/nodes.rs create mode 100644 crates/lang/src/world/registry.rs create mode 100644 crates/lang/src/world/world_creation_pool.rs diff --git a/Cargo.lock b/Cargo.lock index eaa8be7..cd7e5fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -206,6 +206,28 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.4" @@ -228,6 +250,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.18" @@ -286,6 +317,19 @@ dependencies = [ "syn", ] +[[package]] +name = "dashmap" +version = "5.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +dependencies = [ + "cfg-if", + "hashbrown", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "deranged" version = "0.3.11" @@ -563,12 +607,15 @@ version = "0.1.0" dependencies = [ "ariadne", "clap", + "crossbeam", + "dashmap", "drop_bomb", "ego-tree", "enumset", "indexmap", "indoc", "logos", + "num_cpus", "owo-colors", "petgraph", "rowan", @@ -588,9 +635,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" [[package]] name = "libc" -version = "0.2.152" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libredox" @@ -600,7 +647,7 @@ checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ "bitflags 2.4.1", "libc", - "redox_syscall", + "redox_syscall 0.4.1", ] [[package]] @@ -701,6 +748,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "num_threads" version = "0.1.6" @@ -710,6 +767,12 @@ dependencies = [ "libc", ] +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + [[package]] name = "option-ext" version = "0.2.0" @@ -725,6 +788,19 @@ dependencies = [ "supports-color", ] +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.1", + "smallvec", + "windows-targets 0.52.0", +] + [[package]] name = "petgraph" version = "0.6.4" @@ -877,6 +953,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +dependencies = [ + "bitflags 2.4.1", +] + [[package]] name = "redox_users" version = "0.4.4" diff --git a/crates/lang/Cargo.toml b/crates/lang/Cargo.toml index 8a9c80b..0323c98 100644 --- a/crates/lang/Cargo.toml +++ b/crates/lang/Cargo.toml @@ -16,6 +16,9 @@ rowan = "0.15.15" drop_bomb = "0.1.5" enumset = "1.1.3" indoc = "2" +num_cpus = "1.16" +dashmap = "5.5.3" +crossbeam = "0.8.4" owo-colors = {version = "4", features = ["supports-colors"]} [lints] diff --git a/crates/lang/src/lib.rs b/crates/lang/src/lib.rs index d700d45..c8b48f6 100644 --- a/crates/lang/src/lib.rs +++ b/crates/lang/src/lib.rs @@ -2,6 +2,10 @@ use crate::lst_parser::syntax_kind::SyntaxKind; +pub mod ast; +pub mod lst_parser; +pub mod world; + #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Lang {} impl rowan::Language for Lang { @@ -19,6 +23,3 @@ impl rowan::Language for Lang { pub type SyntaxNode = rowan::SyntaxNode; pub type SyntaxToken = rowan::SyntaxNode; pub type SyntaxElement = rowan::NodeOrToken; - -pub mod ast; -pub mod lst_parser; diff --git a/crates/lang/src/main.rs b/crates/lang/src/main.rs index 1dbd527..c0c04d5 100644 --- a/crates/lang/src/main.rs +++ b/crates/lang/src/main.rs @@ -15,18 +15,18 @@ struct Args { fn main() { let args = Args::parse(); let n = args.file.clone(); - // let f = fs::read_to_string(n.clone()).expect("failed to read file"); + let f = fs::read_to_string(n.clone()).expect("failed to read file"); - // let toks = dbg!(syntax_kind::lex(&f)); - // let input = input::Input::new(&toks); - // let mut parser = lst_parser::Parser::new(input); + let toks = dbg!(syntax_kind::lex(&f)); + let input = input::Input::new(&toks); + let mut parser = lst_parser::Parser::new(input); - // grammar::source_file(&mut parser); + grammar::source_file(&mut parser); - // let p_out = dbg!(parser.finish()); - // let o = Output::from_parser_output(toks, p_out); + let p_out = dbg!(parser.finish()); + let o = Output::from_parser_output(toks, p_out); - // println!("{}", o.debug_colored()); + println!("{}", o.debug_colored()); - World::new(n); + // World::new(n); } diff --git a/crates/lang/src/world.rs b/crates/lang/src/world.rs new file mode 100644 index 0000000..8a2506b --- /dev/null +++ b/crates/lang/src/world.rs @@ -0,0 +1,47 @@ +use std::{ + fmt::Display, + path::{Path, PathBuf}, +}; + +use crate::world::world_creation_pool::WorldCreationPool; + +use self::{files::Files, registry::Registry}; + +pub mod error; +pub mod files; +pub mod modules; +pub mod nodes; +pub mod registry; +mod world_creation_pool; + +pub struct World {} + +impl World { + pub fn new(entry_point: &Path) -> Result { + let files = Files::new(); + let reg = Registry::new(); + + let (entry_point_id, errors) = files.add_and_parse(entry_point).map_err(|e| { + WorldCreationError::FailedToOpenEntryPoint(entry_point.to_path_buf(), e) + })?; + + let pool = WorldCreationPool::new(files, reg); + + todo!() + } +} + +#[derive(Debug)] +pub enum WorldCreationError { + FailedToOpenEntryPoint(PathBuf, std::io::Error), +} + +impl Display for WorldCreationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + WorldCreationError::FailedToOpenEntryPoint(entry_path, e) => { + write!(f, "failed to open entry_point '{entry_path:?}': {e}") + } + } + } +} diff --git a/crates/lang/src/world/error.rs b/crates/lang/src/world/error.rs new file mode 100644 index 0000000..6232a68 --- /dev/null +++ b/crates/lang/src/world/error.rs @@ -0,0 +1,17 @@ +use std::path::PathBuf; + +use crate::lst_parser::error::SyntaxError; + +use super::files::FileId; + +#[derive(Debug)] +pub enum Error { + Syntax(FileId, SyntaxError), + FailedToOpenFileMod(PathBuf, std::io::Error), +} + +impl Error { + pub fn from_syntax(file: FileId, e: SyntaxError) -> Self { + Self::Syntax(file, e) + } +} diff --git a/crates/lang/src/world/files.rs b/crates/lang/src/world/files.rs new file mode 100644 index 0000000..ca61d83 --- /dev/null +++ b/crates/lang/src/world/files.rs @@ -0,0 +1,116 @@ +use std::{ + clone, fs, + path::{Path, PathBuf}, + sync::{Arc, Mutex, RwLock}, +}; + +use dashmap::DashMap; +use rowan::ast::{AstNode, AstPtr}; + +use crate::lst_parser::{ + error::SyntaxError, grammar::source_file, input::Input, output::Output, syntax_kind, Parser, +}; + +use super::{error::Error, modules::Module, nodes}; + +pub struct Loc { + file: FileId, + syntax: AstPtr, +} + +#[derive(Clone)] +pub struct Files(Arc); + +impl Files { + pub fn new() -> Self { + Self(Arc::new(FilesInner::new())) + } + + pub fn add_and_parse(&self, file: &Path) -> Result<(FileId, Vec), std::io::Error> { + self.0.add_and_parse(file) + } + + pub fn get(&self, id: FileId) -> Arc { + self.0.get(id) + } +} + +/// global file store +/// contains all known files etc. +struct FilesInner { + paths: Mutex>, + store: DashMap>, +} + +impl FilesInner { + fn new() -> Self { + Self { + paths: Mutex::new(Vec::new()), + store: DashMap::new(), + } + } + + fn add_and_parse(&self, path: &Path) -> Result<(FileId, Vec), std::io::Error> { + let (file, errors) = SourceFile::read_and_parse(&path)?; + + // add file to paths and unlock again + let id = { + let path = path.clone(); + let mut paths = self.paths.lock().unwrap(); + let r = paths.len(); + paths.push(path.to_path_buf()); + FileId(r) + }; + + let _ = self.store.insert(path.to_path_buf(), Arc::new(file)); + + Ok(( + id, + errors + .into_iter() + .map(|e| Error::from_syntax(id, e)) + .collect(), + )) + } + + fn get(&self, id: FileId) -> Arc { + let path = { + let paths = self.paths.lock().unwrap(); + paths[id.0].clone() + }; + + self.store.get(&path).unwrap().clone() + } +} + +pub struct SourceFile { + pub lst: Mutex, + root_module: Option>, +} + +impl SourceFile { + fn read_and_parse(path: &Path) -> Result<(Self, Vec), std::io::Error> { + let source_text = fs::read_to_string(&path)?; + + let toks = syntax_kind::lex(&source_text); + let input = Input::new(&toks); + let mut parser = Parser::new(input); + + source_file(&mut parser); + + let events = parser.finish(); + let out = Output::from_parser_output(toks, events); + let lst = out.syntax(); + + Ok(( + Self { + lst: Mutex::new(nodes::Root::cast(lst).unwrap()), + root_module: None, + }, + out.errors(), + )) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct FileId(pub usize); diff --git a/crates/lang/src/world/modules.rs b/crates/lang/src/world/modules.rs new file mode 100644 index 0000000..0f6d060 --- /dev/null +++ b/crates/lang/src/world/modules.rs @@ -0,0 +1,38 @@ +use std::sync::Arc; + +use dashmap::{DashMap, DashSet}; +use rowan::ast::AstNode; + +use super::{ + files::{FileId, Loc}, + nodes, + registry::ItemPath, + world_creation_pool::WorkerCtx, + // world_creation_pool::WorkerCtx, +}; + +pub struct Module { + /// is none if this is the root module of the entry point file + decl: Option>, + body: ModuleBody, + own_path: ItemPath, + child_modules: Arc>>, + child_defs: Arc>, +} + +impl Module { + pub fn parse_file(ctx: WorkerCtx, file: FileId, decl: Option>) { + let f = ctx.files.get(file); + + let tree = f.lst.lock().unwrap(); + + let children = (&*tree).syntax().children(); + } +} + +struct Def; + +enum ModuleBody { + InLine(Loc), + File(FileId), +} diff --git a/crates/lang/src/world/nodes.rs b/crates/lang/src/world/nodes.rs new file mode 100644 index 0000000..c77399a --- /dev/null +++ b/crates/lang/src/world/nodes.rs @@ -0,0 +1,77 @@ +use crate::lst_parser::syntax_kind::SyntaxKind::*; +use crate::SyntaxNode; +use rowan::Language; +macro_rules! ast_nodes { +($($ast:ident, $kind:ident);+) => { + $( + #[derive(Debug, PartialEq, Eq, Hash)] + #[repr(transparent)] + pub struct $ast(SyntaxNode); + impl rowan::ast::AstNode for $ast { + type Language = crate::Lang; + + fn can_cast(kind: ::Kind) -> bool { + kind == $kind + } + + fn cast(node: SyntaxNode) -> Option { + if node.kind() == $kind { + Some(Self(node)) + } else { + None + } + } + + fn syntax(&self) -> &SyntaxNode { + &self.0 + } + } + )+ +}; +} +ast_nodes!( + Def, DEF; + DefName, DEF_NAME; + DefBody, DEF_BODY; + + Mod, MODULE; + ModName, MODULE_NAME; + ModBody, MODULE_BODY; + + Use, USE; + UsePat, USE_PAT; + PatItem, PAT_ITEM; + PatGlob, PAT_GLOB; + PatGroup, PAT_GROUP; + + Literal, LITERAL; + IntLit, INT_NUM; + FloatLit, FLOAT_NUM; + StringLit, STRING; + + Matrix, MATRIX; + MatrixRow, MAT_ROW; + Vector, VEC; + List, LIST; + CollectionItem, COLLECTION_ITEM; + + ParenthesizedExpr, PARENTHESIZED_EXPR; + Expression, EXPR; + + Pipeline, PIPELINE; + + Instruction, INSTR; + InstructionName, INSTR_NAME; + InstructionParams, INSTR_PARAMS; + + AttributeSet, ATTR_SET; + Attribute, ATTR; + AttributeName, ATTR_NAME; + AttributeValue, ATTR_VALUE; + + ParseError, PARSE_ERR; + LexError, LEX_ERR; + + Root, ROOT; + Eof, EOF +); diff --git a/crates/lang/src/world/registry.rs b/crates/lang/src/world/registry.rs new file mode 100644 index 0000000..791be15 --- /dev/null +++ b/crates/lang/src/world/registry.rs @@ -0,0 +1,41 @@ +use std::sync::{Arc, Mutex, RwLock}; + +use dashmap::{DashMap, DashSet}; + +use super::modules::Module; + +pub type ItemPath = Vec; + +#[derive(Clone)] +pub struct Registry(Arc); + +impl Registry { + pub fn new() -> Self { + Self(Arc::new(RegistryInner::new())) + } +} + +struct RegistryInner { + /// ids will refer to paths of modules + paths: RwLock>, + modules: DashMap>, + names: DashMap>, +} + +impl RegistryInner { + fn new() -> Self { + Self { + paths: RwLock::new(Vec::new()), + modules: DashMap::new(), + names: DashMap::new(), + } + } +} + +pub enum RegistryId { + /// refer to the modlue item itself, the name must be the same as the last item of the path + Module(usize), + /// refer to a def in a module + /// name must refer to name of child of module + Def(usize), +} diff --git a/crates/lang/src/world/world_creation_pool.rs b/crates/lang/src/world/world_creation_pool.rs new file mode 100644 index 0000000..fd04e65 --- /dev/null +++ b/crates/lang/src/world/world_creation_pool.rs @@ -0,0 +1,82 @@ +use std::{path::PathBuf, thread}; + +use crossbeam::channel::{Receiver, Sender}; + +use super::{ + error::Error, + files::{FileId, Files, Loc}, + nodes, + registry::Registry, +}; + +pub(super) struct WorldCreationPool { + workers: Vec, + tx: Sender, +} + +impl WorldCreationPool { + pub fn new(files: Files, reg: Registry) -> Self { + let mut workers = Vec::new(); + let (tx, rx) = crossbeam::channel::unbounded(); + + for i in 0..num_cpus::get() { + workers.push(Worker::new( + i, + files.clone(), + reg.clone(), + JobSender(tx.clone()), + rx.clone(), + )) + } + + Self { workers, tx } + } +} + +pub struct JobSender(Sender); + +enum Job { + ParseFileMod { + file: FileId, + decl: Option>, + }, + OpenFile(PathBuf), +} + +struct Worker { + id: usize, + // thead: thread::JoinHandle>, + thread: thread::JoinHandle<()>, +} + +pub struct WorkerCtx { + errors: Vec, + pub files: Files, + pub reg: Registry, + pub tx: JobSender, +} + +impl Worker { + fn new(id: usize, files: Files, reg: Registry, sender: JobSender, rx: Receiver) -> Self { + let ctx = WorkerCtx { + errors: Vec::new(), + files, + reg, + tx: sender, + }; + + let thread_handle = thread::spawn(move || { + for msg in rx { + match msg { + Job::ParseFileMod { file, decl } => todo!(), + Job::OpenFile(path) => todo!(), + } + } + }); + + Self { + id, + thread: thread_handle, + } + } +}