From 303c7b00940141c8395b15d41ec6a8b05ddb2c2b Mon Sep 17 00:00:00 2001 From: Arthur Cohen Date: Wed, 22 Feb 2023 12:11:28 +0100 Subject: [PATCH 1/4] interpolate: Add base package and tests --- Cargo.lock | 11 ++++ Cargo.toml | 2 + interpolate/Cargo.toml | 12 +++++ interpolate/src/lib.rs | 112 +++++++++++++++++++++++++++++++++++++++++ interpreter/jinko.rs | 2 + 5 files changed, 139 insertions(+) create mode 100644 interpolate/Cargo.toml create mode 100644 interpolate/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 52e93358..f6f023c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -356,6 +356,16 @@ dependencies = [ "xparser", ] +[[package]] +name = "interpolate" +version = "0.1.0" +dependencies = [ + "ast", + "error", + "location", + "symbol", +] + [[package]] name = "jinko" version = "0.3.0-jinx7" @@ -372,6 +382,7 @@ dependencies = [ "fire", "flatten", "include_code", + "interpolate", "lazy_static", "libc", "libffi", diff --git a/Cargo.toml b/Cargo.toml index 5549db76..5ef3c9d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "location", "symbol", "ast", + "interpolate", "fir", "flatten", "name_resolve", @@ -33,6 +34,7 @@ members = [ [dependencies] ast = { path = "ast" } ast-sanitizer = { path = "ast-sanitizer" } +interpolate = { path = "interpolate" } fir = { path = "fir" } debug-fir = { path = "debug-fir" } flatten = { path = "flatten" } diff --git a/interpolate/Cargo.toml b/interpolate/Cargo.toml new file mode 100644 index 00000000..36e32d64 --- /dev/null +++ b/interpolate/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "interpolate" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +ast = { path = "../ast" } +error = { path = "../error" } +symbol = { path = "../symbol" } +location = { path = "../location" } diff --git a/interpolate/src/lib.rs b/interpolate/src/lib.rs new file mode 100644 index 00000000..9cf76eea --- /dev/null +++ b/interpolate/src/lib.rs @@ -0,0 +1,112 @@ +use ast::{Ast, Node, Value, Visitor}; +use error::{ErrKind, Error}; +use location::SpanTuple; + +struct Ctx; + +fn parse_format_string(to_parse: &str) -> Result, Error> { + todo!() +} + +impl Visitor for Ctx { + fn visit_constant(&mut self, location: SpanTuple, value: Value) -> Result { + let s = match value { + Value::Str(s) => s, + _ => todo!(), + }; + + Error::new(ErrKind::Hint) + .with_loc(Some(location.clone())) + .with_msg(format!("saw a string: {s}!")) + .emit_debug(); + + Ok(Ast { + location, + node: Node::Constant(Value::Str(s)), + }) + } +} + +pub trait Interpolator: Sized { + fn interpolate(self) -> Result; +} + +impl Interpolator for Ast { + fn interpolate(self) -> Result { + Ctx.visit(self) + } +} + +pub fn add(left: usize, right: usize) -> usize { + left + right +} + +#[cfg(test)] +mod tests { + use super::*; + use ast::{Node::*, Operator, Value::*}; + + macro_rules! loc { + () => { + location::SpanTuple::with_source( + location::SourceOwned::Empty, + location::Location::new(1, 0), + location::Location::new(1, 0), + ) + }; + } + + macro_rules! ast { + ($n:expr) => { + Ast { + location: loc!(), + node: $n, + } + }; + } + + #[test] + fn parse_one() { + let s = "hello"; + + let expected = ast! { + Constant(Str(String::from("hello"))) + }; + + assert_eq!(parse_format_string(s).unwrap()[0].node, expected.node) + } + + #[test] + fn parse_one_expr() { + let s = "{15}"; + + let expected = ast! { + Constant(Integer(15)) + }; + + assert_eq!(parse_format_string(s).unwrap()[0].node, expected.node) + } + + #[test] + fn parse_one_expr_one_string() { + let s = "hello {15 + 4}"; + + let expected_s = ast! { + Constant(Str(String::from("hello "))) + }; + let expected_expr = ast! { + BinaryOp( + Operator::Add, + Box::new(ast! { + Constant(Integer(15)) + }), + Box::new(ast! { + Constant(Integer(4)) + }), + ) + }; + + assert_eq!(parse_format_string(s).unwrap()[0].node, expected_s.node); + assert_eq!(parse_format_string(s).unwrap()[1].node, expected_expr.node); + } +} diff --git a/interpreter/jinko.rs b/interpreter/jinko.rs index 92d07e75..6adfebac 100644 --- a/interpreter/jinko.rs +++ b/interpreter/jinko.rs @@ -11,6 +11,7 @@ use fire::instance::Instance; use fire::Interpret; use flatten::{FlattenAst, FlattenData}; use include_code::IncludeCode; +use interpolate::Interpolator; use loop_desugar::DesugarLoops; use name_resolve::NameResolve; @@ -123,6 +124,7 @@ fn experimental_pipeline(input: &str, file: &Path) -> InteractResult { let ast = x_try!(ast.desugar_loops()); let ast = x_try!(ast_sanitizer::only_while_loops(ast)); + let ast = x_try!(ast.interpolate()); let ast = x_try!(ast.resolve_includes()); let ast = x_try!(ast_sanitizer::no_incl(ast)); From 34125e190d2e3ee3573d2b1e54413029bd637985 Mon Sep 17 00:00:00 2001 From: Arthur Cohen Date: Wed, 22 Feb 2023 12:20:08 +0100 Subject: [PATCH 2/4] xparser: Expose public functions --- Cargo.lock | 1 + interpolate/Cargo.toml | 1 + xparser/src/lib.rs | 3 +++ 3 files changed, 5 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index f6f023c4..5b3b01e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -364,6 +364,7 @@ dependencies = [ "error", "location", "symbol", + "xparser", ] [[package]] diff --git a/interpolate/Cargo.toml b/interpolate/Cargo.toml index 36e32d64..e92a0243 100644 --- a/interpolate/Cargo.toml +++ b/interpolate/Cargo.toml @@ -10,3 +10,4 @@ ast = { path = "../ast" } error = { path = "../error" } symbol = { path = "../symbol" } location = { path = "../location" } +xparser = { path = "../xparser" } diff --git a/xparser/src/lib.rs b/xparser/src/lib.rs index 819be3fb..70e396c2 100644 --- a/xparser/src/lib.rs +++ b/xparser/src/lib.rs @@ -1,6 +1,9 @@ mod constructs; mod tokens; +pub use constructs::*; +pub use tokens::*; + use ast::Ast; use location::Source; From 37e14cb0ed4e1c9e8e9ba2ec8be5682d327cfcc1 Mon Sep 17 00:00:00 2001 From: Arthur Cohen Date: Wed, 1 Mar 2023 22:58:43 +0100 Subject: [PATCH 3/4] interpolate: wip: Add base for string parser --- Cargo.lock | 2 ++ interpolate/Cargo.toml | 2 ++ interpolate/src/lib.rs | 26 +++++++++++++++++++++++++- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 5b3b01e8..28bd6386 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -363,6 +363,8 @@ dependencies = [ "ast", "error", "location", + "nom 7.1.1", + "nom_locate", "symbol", "xparser", ] diff --git a/interpolate/Cargo.toml b/interpolate/Cargo.toml index e92a0243..ce303d48 100644 --- a/interpolate/Cargo.toml +++ b/interpolate/Cargo.toml @@ -11,3 +11,5 @@ error = { path = "../error" } symbol = { path = "../symbol" } location = { path = "../location" } xparser = { path = "../xparser" } +nom = "7.0" +nom_locate = "4.0" diff --git a/interpolate/src/lib.rs b/interpolate/src/lib.rs index 9cf76eea..459ec11f 100644 --- a/interpolate/src/lib.rs +++ b/interpolate/src/lib.rs @@ -1,10 +1,34 @@ use ast::{Ast, Node, Value, Visitor}; use error::{ErrKind, Error}; -use location::SpanTuple; +use location::{Source, SpanTuple}; +use xparser::{ParseInput, ParseResult}; + +use nom::bytes::complete::take_until; +use nom::character::complete::char; +use nom_locate::{position, LocatedSpan}; struct Ctx; +fn parse_expr(to_parse: ParseInput) -> ParseResult { + let (input, _) = char('{')(to_parse)?; + let (input, expr) = xparser::expr(input)?; + let (input, _) = char('}')(input)?; + + Ok((input, expr)) +} + +fn parse_str(to_parse: ParseInput) -> ParseResult { + // how to handle reaching the end of the string? + take_until("{")(to_parse) +} + +fn parse_inner(to_parse: &str) -> ParseResult { + todo!() +} + +/// This function parses the following grammar: `double_quote ( .* ( '{' expr '}' )* )* double_quote` fn parse_format_string(to_parse: &str) -> Result, Error> { + let input = LocatedSpan::new_extra(to_parse, Source::Input(to_parse)); todo!() } From c29fea2ba14e2d7772215f870b4179e9fee8b6b1 Mon Sep 17 00:00:00 2001 From: Arthur Cohen Date: Fri, 3 Mar 2023 22:47:21 +0100 Subject: [PATCH 4/4] wip: Add more to interpolation parser --- interpolate/src/lib.rs | 72 +++++++++++++++++++++++++++++++++++------- 1 file changed, 61 insertions(+), 11 deletions(-) diff --git a/interpolate/src/lib.rs b/interpolate/src/lib.rs index 459ec11f..c68fb6a6 100644 --- a/interpolate/src/lib.rs +++ b/interpolate/src/lib.rs @@ -1,42 +1,89 @@ use ast::{Ast, Node, Value, Visitor}; use error::{ErrKind, Error}; -use location::{Source, SpanTuple}; -use xparser::{ParseInput, ParseResult}; +use location::{Location, Source, SpanTuple}; +use nom::multi::many0; +use xparser::{Error as ParseError, ParseInput, ParseResult}; -use nom::bytes::complete::take_until; use nom::character::complete::char; +use nom::{bytes::complete::take_until, combinator::opt}; use nom_locate::{position, LocatedSpan}; struct Ctx; -fn parse_expr(to_parse: ParseInput) -> ParseResult { - let (input, _) = char('{')(to_parse)?; +fn parse_expr(input: ParseInput) -> ParseResult { + let (input, _) = char('{')(input)?; let (input, expr) = xparser::expr(input)?; let (input, _) = char('}')(input)?; Ok((input, expr)) } -fn parse_str(to_parse: ParseInput) -> ParseResult { +fn parse_str(input: ParseInput) -> ParseResult { // how to handle reaching the end of the string? - take_until("{")(to_parse) + take_until("{")(input) } -fn parse_inner(to_parse: &str) -> ParseResult { - todo!() +pub(crate) fn pos_to_loc( + input: ParseInput, + start: impl Into, + end: impl Into, +) -> SpanTuple { + SpanTuple::with_source_ref(input.extra, start.into(), end.into()) +} + +fn parse_inner(input: ParseInput) -> ParseResult { + let (input, start) = position(input)?; + let (input, s) = opt(parse_str)(input)?; + let (input, end) = position(input)?; + if let Some(s) = s { + return Ok(( + input, + Ast { + location: pos_to_loc(input, start, end), + node: Node::Constant(Value::Str(s.fragment().to_string())), + }, + )); + } + + let (input, s) = opt(parse_expr)(input)?; + let (input, end) = position(input)?; + if let Some(expr) = s { + return Ok((input, expr)); + } + + let loc = pos_to_loc(input, start, end); + + Error::new(ErrKind::Parsing) + .with_msg(format!("unexpected character in format string: {input}")) + .with_loc(Some(loc)) + .emit(); + + Err(nom::Err::Error(ParseError)) } /// This function parses the following grammar: `double_quote ( .* ( '{' expr '}' )* )* double_quote` fn parse_format_string(to_parse: &str) -> Result, Error> { let input = LocatedSpan::new_extra(to_parse, Source::Input(to_parse)); - todo!() + + let res = many0(parse_inner)(input); + + if let Ok((_, exprs)) = res { + Ok(exprs) + } else { + Err(Error::new(ErrKind::Parsing)) + } } impl Visitor for Ctx { fn visit_constant(&mut self, location: SpanTuple, value: Value) -> Result { let s = match value { Value::Str(s) => s, - _ => todo!(), + any_const => { + return Ok(Ast { + location, + node: Node::Constant(any_const), + }) + } }; Error::new(ErrKind::Hint) @@ -44,6 +91,9 @@ impl Visitor for Ctx { .with_msg(format!("saw a string: {s}!")) .emit_debug(); + let exprs = parse_format_string(&s)?; + dbg!(exprs); + Ok(Ast { location, node: Node::Constant(Value::Str(s)),