From 0861c6956762604766b08e2c70741f4e71686195 Mon Sep 17 00:00:00 2001 From: WATANABE Yuki Date: Mon, 30 Dec 2024 15:36:33 +0900 Subject: [PATCH 1/9] Make Lexer must_use --- yash-syntax/CHANGELOG.md | 1 + yash-syntax/src/parser/lex/core.rs | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/yash-syntax/CHANGELOG.md b/yash-syntax/CHANGELOG.md index 71f698c0..4385f52a 100644 --- a/yash-syntax/CHANGELOG.md +++ b/yash-syntax/CHANGELOG.md @@ -30,6 +30,7 @@ more flexible and readable configurations. - When a simple command is parsed, the parser now checks if the command name is a declaration utility. If it is, following words in an assignment form are parsed like assignments. +- The `parser::lex::Lexer` struct is now `#[must_use]`. ## [0.13.0] - 2024-12-14 diff --git a/yash-syntax/src/parser/lex/core.rs b/yash-syntax/src/parser/lex/core.rs index 74af9da6..8c8e8624 100644 --- a/yash-syntax/src/parser/lex/core.rs +++ b/yash-syntax/src/parser/lex/core.rs @@ -441,6 +441,7 @@ impl fmt::Debug for LexerCore<'_> { /// [`skip_blanks_and_comment`](Lexer::skip_blanks_and_comment) depend on those primitives to /// parse more complex structures in the source code. #[derive(Debug)] +#[must_use] pub struct Lexer<'a> { // `Lexer` is a thin wrapper around `LexerCore`. `Lexer` delegates most // functions to `LexerCore`. `Lexer` adds automatic line-continuation @@ -451,7 +452,6 @@ pub struct Lexer<'a> { impl<'a> Lexer<'a> { /// Creates a new lexer that reads using the given input function. - #[must_use] pub fn new( input: Box, start_line_number: NonZeroU64, @@ -467,7 +467,6 @@ impl<'a> Lexer<'a> { /// /// This is a convenience function that creates a lexer that reads from a /// string using a [`Memory`] input function. The line number starts from 1. - #[must_use] pub fn from_memory>>(code: &'a str, source: S) -> Lexer<'a> { fn inner(code: &str, source: Rc) -> Lexer { let line = NonZeroU64::new(1).unwrap(); From a3ed561c5de94a8d2640241fbdd0d035efe24226 Mon Sep 17 00:00:00 2001 From: WATANABE Yuki Date: Mon, 30 Dec 2024 16:06:33 +0900 Subject: [PATCH 2/9] Add lexer Config --- yash-syntax/CHANGELOG.md | 4 +- yash-syntax/src/parser/lex/core.rs | 87 +++++++++++++++++++++++++++++- 2 files changed, 89 insertions(+), 2 deletions(-) diff --git a/yash-syntax/CHANGELOG.md b/yash-syntax/CHANGELOG.md index 4385f52a..e5cea1b6 100644 --- a/yash-syntax/CHANGELOG.md +++ b/yash-syntax/CHANGELOG.md @@ -17,7 +17,9 @@ more flexible and readable configurations. - The `decl_util` module is added, which contains the `Glossary` trait and the `EmptyGlossary` and `PosixGlossary` structs. - Added the `Config` struct to the `parser` module. Currently, it allows - setting glossaries for aliases and declaration utilities. + setting glossaries for aliases and declaration utilities for the parser. +- Added the `Config` struct to the `parser::lex` module. Currently, it allows + setting the starting line number and the source information for the lexer. - The `syntax::Word::parse_tilde_everywhere_after` method is added. ### Changed diff --git a/yash-syntax/src/parser/lex/core.rs b/yash-syntax/src/parser/lex/core.rs index 8c8e8624..db68a0cd 100644 --- a/yash-syntax/src/parser/lex/core.rs +++ b/yash-syntax/src/parser/lex/core.rs @@ -430,6 +430,64 @@ impl fmt::Debug for LexerCore<'_> { } } +/// Configuration for the [lexer](Lexer) +/// +/// `Config` is a builder for the lexer. A [new](Self::new) instance is created +/// with default settings. You can then customize the settings by modifying the +/// corresponding fields. Finally, you can pass an input object to the +/// [`input`](Self::input) method to create a lexer. +#[derive(Debug)] +#[must_use = "you must call `input` to create a lexer"] +#[non_exhaustive] +pub struct Config { + /// Line number for the first line of the input + /// + /// The lexer counts the line number from this value to annotate the + /// location of the tokens. The line number is saved in the + /// `start_line_number` field of the [`Code`] instance that is contained in + /// the [`Location`] instance of the token. + /// + /// The default value is 1. + pub start_line_number: NonZeroU64, + + /// Source of the input + /// + /// The source is used to annotate the location of the tokens. This value + /// is saved in the `source` field of the [`Code`] instance that is + /// contained in the [`Location`] instance of the token. + /// + /// The default value is `None`, in which case the source is set to + /// [`Source::Unknown`]. + pub source: Option>, +} + +impl Config { + /// Creates a new configuration with default settings. + /// + /// You can also call [`Lexer::config`] to create a new configuration. + pub fn new() -> Self { + Config { + start_line_number: NonZeroU64::MIN, + source: None, + } + } + + /// Creates a lexer with the given input object. + pub fn input<'a>(self, input: Box) -> Lexer<'a> { + Lexer::new( + input, + self.start_line_number, + self.source.unwrap_or_else(|| Rc::new(Source::Unknown)), + ) + } +} + +impl Default for Config { + fn default() -> Self { + Self::new() + } +} + /// Lexical analyzer. /// /// A lexer reads lines using an input function and parses the characters into tokens. It has an @@ -439,7 +497,25 @@ impl fmt::Debug for LexerCore<'_> { /// `Lexer` has primitive functions such as [`peek_char`](Lexer::peek_char) that provide access /// to the character at the current position. Derived functions such as /// [`skip_blanks_and_comment`](Lexer::skip_blanks_and_comment) depend on those primitives to -/// parse more complex structures in the source code. +/// parse more complex structures in the source code. Usually, the lexer is used by a +/// [parser](super::super::Parser) to read the source code and produce a syntax +/// tree, so you don't need to call these functions directly. +/// +/// To construct a lexer, you can use the [`Lexer::new`] function with an input object. +/// You can also use the [`Lexer::config`] function to create a configuration that allows you to +/// customize the settings before creating a lexer. +/// +/// ``` +/// # use yash_syntax::input::Memory; +/// # use yash_syntax::parser::{lex::Lexer, Parser}; +/// # use yash_syntax::source::Source; +/// let mut config = Lexer::config(); +/// config.start_line_number = 10.try_into().unwrap(); +/// config.source = Some(Source::CommandString.into()); +/// let mut lexer = config.input(Box::new(Memory::new("echo hello\n"))); +/// let mut parser = Parser::new(&mut lexer); +/// _ = parser.command_line(); +/// ``` #[derive(Debug)] #[must_use] pub struct Lexer<'a> { @@ -451,6 +527,15 @@ pub struct Lexer<'a> { } impl<'a> Lexer<'a> { + /// Creates a new configuration with default settings. + /// + /// This is a synonym for [`Config::new`]. You can modify the settings and + /// then create a lexer with the [`input`](Config::input) method. + #[inline(always)] + pub fn config() -> Config { + Config::new() + } + /// Creates a new lexer that reads using the given input function. pub fn new( input: Box, From 305fd4e6b14402ebf8ba4e69410872b1ca4d6e0a Mon Sep 17 00:00:00 2001 From: WATANABE Yuki Date: Mon, 30 Dec 2024 16:51:22 +0900 Subject: [PATCH 3/9] Remove start_line_number and source arguments from Lexer::new The Lexer::new method now only takes an input object. The start_line_number and source arguments have been removed in favor of construction with a Config struct. The two arguments are not essential for the lexer's operation, so they should not be required in the convenience constructor. --- yash-builtin/src/eval.rs | 10 ++++----- yash-builtin/src/source/semantics.rs | 11 +++++----- yash-cli/src/lib.rs | 6 +++--- yash-cli/src/startup/init_file.rs | 11 +++++----- yash-prompt/src/lib.rs | 5 +++-- yash-semantics/src/runner.rs | 13 +++--------- yash-semantics/src/runner_legacy.rs | 9 ++------ yash-syntax/CHANGELOG.md | 3 +++ yash-syntax/src/parser.rs | 6 +----- yash-syntax/src/parser/lex/core.rs | 31 ++++++++++++++-------------- yash-syntax/src/parser/lex/op.rs | 9 +------- 11 files changed, 45 insertions(+), 69 deletions(-) diff --git a/yash-builtin/src/eval.rs b/yash-builtin/src/eval.rs index b427ee05..ede2f5e9 100644 --- a/yash-builtin/src/eval.rs +++ b/yash-builtin/src/eval.rs @@ -61,7 +61,6 @@ use crate::Result; use std::cell::RefCell; -use std::num::NonZeroU64; use std::rc::Rc; #[cfg(doc)] use yash_env::semantics::ExitStatus; @@ -80,12 +79,11 @@ pub async fn main(env: &mut Env, args: Vec) -> Result { }; // Parse and execute the command string - let input = Box::new(Memory::new(&command.value)); - let start_line_number = NonZeroU64::new(1).unwrap(); - let source = Rc::new(Source::Eval { + let mut config = Lexer::config(); + config.source = Some(Rc::new(Source::Eval { original: command.origin, - }); - let mut lexer = Lexer::new(input, start_line_number, source); + })); + let mut lexer = config.input(Box::new(Memory::new(&command.value))); let divert = read_eval_loop(&RefCell::new(env), &mut lexer).await; Result::with_exit_status_and_divert(env.exit_status, divert) } diff --git a/yash-builtin/src/source/semantics.rs b/yash-builtin/src/source/semantics.rs index f06e457e..419ccb14 100644 --- a/yash-builtin/src/source/semantics.rs +++ b/yash-builtin/src/source/semantics.rs @@ -21,7 +21,6 @@ use crate::common::report_failure; use std::cell::RefCell; use std::ffi::CStr; use std::ffi::CString; -use std::num::NonZeroU64; use std::ops::ControlFlow; use std::rc::Rc; use yash_env::input::Echo; @@ -65,13 +64,13 @@ impl Command { // Parse and execute the command script let system = env.system.clone(); let ref_env = RefCell::new(&mut *env); - let input = Box::new(Echo::new(FdReader::new(fd, system), &ref_env)); - let start_line_number = NonZeroU64::new(1).unwrap(); - let source = Rc::new(Source::DotScript { + let mut config = Lexer::config(); + config.source = Some(Rc::new(Source::DotScript { name: self.file.value, origin: self.file.origin, - }); - let mut lexer = Lexer::new(input, start_line_number, source); + })); + let input = Box::new(Echo::new(FdReader::new(fd, system), &ref_env)); + let mut lexer = config.input(input); let divert = read_eval_loop(&ref_env, &mut { lexer }).await; _ = env.system.close(fd); diff --git a/yash-cli/src/lib.rs b/yash-cli/src/lib.rs index 7ec20820..39eec5b7 100644 --- a/yash-cli/src/lib.rs +++ b/yash-cli/src/lib.rs @@ -31,7 +31,6 @@ use self::startup::init_file::run_rcfile; use self::startup::input::prepare_input; use self::startup::input::SourceInput; use std::cell::RefCell; -use std::num::NonZeroU64; use std::ops::ControlFlow::{Break, Continue}; use yash_env::option::{Interactive, On}; use yash_env::signal; @@ -95,8 +94,9 @@ async fn parse_and_print(mut env: Env) -> ExitStatus { }; } }; - let line = NonZeroU64::new(1).unwrap(); - let mut lexer = Lexer::new(input, line, source.into()); + let mut config = Lexer::config(); + config.source = Some(source.into()); + let mut lexer = config.input(input); // Run the read-eval loop let result = if is_interactive { diff --git a/yash-cli/src/startup/init_file.rs b/yash-cli/src/startup/init_file.rs index 165795aa..193b1921 100644 --- a/yash-cli/src/startup/init_file.rs +++ b/yash-cli/src/startup/init_file.rs @@ -30,7 +30,6 @@ use super::args::InitFile; use std::cell::RefCell; use std::ffi::CString; -use std::num::NonZeroU64; use std::rc::Rc; use thiserror::Error; use yash_env::input::{Echo, FdReader}; @@ -171,12 +170,12 @@ pub async fn run_init_file(env: &mut Env, path: &str) { let env = &mut *env.push_frame(Frame::InitFile); let system = env.system.clone(); let ref_env = RefCell::new(&mut *env); - let input = Box::new(Echo::new(FdReader::new(fd, system), &ref_env)); - let start_line_number = NonZeroU64::MIN; - let source = Rc::new(Source::InitFile { + let mut config = Lexer::config(); + config.source = Some(Rc::new(Source::InitFile { path: path.to_owned(), - }); - let mut lexer = Lexer::new(input, start_line_number, source); + })); + let input = Box::new(Echo::new(FdReader::new(fd, system), &ref_env)); + let mut lexer = config.input(input); read_eval_loop(&ref_env, &mut { lexer }).await; if let Err(errno) = env.system.close(fd) { diff --git a/yash-prompt/src/lib.rs b/yash-prompt/src/lib.rs index 8498c94c..e3e85154 100644 --- a/yash-prompt/src/lib.rs +++ b/yash-prompt/src/lib.rs @@ -43,7 +43,6 @@ //! # use futures_util::future::FutureExt as _; //! # async { //! use std::cell::RefCell; -//! use std::num::NonZeroU64; //! use std::ops::ControlFlow::Continue; //! use yash_env::Env; //! use yash_env::input::FdReader; @@ -58,7 +57,9 @@ //! let reader = FdReader::new(Fd::STDIN, env.system.clone()); //! let mut ref_env = RefCell::new(&mut env); //! let input = Box::new(Prompter::new(reader, &ref_env)); -//! let mut lexer = Lexer::new(input, NonZeroU64::MIN, Source::Stdin.into()); +//! let mut config = Lexer::config(); +//! config.source = Some(Source::Stdin.into()); +//! let mut lexer = config.input(input); //! let result = read_eval_loop(&ref_env, &mut lexer).await; //! drop(lexer); //! assert_eq!(result, Continue(())); diff --git a/yash-semantics/src/runner.rs b/yash-semantics/src/runner.rs index dc9cbb75..1559114b 100644 --- a/yash-semantics/src/runner.rs +++ b/yash-semantics/src/runner.rs @@ -79,20 +79,17 @@ use yash_syntax::syntax::List; /// ``` /// # futures_executor::block_on(async { /// # use std::cell::RefCell; -/// # use std::num::NonZeroU64; /// # use std::ops::ControlFlow::Continue; -/// # use std::rc::Rc; /// # use yash_env::Env; /// # use yash_env::input::Echo; /// # use yash_semantics::ExitStatus; /// # use yash_semantics::read_eval_loop; /// # use yash_syntax::input::Memory; /// # use yash_syntax::parser::lex::Lexer; -/// # use yash_syntax::source::Source; /// let mut env = Env::new_virtual(); /// let mut ref_env = RefCell::new(&mut env); /// let input = Box::new(Echo::new(Memory::new("case foo in (bar) ;; esac"), &ref_env)); -/// let mut lexer = Lexer::new(input, NonZeroU64::MIN, Rc::new(Source::CommandString)); +/// let mut lexer = Lexer::new(input); /// let result = read_eval_loop(&ref_env, &mut lexer).await; /// drop(lexer); /// assert_eq!(result, Continue(())); @@ -206,7 +203,6 @@ mod tests { use crate::tests::echo_builtin; use crate::tests::return_builtin; use futures_util::FutureExt; - use std::num::NonZeroU64; use std::rc::Rc; use yash_env::input::Echo; use yash_env::input::Memory; @@ -294,8 +290,7 @@ mod tests { env.options.set(Verbose, On); let ref_env = RefCell::new(&mut env); let input = Box::new(Echo::new(Memory::new("case _ in esac"), &ref_env)); - let line = NonZeroU64::new(1).unwrap(); - let mut lexer = Lexer::new(input, line, Rc::new(Source::CommandString)); + let mut lexer = Lexer::new(input); let result = read_eval_loop(&ref_env, &mut lexer).now_or_never().unwrap(); drop(lexer); @@ -410,9 +405,7 @@ mod tests { } } - let input = Box::new(BrokenInput); - let line = NonZeroU64::new(1).unwrap(); - let mut lexer = Lexer::new(input, line, Rc::new(Source::Unknown)); + let mut lexer = Lexer::new(Box::new(BrokenInput)); let mut env = Env::new_virtual(); let ref_env = RefCell::new(&mut env); diff --git a/yash-semantics/src/runner_legacy.rs b/yash-semantics/src/runner_legacy.rs index 46717c67..5296ab69 100644 --- a/yash-semantics/src/runner_legacy.rs +++ b/yash-semantics/src/runner_legacy.rs @@ -98,7 +98,6 @@ impl<'a, 'b> ReadEvalLoop<'a, 'b> { /// ``` /// # futures_executor::block_on(async { /// # use std::cell::Cell; - /// # use std::num::NonZeroU64; /// # use std::rc::Rc; /// # use yash_env::Env; /// # use yash_env::input::FdReader; @@ -106,13 +105,11 @@ impl<'a, 'b> ReadEvalLoop<'a, 'b> { /// # use yash_env::option::{Verbose, State}; /// # use yash_semantics::*; /// # use yash_syntax::parser::lex::Lexer; - /// # use yash_syntax::source::Source; /// let mut env = Env::new_virtual(); /// let mut input = Box::new(FdReader::new(Fd::STDIN, Clone::clone(&env.system))); /// let verbose = Rc::new(Cell::new(State::Off)); /// input.set_echo(Some(Rc::clone(&verbose))); - /// let line = NonZeroU64::new(1).unwrap(); - /// let mut lexer = Lexer::new(input, line, Rc::new(Source::Stdin)); + /// let mut lexer = Lexer::new(input); /// let mut rel = ReadEvalLoop::new(&mut env, &mut lexer); /// rel.set_verbose(Some(Rc::clone(&verbose))); /// let _ = rel.run().await; @@ -173,7 +170,6 @@ mod tests { use crate::tests::return_builtin; use futures_util::FutureExt; use std::cell::Cell; - use std::num::NonZeroU64; use std::ops::ControlFlow::Break; use std::rc::Rc; use yash_env::input::FdReader; @@ -272,8 +268,7 @@ mod tests { let verbose = Rc::new(Cell::new(Off)); #[allow(deprecated)] input.set_echo(Some(Rc::clone(&verbose))); - let line = NonZeroU64::new(1).unwrap(); - let mut lexer = Lexer::new(input, line, Rc::new(Source::Stdin)); + let mut lexer = Lexer::new(input); #[allow(deprecated)] let mut rel = ReadEvalLoop::new(&mut env, &mut lexer); #[allow(deprecated)] diff --git a/yash-syntax/CHANGELOG.md b/yash-syntax/CHANGELOG.md index e5cea1b6..f9ab8708 100644 --- a/yash-syntax/CHANGELOG.md +++ b/yash-syntax/CHANGELOG.md @@ -33,6 +33,9 @@ more flexible and readable configurations. a declaration utility. If it is, following words in an assignment form are parsed like assignments. - The `parser::lex::Lexer` struct is now `#[must_use]`. +- The `parser::lex::Lexer::new` method now only takes a `Box` + argument. The `start_line_number: NonZeroU64` and `source: Rc` + arguments have been removed in favor of construction with a `Config` struct. ## [0.13.0] - 2024-12-14 diff --git a/yash-syntax/src/parser.rs b/yash-syntax/src/parser.rs index 295c6788..32e6b1cf 100644 --- a/yash-syntax/src/parser.rs +++ b/yash-syntax/src/parser.rs @@ -33,15 +33,11 @@ //! ``` //! // First, prepare an input object that the lexer reads from. //! use yash_syntax::input::Memory; -//! use yash_syntax::source::Source; -//! # // TODO demonstrate with a Source other than Unknown //! let input = Box::new(Memory::new("echo $?")); //! //! // Next, create a lexer. -//! use std::num::NonZeroU64; //! use yash_syntax::parser::lex::Lexer; -//! let line = NonZeroU64::new(1).unwrap(); -//! let mut lexer = Lexer::new(input, line, Source::Unknown.into()); +//! let mut lexer = Lexer::new(input); //! //! // Then, create a new parser borrowing the lexer. //! use yash_syntax::parser::Parser; diff --git a/yash-syntax/src/parser/lex/core.rs b/yash-syntax/src/parser/lex/core.rs index db68a0cd..9021af87 100644 --- a/yash-syntax/src/parser/lex/core.rs +++ b/yash-syntax/src/parser/lex/core.rs @@ -474,11 +474,12 @@ impl Config { /// Creates a lexer with the given input object. pub fn input<'a>(self, input: Box) -> Lexer<'a> { - Lexer::new( - input, - self.start_line_number, - self.source.unwrap_or_else(|| Rc::new(Source::Unknown)), - ) + let start_line_number = self.start_line_number; + let source = self.source.unwrap_or_else(|| Rc::new(Source::Unknown)); + Lexer { + core: LexerCore::new(input, start_line_number, source), + line_continuation_enabled: true, + } } } @@ -537,15 +538,12 @@ impl<'a> Lexer<'a> { } /// Creates a new lexer that reads using the given input function. - pub fn new( - input: Box, - start_line_number: NonZeroU64, - source: Rc, - ) -> Lexer<'a> { - Lexer { - core: LexerCore::new(input, start_line_number, source), - line_continuation_enabled: true, - } + /// + /// This is a convenience function that creates a lexer with the given input + /// object and the default configuration. To customize the configuration, + /// use the [`config`](Self::config) function. + pub fn new(input: Box) -> Lexer<'a> { + Self::config().input(input) } /// Creates a new lexer with a fixed source code. @@ -554,8 +552,9 @@ impl<'a> Lexer<'a> { /// string using a [`Memory`] input function. The line number starts from 1. pub fn from_memory>>(code: &'a str, source: S) -> Lexer<'a> { fn inner(code: &str, source: Rc) -> Lexer { - let line = NonZeroU64::new(1).unwrap(); - Lexer::new(Box::new(Memory::new(code)), line, source) + let mut config = Lexer::config(); + config.source = Some(source); + config.input(Box::new(Memory::new(code))) } inner(code, source.into()) } diff --git a/yash-syntax/src/parser/lex/op.rs b/yash-syntax/src/parser/lex/op.rs index 594c98f2..569ee650 100644 --- a/yash-syntax/src/parser/lex/op.rs +++ b/yash-syntax/src/parser/lex/op.rs @@ -447,8 +447,6 @@ mod tests { use crate::syntax::TextUnit; use crate::syntax::WordUnit; use futures_util::FutureExt; - use std::num::NonZeroU64; - use std::rc::Rc; fn ensure_sorted(trie: &Trie) { assert!( @@ -559,12 +557,7 @@ mod tests { } } - let line_number = NonZeroU64::new(1).unwrap(); - let mut lexer = Lexer::new( - Box::new(OneLineInput(Some("\n".to_owned()))), - line_number, - Rc::new(Source::Unknown), - ); + let mut lexer = Lexer::new(Box::new(OneLineInput(Some("\n".to_owned())))); let t = lexer.operator().now_or_never().unwrap().unwrap().unwrap(); assert_eq!(t.word.units, [WordUnit::Unquoted(TextUnit::Literal('\n'))]); From c773d7ca26977821e580f04cfed7564a4694c8e3 Mon Sep 17 00:00:00 2001 From: WATANABE Yuki Date: Mon, 30 Dec 2024 17:22:45 +0900 Subject: [PATCH 4/9] Implement From<&str> for Memory --- yash-syntax/CHANGELOG.md | 1 + yash-syntax/src/input.rs | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/yash-syntax/CHANGELOG.md b/yash-syntax/CHANGELOG.md index f9ab8708..d9b69522 100644 --- a/yash-syntax/CHANGELOG.md +++ b/yash-syntax/CHANGELOG.md @@ -21,6 +21,7 @@ more flexible and readable configurations. - Added the `Config` struct to the `parser::lex` module. Currently, it allows setting the starting line number and the source information for the lexer. - The `syntax::Word::parse_tilde_everywhere_after` method is added. +- The `From<&str>` trait is now implemented for `input::Memory`. ### Changed diff --git a/yash-syntax/src/input.rs b/yash-syntax/src/input.rs index 5af5903d..8d26f8ee 100644 --- a/yash-syntax/src/input.rs +++ b/yash-syntax/src/input.rs @@ -128,6 +128,12 @@ impl Memory<'_> { } } +impl<'a> From<&'a str> for Memory<'a> { + fn from(code: &'a str) -> Memory<'a> { + Memory::new(code) + } +} + impl Input for Memory<'_> { async fn next_line(&mut self, _context: &Context) -> Result { Ok(self.lines.next().unwrap_or("").to_owned()) From 5523609fde8ad5d48fa625ffeb544a2d9a60c967 Mon Sep 17 00:00:00 2001 From: WATANABE Yuki Date: Mon, 30 Dec 2024 18:12:18 +0900 Subject: [PATCH 5/9] Revise lexer documentation --- yash-syntax/src/parser/lex/core.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/yash-syntax/src/parser/lex/core.rs b/yash-syntax/src/parser/lex/core.rs index 9021af87..09b3f8ec 100644 --- a/yash-syntax/src/parser/lex/core.rs +++ b/yash-syntax/src/parser/lex/core.rs @@ -457,7 +457,11 @@ pub struct Config { /// contained in the [`Location`] instance of the token. /// /// The default value is `None`, in which case the source is set to - /// [`Source::Unknown`]. + /// [`Source::Unknown`]. It is recommended to set this to a more informative + /// value, so that the locations in the parsed syntax tree can be traced + /// back to the source code. Especially, the correct source is necessary to + /// indicate the location of possible errors that occur during parsing and + /// execution. pub source: Option>, } @@ -542,6 +546,12 @@ impl<'a> Lexer<'a> { /// This is a convenience function that creates a lexer with the given input /// object and the default configuration. To customize the configuration, /// use the [`config`](Self::config) function. + /// + /// This function is best used for testing or for simple cases where you + /// don't need to customize the lexer. For practical use, it is recommended + /// to use the [`config`](Self::config) function to create a configuration + /// and provide it with supplementary information, especially + /// [`source`](Config::source), before creating a lexer. pub fn new(input: Box) -> Lexer<'a> { Self::config().input(input) } From 06832981311d980bd9c3c7fb4c279f0502a6f7ac Mon Sep 17 00:00:00 2001 From: WATANABE Yuki Date: Mon, 30 Dec 2024 22:26:01 +0900 Subject: [PATCH 6/9] Introduce Lexer::with_code Almost all calls to Lexer::from_memory were passed Source::Unknown. This commit introduces a new function, Lexer::with_code, which only takes a string argument and uses Source::Unknown internally. This makes the code more readable and reduces boilerplate. --- yash-prompt/src/expand_posix.rs | 3 +- yash-semantics/src/runner.rs | 26 +++---- yash-semantics/src/runner_legacy.rs | 18 ++--- yash-syntax/CHANGELOG.md | 1 + yash-syntax/src/parser/and_or.rs | 8 +- yash-syntax/src/parser/case.rs | 51 ++++++------- yash-syntax/src/parser/command.rs | 9 +-- yash-syntax/src/parser/compound_command.rs | 22 +++--- yash-syntax/src/parser/core.rs | 49 ++++++------ yash-syntax/src/parser/for_loop.rs | 36 ++++----- yash-syntax/src/parser/from_str.rs | 33 ++++----- yash-syntax/src/parser/function.rs | 16 ++-- yash-syntax/src/parser/grouping.rs | 18 ++--- yash-syntax/src/parser/if.rs | 31 ++++---- yash-syntax/src/parser/lex/arith.rs | 14 ++-- yash-syntax/src/parser/lex/backquote.rs | 16 ++-- yash-syntax/src/parser/lex/braced_param.rs | 59 +++++++-------- yash-syntax/src/parser/lex/command_subst.rs | 6 +- yash-syntax/src/parser/lex/core.rs | 82 +++++++++++++-------- yash-syntax/src/parser/lex/dollar.rs | 20 ++--- yash-syntax/src/parser/lex/escape.rs | 56 +++++++------- yash-syntax/src/parser/lex/heredoc.rs | 20 +++-- yash-syntax/src/parser/lex/misc.rs | 17 ++--- yash-syntax/src/parser/lex/modifier.rs | 45 ++++++----- yash-syntax/src/parser/lex/op.rs | 10 +-- yash-syntax/src/parser/lex/raw_param.rs | 10 +-- yash-syntax/src/parser/lex/text.rs | 36 ++++----- yash-syntax/src/parser/lex/token.rs | 12 +-- yash-syntax/src/parser/lex/word.rs | 38 +++++----- yash-syntax/src/parser/list.rs | 36 ++++----- yash-syntax/src/parser/pipeline.rs | 20 ++--- yash-syntax/src/parser/redir.rs | 39 +++++----- yash-syntax/src/parser/simple_command.rs | 59 ++++++++------- yash-syntax/src/parser/while_loop.rs | 20 ++--- 34 files changed, 464 insertions(+), 472 deletions(-) diff --git a/yash-prompt/src/expand_posix.rs b/yash-prompt/src/expand_posix.rs index 765b3ba9..9fc258a1 100644 --- a/yash-prompt/src/expand_posix.rs +++ b/yash-prompt/src/expand_posix.rs @@ -20,7 +20,6 @@ use futures_util::FutureExt as _; use yash_env::Env; use yash_semantics::expansion::expand_text; use yash_syntax::parser::lex::Lexer; -use yash_syntax::source::Source; use yash_syntax::syntax::Text; use yash_syntax::syntax::TextUnit::{self, Literal}; @@ -42,7 +41,7 @@ use yash_syntax::syntax::TextUnit::{self, Literal}; /// implementations support backslash escapes in the prompt string. This /// discrepancy may be reconsidered in the future. pub async fn expand_posix(env: &mut Env, prompt: &str, excl: bool) -> String { - let mut lexer = Lexer::from_memory(prompt, Source::Unknown); + let mut lexer = Lexer::with_code(prompt); let text_result = lexer.text(|_| false, |_| false).now_or_never().unwrap(); let mut text = text_result.unwrap_or_else(|_| { diff --git a/yash-semantics/src/runner.rs b/yash-semantics/src/runner.rs index 1559114b..24c4d7f4 100644 --- a/yash-semantics/src/runner.rs +++ b/yash-semantics/src/runner.rs @@ -65,9 +65,8 @@ use yash_syntax::syntax::List; /// # use yash_semantics::ExitStatus; /// # use yash_semantics::read_eval_loop; /// # use yash_syntax::parser::lex::Lexer; -/// # use yash_syntax::source::Source; /// let mut env = Env::new_virtual(); -/// let mut lexer = Lexer::from_memory("case foo in (bar) ;; esac", Source::Unknown); +/// let mut lexer = Lexer::with_code("case foo in (bar) ;; esac"); /// let result = read_eval_loop(&RefCell::new(&mut env), &mut lexer).await; /// assert_eq!(result, Continue(())); /// assert_eq!(env.exit_status, ExitStatus::SUCCESS); @@ -215,13 +214,12 @@ mod tests { use yash_env_test_helper::assert_stdout; use yash_syntax::input::Context; use yash_syntax::source::Location; - use yash_syntax::source::Source; #[test] fn exit_status_zero_with_no_commands() { let mut env = Env::new_virtual(); env.exit_status = ExitStatus(5); - let mut lexer = Lexer::from_memory("", Source::Unknown); + let mut lexer = Lexer::with_code(""); let ref_env = RefCell::new(&mut env); let result = read_eval_loop(&ref_env, &mut lexer).now_or_never().unwrap(); @@ -237,7 +235,7 @@ mod tests { env.exit_status = ExitStatus(42); env.builtins.insert("echo", echo_builtin()); env.builtins.insert("return", return_builtin()); - let mut lexer = Lexer::from_memory("echo $?; return -n 7", Source::Unknown); + let mut lexer = Lexer::with_code("echo $?; return -n 7"); let ref_env = RefCell::new(&mut env); let result = read_eval_loop(&ref_env, &mut lexer).now_or_never().unwrap(); @@ -252,7 +250,7 @@ mod tests { let state = Rc::clone(&system.state); let mut env = Env::with_system(Box::new(system)); env.builtins.insert("echo", echo_builtin()); - let mut lexer = Lexer::from_memory("echo 1\necho 2\necho 3;", Source::Unknown); + let mut lexer = Lexer::with_code("echo 1\necho 2\necho 3;"); let ref_env = RefCell::new(&mut env); let result = read_eval_loop(&ref_env, &mut lexer).now_or_never().unwrap(); @@ -273,7 +271,7 @@ mod tests { origin: Location::dummy(""), }))); env.builtins.insert("echo", echo_builtin()); - let mut lexer = Lexer::from_memory("echo", Source::Unknown); + let mut lexer = Lexer::with_code("echo"); let ref_env = RefCell::new(&mut env); let result = read_eval_loop(&ref_env, &mut lexer).now_or_never().unwrap(); @@ -307,7 +305,7 @@ mod tests { let state = Rc::clone(&system.state); let mut env = Env::with_system(Box::new(system)); env.builtins.insert("echo", echo_builtin()); - let mut lexer = Lexer::from_memory("${X?}\necho $?\n", Source::Unknown); + let mut lexer = Lexer::with_code("${X?}\necho $?\n"); let ref_env = RefCell::new(&mut env); let result = interactive_read_eval_loop(&ref_env, &mut lexer) @@ -326,7 +324,7 @@ mod tests { let mut env = Env::with_system(Box::new(system)); env.builtins.insert("echo", echo_builtin()); env.builtins.insert("return", return_builtin()); - let mut lexer = Lexer::from_memory("return 123\necho $?\n", Source::Unknown); + let mut lexer = Lexer::with_code("return 123\necho $?\n"); let ref_env = RefCell::new(&mut env); let result = interactive_read_eval_loop(&ref_env, &mut lexer) @@ -344,7 +342,7 @@ mod tests { let state = Rc::clone(&system.state); let mut env = Env::with_system(Box::new(system)); env.builtins.insert("echo", echo_builtin()); - let mut lexer = Lexer::from_memory("${X?}\necho $?\n", Source::Unknown); + let mut lexer = Lexer::with_code("${X?}\necho $?\n"); let ref_env = RefCell::new(&mut env); let result = read_eval_loop(&ref_env, &mut lexer).now_or_never().unwrap(); @@ -357,7 +355,7 @@ mod tests { let system = VirtualSystem::new(); let state = Rc::clone(&system.state); let mut env = Env::with_system(Box::new(system)); - let mut lexer = Lexer::from_memory(";;", Source::Unknown); + let mut lexer = Lexer::with_code(";;"); let ref_env = RefCell::new(&mut env); let result = read_eval_loop(&ref_env, &mut lexer).now_or_never().unwrap(); assert_eq!(result, Break(Divert::Interrupt(Some(ExitStatus::ERROR)))); @@ -370,7 +368,7 @@ mod tests { let state = Rc::clone(&system.state); let mut env = Env::with_system(Box::new(system)); env.builtins.insert("echo", echo_builtin()); - let mut lexer = Lexer::from_memory(";;\necho !", Source::Unknown); + let mut lexer = Lexer::with_code(";;\necho !"); let ref_env = RefCell::new(&mut env); let result = read_eval_loop(&ref_env, &mut lexer).now_or_never().unwrap(); @@ -386,7 +384,7 @@ mod tests { env.builtins.insert("echo", echo_builtin()); // The ";;" causes a syntax error, the following "(" is ignored, and the // loop continues with the command "echo $?" on the next line. - let mut lexer = Lexer::from_memory(";; (\necho $?", Source::Unknown); + let mut lexer = Lexer::with_code(";; (\necho $?"); let ref_env = RefCell::new(&mut env); let result = interactive_read_eval_loop(&ref_env, &mut lexer) @@ -436,7 +434,7 @@ mod tests { .get_mut(&system.process_id) .unwrap() .raise_signal(SIGUSR1); - let mut lexer = Lexer::from_memory("echo $?", Source::Unknown); + let mut lexer = Lexer::with_code("echo $?"); let ref_env = RefCell::new(&mut env); let result = read_eval_loop(&ref_env, &mut lexer).now_or_never().unwrap(); diff --git a/yash-semantics/src/runner_legacy.rs b/yash-semantics/src/runner_legacy.rs index 5296ab69..a29e89fb 100644 --- a/yash-semantics/src/runner_legacy.rs +++ b/yash-semantics/src/runner_legacy.rs @@ -55,9 +55,8 @@ use yash_syntax::parser::Parser; /// # use yash_env::Env; /// # use yash_semantics::*; /// # use yash_syntax::parser::lex::Lexer; -/// # use yash_syntax::source::Source; /// let mut env = Env::new_virtual(); -/// let mut lexer = Lexer::from_memory("case foo in (bar) ;; esac", Source::Unknown); +/// let mut lexer = Lexer::with_code("case foo in (bar) ;; esac"); /// let result = ReadEvalLoop::new(&mut env, &mut lexer).run().await; /// assert_eq!(result, Continue(())); /// assert_eq!(env.exit_status, ExitStatus::SUCCESS); @@ -184,13 +183,12 @@ mod tests { use yash_env_test_helper::assert_stderr; use yash_env_test_helper::assert_stdout; use yash_syntax::source::Location; - use yash_syntax::source::Source; #[test] fn exit_status_zero_with_no_commands() { let mut env = Env::new_virtual(); env.exit_status = ExitStatus(5); - let mut lexer = Lexer::from_memory("", Source::Unknown); + let mut lexer = Lexer::with_code(""); #[allow(deprecated)] let rel = ReadEvalLoop::new(&mut env, &mut lexer); let result = rel.run().now_or_never().unwrap(); @@ -206,7 +204,7 @@ mod tests { env.exit_status = ExitStatus(42); env.builtins.insert("echo", echo_builtin()); env.builtins.insert("return", return_builtin()); - let mut lexer = Lexer::from_memory("echo $?; return -n 7", Source::Unknown); + let mut lexer = Lexer::with_code("echo $?; return -n 7"); #[allow(deprecated)] let rel = ReadEvalLoop::new(&mut env, &mut lexer); let result = rel.run().now_or_never().unwrap(); @@ -221,7 +219,7 @@ mod tests { let state = Rc::clone(&system.state); let mut env = Env::with_system(Box::new(system)); env.builtins.insert("echo", echo_builtin()); - let mut lexer = Lexer::from_memory("echo 1\necho 2\necho 3;", Source::Unknown); + let mut lexer = Lexer::with_code("echo 1\necho 2\necho 3;"); #[allow(deprecated)] let rel = ReadEvalLoop::new(&mut env, &mut lexer); let result = rel.run().now_or_never().unwrap(); @@ -242,7 +240,7 @@ mod tests { origin: Location::dummy(""), }))); env.builtins.insert("echo", echo_builtin()); - let mut lexer = Lexer::from_memory("echo", Source::Unknown); + let mut lexer = Lexer::with_code("echo"); #[allow(deprecated)] let rel = ReadEvalLoop::new(&mut env, &mut lexer); let result = rel.run().now_or_never().unwrap(); @@ -286,7 +284,7 @@ mod tests { let system = VirtualSystem::new(); let state = Rc::clone(&system.state); let mut env = Env::with_system(Box::new(system)); - let mut lexer = Lexer::from_memory(";;", Source::Unknown); + let mut lexer = Lexer::with_code(";;"); #[allow(deprecated)] let rel = ReadEvalLoop::new(&mut env, &mut lexer); let result = rel.run().now_or_never().unwrap(); @@ -300,7 +298,7 @@ mod tests { let state = Rc::clone(&system.state); let mut env = Env::with_system(Box::new(system)); env.builtins.insert("echo", echo_builtin()); - let mut lexer = Lexer::from_memory(";;\necho !", Source::Unknown); + let mut lexer = Lexer::with_code(";;\necho !"); #[allow(deprecated)] let rel = ReadEvalLoop::new(&mut env, &mut lexer); let result = rel.run().now_or_never().unwrap(); @@ -329,7 +327,7 @@ mod tests { .get_mut(&system.process_id) .unwrap() .raise_signal(SIGUSR1); - let mut lexer = Lexer::from_memory("echo $?", Source::Unknown); + let mut lexer = Lexer::with_code("echo $?"); #[allow(deprecated)] let rel = ReadEvalLoop::new(&mut env, &mut lexer); let result = rel.run().now_or_never().unwrap(); diff --git a/yash-syntax/CHANGELOG.md b/yash-syntax/CHANGELOG.md index d9b69522..5aa47369 100644 --- a/yash-syntax/CHANGELOG.md +++ b/yash-syntax/CHANGELOG.md @@ -21,6 +21,7 @@ more flexible and readable configurations. - Added the `Config` struct to the `parser::lex` module. Currently, it allows setting the starting line number and the source information for the lexer. - The `syntax::Word::parse_tilde_everywhere_after` method is added. +- The `with_code` function is added to the `parser::lex::Lexer` struct. - The `From<&str>` trait is now implemented for `input::Memory`. ### Changed diff --git a/yash-syntax/src/parser/and_or.rs b/yash-syntax/src/parser/and_or.rs index 956ffa37..9d8fdefe 100644 --- a/yash-syntax/src/parser/and_or.rs +++ b/yash-syntax/src/parser/and_or.rs @@ -80,7 +80,7 @@ mod tests { #[test] fn parser_and_or_list_eof() { - let mut lexer = Lexer::from_memory("", Source::Unknown); + let mut lexer = Lexer::with_code(""); let mut parser = Parser::new(&mut lexer); let result = parser.and_or_list().now_or_never().unwrap(); @@ -89,7 +89,7 @@ mod tests { #[test] fn parser_and_or_list_one() { - let mut lexer = Lexer::from_memory("foo", Source::Unknown); + let mut lexer = Lexer::with_code("foo"); let mut parser = Parser::new(&mut lexer); let result = parser.and_or_list().now_or_never().unwrap(); @@ -100,7 +100,7 @@ mod tests { #[test] fn parser_and_or_list_many() { - let mut lexer = Lexer::from_memory("first && second || \n\n third;", Source::Unknown); + let mut lexer = Lexer::with_code("first && second || \n\n third;"); let mut parser = Parser::new(&mut lexer); let result = parser.and_or_list().now_or_never().unwrap(); @@ -115,7 +115,7 @@ mod tests { #[test] fn parser_and_or_list_missing_command_after_and_and() { - let mut lexer = Lexer::from_memory("foo &&", Source::Unknown); + let mut lexer = Lexer::with_code("foo &&"); let mut parser = Parser::new(&mut lexer); let e = parser.and_or_list().now_or_never().unwrap().unwrap_err(); diff --git a/yash-syntax/src/parser/case.rs b/yash-syntax/src/parser/case.rs index 5317446b..221bedbb 100644 --- a/yash-syntax/src/parser/case.rs +++ b/yash-syntax/src/parser/case.rs @@ -200,7 +200,7 @@ mod tests { #[test] fn parser_case_item_esac() { - let mut lexer = Lexer::from_memory("\nESAC", Source::Unknown); + let mut lexer = Lexer::with_code("\nESAC"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); let origin = Location::dummy(""); @@ -227,7 +227,7 @@ mod tests { #[test] fn parser_case_item_minimum() { - let mut lexer = Lexer::from_memory("foo)", Source::Unknown); + let mut lexer = Lexer::with_code("foo)"); let mut parser = Parser::new(&mut lexer); let (item, continued) = parser.case_item().now_or_never().unwrap().unwrap().unwrap(); @@ -243,7 +243,7 @@ mod tests { #[test] fn parser_case_item_with_open_paren() { - let mut lexer = Lexer::from_memory("(foo)", Source::Unknown); + let mut lexer = Lexer::with_code("(foo)"); let mut parser = Parser::new(&mut lexer); let (item, continued) = parser.case_item().now_or_never().unwrap().unwrap().unwrap(); @@ -259,7 +259,7 @@ mod tests { #[test] fn parser_case_item_many_patterns() { - let mut lexer = Lexer::from_memory("1 | esac | $three)", Source::Unknown); + let mut lexer = Lexer::with_code("1 | esac | $three)"); let mut parser = Parser::new(&mut lexer); let (item, continued) = parser.case_item().now_or_never().unwrap().unwrap().unwrap(); @@ -277,7 +277,7 @@ mod tests { #[test] fn parser_case_item_non_empty_body() { - let mut lexer = Lexer::from_memory("foo)\necho ok\n:&\n", Source::Unknown); + let mut lexer = Lexer::with_code("foo)\necho ok\n:&\n"); let mut parser = Parser::new(&mut lexer); let (item, continued) = parser.case_item().now_or_never().unwrap().unwrap().unwrap(); @@ -295,7 +295,7 @@ mod tests { #[test] fn parser_case_item_with_double_semicolon() { - let mut lexer = Lexer::from_memory("foo);;", Source::Unknown); + let mut lexer = Lexer::with_code("foo);;"); let mut parser = Parser::new(&mut lexer); let (item, continued) = parser.case_item().now_or_never().unwrap().unwrap().unwrap(); @@ -311,7 +311,7 @@ mod tests { #[test] fn parser_case_item_with_non_empty_body_and_double_semicolon() { - let mut lexer = Lexer::from_memory("foo):;\n;;", Source::Unknown); + let mut lexer = Lexer::with_code("foo):;\n;;"); let mut parser = Parser::new(&mut lexer); let (item, continued) = parser.case_item().now_or_never().unwrap().unwrap().unwrap(); @@ -328,7 +328,7 @@ mod tests { #[test] fn parser_case_item_with_semicolon_and() { - let mut lexer = Lexer::from_memory("foo);&", Source::Unknown); + let mut lexer = Lexer::with_code("foo);&"); let mut parser = Parser::new(&mut lexer); let (item, continued) = parser.case_item().now_or_never().unwrap().unwrap().unwrap(); @@ -344,7 +344,7 @@ mod tests { #[test] fn parser_case_item_missing_pattern_without_open_paren() { - let mut lexer = Lexer::from_memory(")", Source::Unknown); + let mut lexer = Lexer::with_code(")"); let mut parser = Parser::new(&mut lexer); let e = parser.case_item().now_or_never().unwrap().unwrap_err(); @@ -357,7 +357,7 @@ mod tests { #[test] fn parser_case_item_esac_after_paren() { - let mut lexer = Lexer::from_memory("(esac)", Source::Unknown); + let mut lexer = Lexer::with_code("(esac)"); let mut parser = Parser::new(&mut lexer); let (item, continued) = parser.case_item().now_or_never().unwrap().unwrap().unwrap(); @@ -370,7 +370,7 @@ mod tests { #[test] fn parser_case_item_first_pattern_not_word_after_open_paren() { - let mut lexer = Lexer::from_memory("(&", Source::Unknown); + let mut lexer = Lexer::with_code("(&"); let mut parser = Parser::new(&mut lexer); let e = parser.case_item().now_or_never().unwrap().unwrap_err(); @@ -383,7 +383,7 @@ mod tests { #[test] fn parser_case_item_missing_pattern_after_bar() { - let mut lexer = Lexer::from_memory("(foo| |", Source::Unknown); + let mut lexer = Lexer::with_code("(foo| |"); let mut parser = Parser::new(&mut lexer); let e = parser.case_item().now_or_never().unwrap().unwrap_err(); @@ -396,7 +396,7 @@ mod tests { #[test] fn parser_case_item_missing_close_paren() { - let mut lexer = Lexer::from_memory("(foo bar", Source::Unknown); + let mut lexer = Lexer::with_code("(foo bar"); let mut parser = Parser::new(&mut lexer); let e = parser.case_item().now_or_never().unwrap().unwrap_err(); @@ -412,7 +412,7 @@ mod tests { #[test] fn parser_case_command_minimum() { - let mut lexer = Lexer::from_memory("case foo in esac", Source::Unknown); + let mut lexer = Lexer::with_code("case foo in esac"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -429,7 +429,7 @@ mod tests { #[test] fn parser_case_command_newline_before_in() { // Alias substitution results in "case x \n\n \nin esac" - let mut lexer = Lexer::from_memory("CASE_X IN_ESAC", Source::Unknown); + let mut lexer = Lexer::with_code("CASE_X IN_ESAC"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); let origin = Location::dummy(""); @@ -464,7 +464,7 @@ mod tests { #[test] fn parser_case_command_alias_on_subject() { // Alias substitution results in " case in in a|b) esac" - let mut lexer = Lexer::from_memory("CASE in a|b) esac", Source::Unknown); + let mut lexer = Lexer::with_code("CASE in a|b) esac"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); let origin = Location::dummy(""); @@ -500,7 +500,7 @@ mod tests { #[test] fn parser_case_command_alias_on_in() { // Alias substitution results in "case x in esac" - let mut lexer = Lexer::from_memory("CASE_X in esac", Source::Unknown); + let mut lexer = Lexer::with_code("CASE_X in esac"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); let origin = Location::dummy(""); @@ -534,7 +534,7 @@ mod tests { #[test] fn parser_case_command_one_item() { - let mut lexer = Lexer::from_memory("case foo in bar) esac", Source::Unknown); + let mut lexer = Lexer::with_code("case foo in bar) esac"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -551,10 +551,7 @@ mod tests { #[test] fn parser_case_command_many_items_without_final_double_semicolon() { - let mut lexer = Lexer::from_memory( - "case x in\n\na) ;; (b|c):&:; ;;\n d)echo\nesac", - Source::Unknown, - ); + let mut lexer = Lexer::with_code("case x in\n\na) ;; (b|c):&:; ;;\n d)echo\nesac"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -573,7 +570,7 @@ mod tests { #[test] fn parser_case_command_many_items_with_final_double_semicolon() { - let mut lexer = Lexer::from_memory("case x in(1);; 2)echo\n\n;;\n\nesac", Source::Unknown); + let mut lexer = Lexer::with_code("case x in(1);; 2)echo\n\n;;\n\nesac"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -591,7 +588,7 @@ mod tests { #[test] fn parser_case_command_missing_subject() { - let mut lexer = Lexer::from_memory(" case ", Source::Unknown); + let mut lexer = Lexer::with_code(" case "); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -605,7 +602,7 @@ mod tests { #[test] fn parser_case_command_invalid_subject() { - let mut lexer = Lexer::from_memory(" case ; ", Source::Unknown); + let mut lexer = Lexer::with_code(" case ; "); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -619,7 +616,7 @@ mod tests { #[test] fn parser_case_command_missing_in() { - let mut lexer = Lexer::from_memory(" case x esac", Source::Unknown); + let mut lexer = Lexer::with_code(" case x esac"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -639,7 +636,7 @@ mod tests { #[test] fn parser_case_command_missing_esac() { - let mut lexer = Lexer::from_memory("case x in a) }", Source::Unknown); + let mut lexer = Lexer::with_code("case x in a) }"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); diff --git a/yash-syntax/src/parser/command.rs b/yash-syntax/src/parser/command.rs index 0f97bea8..6528f2dd 100644 --- a/yash-syntax/src/parser/command.rs +++ b/yash-syntax/src/parser/command.rs @@ -49,13 +49,12 @@ mod tests { use super::super::lex::Lexer; use super::super::lex::TokenId::EndOfInput; use super::*; - use crate::source::Source; use assert_matches::assert_matches; use futures_util::FutureExt; #[test] fn parser_command_simple() { - let mut lexer = Lexer::from_memory("foo < bar", Source::Unknown); + let mut lexer = Lexer::with_code("foo < bar"); let mut parser = Parser::new(&mut lexer); let result = parser.command().now_or_never().unwrap(); @@ -70,7 +69,7 @@ mod tests { #[test] fn parser_command_compound() { - let mut lexer = Lexer::from_memory("(foo) < bar", Source::Unknown); + let mut lexer = Lexer::with_code("(foo) < bar"); let mut parser = Parser::new(&mut lexer); let result = parser.command().now_or_never().unwrap(); @@ -85,7 +84,7 @@ mod tests { #[test] fn parser_command_function() { - let mut lexer = Lexer::from_memory("fun () ( echo )", Source::Unknown); + let mut lexer = Lexer::with_code("fun () ( echo )"); let mut parser = Parser::new(&mut lexer); let result = parser.command().now_or_never().unwrap(); @@ -100,7 +99,7 @@ mod tests { #[test] fn parser_command_eof() { - let mut lexer = Lexer::from_memory("", Source::Unknown); + let mut lexer = Lexer::with_code(""); let mut parser = Parser::new(&mut lexer); let result = parser.command().now_or_never().unwrap().unwrap(); diff --git a/yash-syntax/src/parser/compound_command.rs b/yash-syntax/src/parser/compound_command.rs index 34bfe810..e4178a31 100644 --- a/yash-syntax/src/parser/compound_command.rs +++ b/yash-syntax/src/parser/compound_command.rs @@ -107,7 +107,7 @@ mod tests { #[test] fn parser_do_clause_none() { - let mut lexer = Lexer::from_memory("done", Source::Unknown); + let mut lexer = Lexer::with_code("done"); let mut parser = Parser::new(&mut lexer); let result = parser.do_clause().now_or_never().unwrap().unwrap(); @@ -116,7 +116,7 @@ mod tests { #[test] fn parser_do_clause_short() { - let mut lexer = Lexer::from_memory("do :; done", Source::Unknown); + let mut lexer = Lexer::with_code("do :; done"); let mut parser = Parser::new(&mut lexer); let result = parser.do_clause().now_or_never().unwrap().unwrap().unwrap(); @@ -128,7 +128,7 @@ mod tests { #[test] fn parser_do_clause_long() { - let mut lexer = Lexer::from_memory("do foo; bar& done", Source::Unknown); + let mut lexer = Lexer::with_code("do foo; bar& done"); let mut parser = Parser::new(&mut lexer); let result = parser.do_clause().now_or_never().unwrap().unwrap().unwrap(); @@ -140,7 +140,7 @@ mod tests { #[test] fn parser_do_clause_unclosed() { - let mut lexer = Lexer::from_memory(" do not close ", Source::Unknown); + let mut lexer = Lexer::with_code(" do not close "); let mut parser = Parser::new(&mut lexer); let e = parser.do_clause().now_or_never().unwrap().unwrap_err(); @@ -159,7 +159,7 @@ mod tests { #[test] fn parser_do_clause_empty_posix() { - let mut lexer = Lexer::from_memory("do done", Source::Unknown); + let mut lexer = Lexer::with_code("do done"); let mut parser = Parser::new(&mut lexer); let e = parser.do_clause().now_or_never().unwrap().unwrap_err(); @@ -172,7 +172,7 @@ mod tests { #[test] fn parser_do_clause_aliasing() { - let mut lexer = Lexer::from_memory(" do :; end ", Source::Unknown); + let mut lexer = Lexer::with_code(" do :; end "); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); let origin = Location::dummy(""); @@ -205,7 +205,7 @@ mod tests { #[test] fn parser_compound_command_none() { - let mut lexer = Lexer::from_memory("}", Source::Unknown); + let mut lexer = Lexer::with_code("}"); let mut parser = Parser::new(&mut lexer); let option = parser.compound_command().now_or_never().unwrap().unwrap(); @@ -214,7 +214,7 @@ mod tests { #[test] fn parser_full_compound_command_without_redirections() { - let mut lexer = Lexer::from_memory("(:)", Source::Unknown); + let mut lexer = Lexer::with_code("(:)"); let mut parser = Parser::new(&mut lexer); let result = parser.full_compound_command().now_or_never().unwrap(); @@ -225,7 +225,7 @@ mod tests { #[test] fn parser_full_compound_command_with_redirections() { - let mut lexer = Lexer::from_memory("(command) bar ;", Source::Unknown); + let mut lexer = Lexer::with_code("(command) bar ;"); let mut parser = Parser::new(&mut lexer); let result = parser.full_compound_command().now_or_never().unwrap(); @@ -241,7 +241,7 @@ mod tests { #[test] fn parser_full_compound_command_none() { - let mut lexer = Lexer::from_memory("}", Source::Unknown); + let mut lexer = Lexer::with_code("}"); let mut parser = Parser::new(&mut lexer); let result = parser.full_compound_command().now_or_never().unwrap(); @@ -250,7 +250,7 @@ mod tests { #[test] fn parser_short_function_definition_ok() { - let mut lexer = Lexer::from_memory(" ( ) ( : ) > /dev/null ", Source::Unknown); + let mut lexer = Lexer::with_code(" ( ) ( : ) > /dev/null "); let mut parser = Parser::new(&mut lexer); let c = SimpleCommand { assigns: vec![], diff --git a/yash-syntax/src/parser/core.rs b/yash-syntax/src/parser/core.rs index 5e605b2c..434c9501 100644 --- a/yash-syntax/src/parser/core.rs +++ b/yash-syntax/src/parser/core.rs @@ -446,14 +446,13 @@ mod tests { use crate::alias::AliasSet; use crate::alias::HashEntry; use crate::source::Location; - use crate::source::Source; use assert_matches::assert_matches; use futures_util::FutureExt; use std::cell::OnceCell; #[test] fn parser_take_token_manual_successful_substitution() { - let mut lexer = Lexer::from_memory("X", Source::Unknown); + let mut lexer = Lexer::with_code("X"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); aliases.insert(HashEntry::new( @@ -474,7 +473,7 @@ mod tests { #[test] fn parser_take_token_manual_not_command_name() { - let mut lexer = Lexer::from_memory("X", Source::Unknown); + let mut lexer = Lexer::with_code("X"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); aliases.insert(HashEntry::new( @@ -492,7 +491,7 @@ mod tests { #[test] fn parser_take_token_manual_not_literal() { - let mut lexer = Lexer::from_memory(r"\X", Source::Unknown); + let mut lexer = Lexer::with_code(r"\X"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); aliases.insert(HashEntry::new( @@ -516,7 +515,7 @@ mod tests { #[test] fn parser_take_token_manual_operator() { - let mut lexer = Lexer::from_memory(";", Source::Unknown); + let mut lexer = Lexer::with_code(";"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); aliases.insert(HashEntry::new( @@ -535,7 +534,7 @@ mod tests { #[test] fn parser_take_token_manual_no_match() { - let mut lexer = Lexer::from_memory("X", Source::Unknown); + let mut lexer = Lexer::with_code("X"); let mut parser = Parser::new(&mut lexer); let result = parser.take_token_manual(true).now_or_never().unwrap(); @@ -545,7 +544,7 @@ mod tests { #[test] fn parser_take_token_manual_recursive_substitution() { - let mut lexer = Lexer::from_memory("X", Source::Unknown); + let mut lexer = Lexer::with_code("X"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); aliases.insert(HashEntry::new( @@ -583,7 +582,7 @@ mod tests { #[test] fn parser_take_token_manual_after_blank_ending_substitution() { - let mut lexer = Lexer::from_memory("X\tY", Source::Unknown); + let mut lexer = Lexer::with_code("X\tY"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); aliases.insert(HashEntry::new( @@ -617,7 +616,7 @@ mod tests { #[test] fn parser_take_token_manual_not_after_blank_ending_substitution() { - let mut lexer = Lexer::from_memory("X\tY", Source::Unknown); + let mut lexer = Lexer::with_code("X\tY"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); aliases.insert(HashEntry::new( @@ -648,7 +647,7 @@ mod tests { #[test] fn parser_take_token_manual_global() { - let mut lexer = Lexer::from_memory("X", Source::Unknown); + let mut lexer = Lexer::with_code("X"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); aliases.insert(HashEntry::new( @@ -669,7 +668,7 @@ mod tests { #[test] fn parser_take_token_auto_non_keyword() { - let mut lexer = Lexer::from_memory("X", Source::Unknown); + let mut lexer = Lexer::with_code("X"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); aliases.insert(HashEntry::new( @@ -686,7 +685,7 @@ mod tests { #[test] fn parser_take_token_auto_keyword_matched() { - let mut lexer = Lexer::from_memory("if", Source::Unknown); + let mut lexer = Lexer::with_code("if"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); aliases.insert(HashEntry::new( @@ -707,7 +706,7 @@ mod tests { #[test] fn parser_take_token_auto_keyword_unmatched() { - let mut lexer = Lexer::from_memory("if", Source::Unknown); + let mut lexer = Lexer::with_code("if"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); aliases.insert(HashEntry::new( @@ -724,7 +723,7 @@ mod tests { #[test] fn parser_take_token_auto_alias_substitution_to_keyword_matched() { - let mut lexer = Lexer::from_memory("X", Source::Unknown); + let mut lexer = Lexer::with_code("X"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); aliases.insert(HashEntry::new( @@ -751,7 +750,7 @@ mod tests { #[test] fn parser_has_blank_true() { - let mut lexer = Lexer::from_memory(" ", Source::Unknown); + let mut lexer = Lexer::with_code(" "); let mut parser = Parser::new(&mut lexer); let result = parser.has_blank().now_or_never().unwrap(); assert_eq!(result, Ok(true)); @@ -759,7 +758,7 @@ mod tests { #[test] fn parser_has_blank_false() { - let mut lexer = Lexer::from_memory("(", Source::Unknown); + let mut lexer = Lexer::with_code("("); let mut parser = Parser::new(&mut lexer); let result = parser.has_blank().now_or_never().unwrap(); assert_eq!(result, Ok(false)); @@ -767,7 +766,7 @@ mod tests { #[test] fn parser_has_blank_eof() { - let mut lexer = Lexer::from_memory("", Source::Unknown); + let mut lexer = Lexer::with_code(""); let mut parser = Parser::new(&mut lexer); let result = parser.has_blank().now_or_never().unwrap(); assert_eq!(result, Ok(false)); @@ -775,7 +774,7 @@ mod tests { #[test] fn parser_has_blank_true_with_line_continuations() { - let mut lexer = Lexer::from_memory("\\\n\\\n ", Source::Unknown); + let mut lexer = Lexer::with_code("\\\n\\\n "); let mut parser = Parser::new(&mut lexer); let result = parser.has_blank().now_or_never().unwrap(); assert_eq!(result, Ok(true)); @@ -783,7 +782,7 @@ mod tests { #[test] fn parser_has_blank_false_with_line_continuations() { - let mut lexer = Lexer::from_memory("\\\n\\\n\\\n(", Source::Unknown); + let mut lexer = Lexer::with_code("\\\n\\\n\\\n("); let mut parser = Parser::new(&mut lexer); let result = parser.has_blank().now_or_never().unwrap(); assert_eq!(result, Ok(false)); @@ -792,7 +791,7 @@ mod tests { #[test] #[should_panic(expected = "There should be no pending token")] fn parser_has_blank_with_pending_token() { - let mut lexer = Lexer::from_memory("foo", Source::Unknown); + let mut lexer = Lexer::with_code("foo"); let mut parser = Parser::new(&mut lexer); parser.peek_token().now_or_never().unwrap().unwrap(); let _ = parser.has_blank().now_or_never().unwrap(); @@ -800,7 +799,7 @@ mod tests { #[test] fn parser_reading_no_here_doc_contents() { - let mut lexer = Lexer::from_memory("X", Source::Unknown); + let mut lexer = Lexer::with_code("X"); let mut parser = Parser::new(&mut lexer); parser.here_doc_contents().now_or_never().unwrap().unwrap(); @@ -813,7 +812,7 @@ mod tests { fn parser_reading_one_here_doc_content() { let delimiter = "END".parse().unwrap(); - let mut lexer = Lexer::from_memory("END\nX", Source::Unknown); + let mut lexer = Lexer::with_code("END\nX"); let mut parser = Parser::new(&mut lexer); let remove_tabs = false; let here_doc = Rc::new(HereDoc { @@ -838,7 +837,7 @@ mod tests { let delimiter2 = "TWO".parse().unwrap(); let delimiter3 = "THREE".parse().unwrap(); - let mut lexer = Lexer::from_memory("1\nONE\nTWO\n3\nTHREE\nX", Source::Unknown); + let mut lexer = Lexer::with_code("1\nONE\nTWO\n3\nTHREE\nX"); let mut parser = Parser::new(&mut lexer); let here_doc1 = Rc::new(HereDoc { delimiter: delimiter1, @@ -875,7 +874,7 @@ mod tests { let delimiter1 = "ONE".parse().unwrap(); let delimiter2 = "TWO".parse().unwrap(); - let mut lexer = Lexer::from_memory("1\nONE\n2\nTWO\n", Source::Unknown); + let mut lexer = Lexer::with_code("1\nONE\n2\nTWO\n"); let mut parser = Parser::new(&mut lexer); let here_doc1 = Rc::new(HereDoc { delimiter: delimiter1, @@ -902,7 +901,7 @@ mod tests { #[test] #[should_panic(expected = "No token must be peeked before reading here-doc contents")] fn parser_here_doc_contents_must_be_called_without_pending_token() { - let mut lexer = Lexer::from_memory("X", Source::Unknown); + let mut lexer = Lexer::with_code("X"); let mut parser = Parser::new(&mut lexer); parser.peek_token().now_or_never().unwrap().unwrap(); parser.here_doc_contents().now_or_never().unwrap().unwrap(); diff --git a/yash-syntax/src/parser/for_loop.rs b/yash-syntax/src/parser/for_loop.rs index 08cf5c5f..12f5db49 100644 --- a/yash-syntax/src/parser/for_loop.rs +++ b/yash-syntax/src/parser/for_loop.rs @@ -166,7 +166,7 @@ mod tests { #[test] fn parser_for_loop_short() { - let mut lexer = Lexer::from_memory("for A do :; done", Source::Unknown); + let mut lexer = Lexer::with_code("for A do :; done"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -183,7 +183,7 @@ mod tests { #[test] fn parser_for_loop_with_semicolon_before_do() { - let mut lexer = Lexer::from_memory("for B ; do :; done", Source::Unknown); + let mut lexer = Lexer::with_code("for B ; do :; done"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -200,7 +200,7 @@ mod tests { #[test] fn parser_for_loop_with_semicolon_and_newlines_before_do() { - let mut lexer = Lexer::from_memory("for B ; \n\t\n do :; done", Source::Unknown); + let mut lexer = Lexer::with_code("for B ; \n\t\n do :; done"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -217,7 +217,7 @@ mod tests { #[test] fn parser_for_loop_with_newlines_before_do() { - let mut lexer = Lexer::from_memory("for B \n \\\n \n do :; done", Source::Unknown); + let mut lexer = Lexer::with_code("for B \n \\\n \n do :; done"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -234,7 +234,7 @@ mod tests { #[test] fn parser_for_loop_with_zero_values_delimited_by_semicolon() { - let mut lexer = Lexer::from_memory("for foo in; do :; done", Source::Unknown); + let mut lexer = Lexer::with_code("for foo in; do :; done"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -251,7 +251,7 @@ mod tests { #[test] fn parser_for_loop_with_one_value_delimited_by_semicolon_and_newlines() { - let mut lexer = Lexer::from_memory("for foo in bar; \n \n do :; done", Source::Unknown); + let mut lexer = Lexer::with_code("for foo in bar; \n \n do :; done"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -273,7 +273,7 @@ mod tests { #[test] fn parser_for_loop_with_many_values_delimited_by_one_newline() { - let mut lexer = Lexer::from_memory("for in in in a b c\ndo :; done", Source::Unknown); + let mut lexer = Lexer::with_code("for in in in a b c\ndo :; done"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -295,7 +295,7 @@ mod tests { #[test] fn parser_for_loop_with_zero_values_delimited_by_many_newlines() { - let mut lexer = Lexer::from_memory("for foo in \n \n \n do :; done", Source::Unknown); + let mut lexer = Lexer::with_code("for foo in \n \n \n do :; done"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -312,7 +312,7 @@ mod tests { #[test] fn parser_for_loop_newlines_before_in() { - let mut lexer = Lexer::from_memory("for foo\n \n\nin\ndo :; done", Source::Unknown); + let mut lexer = Lexer::with_code("for foo\n \n\nin\ndo :; done"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -329,7 +329,7 @@ mod tests { #[test] fn parser_for_loop_aliasing_on_semicolon() { - let mut lexer = Lexer::from_memory(" FOR_A if :; done", Source::Unknown); + let mut lexer = Lexer::with_code(" FOR_A if :; done"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); let origin = Location::dummy(""); @@ -360,7 +360,7 @@ mod tests { #[test] fn parser_for_loop_aliasing_on_do() { - let mut lexer = Lexer::from_memory(" FOR_A if :; done", Source::Unknown); + let mut lexer = Lexer::with_code(" FOR_A if :; done"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); let origin = Location::dummy(""); @@ -391,7 +391,7 @@ mod tests { #[test] fn parser_for_loop_missing_name_eof() { - let mut lexer = Lexer::from_memory(" for ", Source::Unknown); + let mut lexer = Lexer::with_code(" for "); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -405,7 +405,7 @@ mod tests { #[test] fn parser_for_loop_missing_name_newline() { - let mut lexer = Lexer::from_memory(" for\ndo :; done", Source::Unknown); + let mut lexer = Lexer::with_code(" for\ndo :; done"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -419,7 +419,7 @@ mod tests { #[test] fn parser_for_loop_missing_name_semicolon() { - let mut lexer = Lexer::from_memory("for; do :; done", Source::Unknown); + let mut lexer = Lexer::with_code("for; do :; done"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -434,7 +434,7 @@ mod tests { #[test] fn parser_for_loop_invalid_name() { // Alias substitution results in "for & do :; done" - let mut lexer = Lexer::from_memory("FOR if do :; done", Source::Unknown); + let mut lexer = Lexer::with_code("FOR if do :; done"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); let origin = Location::dummy(""); @@ -472,7 +472,7 @@ mod tests { #[test] fn parser_for_loop_semicolon_after_newline() { - let mut lexer = Lexer::from_memory("for X\n; do :; done", Source::Unknown); + let mut lexer = Lexer::with_code("for X\n; do :; done"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -493,7 +493,7 @@ mod tests { #[test] fn parser_for_loop_invalid_values_delimiter() { // Alias substitution results in "for A in a b & c; do :; done" - let mut lexer = Lexer::from_memory("for_A_in_a_b if c; do :; done", Source::Unknown); + let mut lexer = Lexer::with_code("for_A_in_a_b if c; do :; done"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); let origin = Location::dummy(""); @@ -531,7 +531,7 @@ mod tests { #[test] fn parser_for_loop_invalid_token_after_semicolon() { - let mut lexer = Lexer::from_memory(" for X; ! do :; done", Source::Unknown); + let mut lexer = Lexer::with_code(" for X; ! do :; done"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); diff --git a/yash-syntax/src/parser/from_str.rs b/yash-syntax/src/parser/from_str.rs index 4892019b..0df32b89 100644 --- a/yash-syntax/src/parser/from_str.rs +++ b/yash-syntax/src/parser/from_str.rs @@ -25,7 +25,6 @@ use super::Error; use super::ErrorCause; use super::Parser; use super::SyntaxError; -use crate::source::Source; use crate::syntax::*; use std::future::Future; use std::str::FromStr; @@ -92,7 +91,7 @@ impl FromStr for TextUnit { type Err = Option; fn from_str(s: &str) -> Result> { - let mut lexer = Lexer::from_memory(s, Source::Unknown); + let mut lexer = Lexer::with_code(s); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -105,7 +104,7 @@ impl FromStr for TextUnit { impl FromStr for Text { type Err = Error; fn from_str(s: &str) -> Result { - let mut lexer = Lexer::from_memory(s, Source::Unknown); + let mut lexer = Lexer::with_code(s); unwrap_ready(lexer.text(|_| false, |_| true)) } } @@ -118,7 +117,7 @@ impl FromStr for EscapeUnit { type Err = Option; fn from_str(s: &str) -> Result { - let mut lexer = Lexer::from_memory(s, Source::Unknown); + let mut lexer = Lexer::with_code(s); unwrap_ready(lexer.escape_unit()).shift() } } @@ -127,7 +126,7 @@ impl FromStr for EscapeUnit { impl FromStr for EscapedString { type Err = Error; fn from_str(s: &str) -> Result { - let mut lexer = Lexer::from_memory(s, Source::Unknown); + let mut lexer = Lexer::with_code(s); unwrap_ready(lexer.escaped_string(|_| false)) } } @@ -140,7 +139,7 @@ impl FromStr for WordUnit { type Err = Option; fn from_str(s: &str) -> Result> { - let mut lexer = Lexer::from_memory(s, Source::Unknown); + let mut lexer = Lexer::with_code(s); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -158,7 +157,7 @@ impl FromStr for Word { type Err = Error; fn from_str(s: &str) -> Result { - let mut lexer = Lexer::from_memory(s, Source::Unknown); + let mut lexer = Lexer::with_code(s); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -215,7 +214,7 @@ impl FromStr for Assign { impl FromStr for Operator { type Err = ParseOperatorError; fn from_str(s: &str) -> Result { - let mut lexer = Lexer::from_memory(s, Source::Unknown); + let mut lexer = Lexer::with_code(s); match unwrap_ready(lexer.operator()) { Ok(Some(Token { id: TokenId::Operator(op), @@ -247,7 +246,7 @@ impl FromStr for Redir { type Err = Option; fn from_str(s: &str) -> Result> { - let mut lexer = Lexer::from_memory(s, Source::Unknown); + let mut lexer = Lexer::with_code(s); let mut parser = Parser::new(&mut lexer); unwrap_ready(async { let redir = parser.redirection().await?; @@ -276,7 +275,7 @@ impl FromStr for SimpleCommand { type Err = Option; fn from_str(s: &str) -> Result> { - let mut lexer = Lexer::from_memory(s, Source::Unknown); + let mut lexer = Lexer::with_code(s); let mut parser = Parser::new(&mut lexer); unwrap_ready(async { let command = parser.simple_command().await?.unwrap(); @@ -302,7 +301,7 @@ impl FromStr for CaseItem { type Err = Option; fn from_str(s: &str) -> Result> { - let mut lexer = Lexer::from_memory(s, Source::Unknown); + let mut lexer = Lexer::with_code(s); let mut parser = Parser::new(&mut lexer); unwrap_ready(async { let item = parser.case_item().await?.map(|(item, _)| item); @@ -329,7 +328,7 @@ impl FromStr for CompoundCommand { type Err = Option; fn from_str(s: &str) -> Result> { - let mut lexer = Lexer::from_memory(s, Source::Unknown); + let mut lexer = Lexer::with_code(s); let mut parser = Parser::new(&mut lexer); unwrap_ready(async { let command = parser.compound_command().await?; @@ -352,7 +351,7 @@ impl FromStr for FullCompoundCommand { type Err = Option; fn from_str(s: &str) -> Result> { - let mut lexer = Lexer::from_memory(s, Source::Unknown); + let mut lexer = Lexer::with_code(s); let mut parser = Parser::new(&mut lexer); unwrap_ready(async { let command = parser.full_compound_command().await?; @@ -375,7 +374,7 @@ impl FromStr for Command { type Err = Option; fn from_str(s: &str) -> Result> { - let mut lexer = Lexer::from_memory(s, Source::Unknown); + let mut lexer = Lexer::with_code(s); let mut parser = Parser::new(&mut lexer); unwrap_ready(async { let command = parser.command().await?.unwrap(); @@ -398,7 +397,7 @@ impl FromStr for Pipeline { type Err = Option; fn from_str(s: &str) -> Result> { - let mut lexer = Lexer::from_memory(s, Source::Unknown); + let mut lexer = Lexer::with_code(s); let mut parser = Parser::new(&mut lexer); unwrap_ready(async { let pipeline = parser.pipeline().await?.unwrap(); @@ -430,7 +429,7 @@ impl FromStr for AndOrList { type Err = Option; fn from_str(s: &str) -> Result> { - let mut lexer = Lexer::from_memory(s, Source::Unknown); + let mut lexer = Lexer::with_code(s); let mut parser = Parser::new(&mut lexer); unwrap_ready(async { let list = parser.and_or_list().await?.unwrap(); @@ -448,7 +447,7 @@ impl FromStr for AndOrList { impl FromStr for List { type Err = Error; fn from_str(s: &str) -> Result { - let mut lexer = Lexer::from_memory(s, Source::Unknown); + let mut lexer = Lexer::with_code(s); let mut parser = Parser::new(&mut lexer); let list = unwrap_ready(parser.maybe_compound_list())?; parser.ensure_no_unread_here_doc()?; diff --git a/yash-syntax/src/parser/function.rs b/yash-syntax/src/parser/function.rs index 020acd8c..69383cd0 100644 --- a/yash-syntax/src/parser/function.rs +++ b/yash-syntax/src/parser/function.rs @@ -101,7 +101,7 @@ mod tests { #[test] fn parser_short_function_definition_not_one_word_name() { - let mut lexer = Lexer::from_memory("(", Source::Unknown); + let mut lexer = Lexer::with_code("("); let mut parser = Parser::new(&mut lexer); let c = SimpleCommand { assigns: vec![], @@ -121,7 +121,7 @@ mod tests { #[test] fn parser_short_function_definition_eof() { - let mut lexer = Lexer::from_memory("", Source::Unknown); + let mut lexer = Lexer::with_code(""); let mut parser = Parser::new(&mut lexer); let c = SimpleCommand { assigns: vec![], @@ -138,7 +138,7 @@ mod tests { #[test] fn parser_short_function_definition_unmatched_parenthesis() { - let mut lexer = Lexer::from_memory("( ", Source::Unknown); + let mut lexer = Lexer::with_code("( "); let mut parser = Parser::new(&mut lexer); let c = SimpleCommand { assigns: vec![], @@ -160,7 +160,7 @@ mod tests { #[test] fn parser_short_function_definition_missing_function_body() { - let mut lexer = Lexer::from_memory("( ) ", Source::Unknown); + let mut lexer = Lexer::with_code("( ) "); let mut parser = Parser::new(&mut lexer); let c = SimpleCommand { assigns: vec![], @@ -182,7 +182,7 @@ mod tests { #[test] fn parser_short_function_definition_invalid_function_body() { - let mut lexer = Lexer::from_memory("() foo ; ", Source::Unknown); + let mut lexer = Lexer::with_code("() foo ; "); let mut parser = Parser::new(&mut lexer); let c = SimpleCommand { assigns: vec![], @@ -204,7 +204,7 @@ mod tests { #[test] fn parser_short_function_definition_close_parenthesis_alias() { - let mut lexer = Lexer::from_memory(" a b ", Source::Unknown); + let mut lexer = Lexer::with_code(" a b "); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); let origin = Location::dummy(""); @@ -245,7 +245,7 @@ mod tests { #[test] fn parser_short_function_definition_body_alias_and_newline() { - let mut lexer = Lexer::from_memory(" a b ", Source::Unknown); + let mut lexer = Lexer::with_code(" a b "); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); let origin = Location::dummy(""); @@ -286,7 +286,7 @@ mod tests { #[test] fn parser_short_function_definition_alias_inapplicable() { - let mut lexer = Lexer::from_memory("()b", Source::Unknown); + let mut lexer = Lexer::with_code("()b"); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); let origin = Location::dummy(""); diff --git a/yash-syntax/src/parser/grouping.rs b/yash-syntax/src/parser/grouping.rs index f5ffb40f..353f9aca 100644 --- a/yash-syntax/src/parser/grouping.rs +++ b/yash-syntax/src/parser/grouping.rs @@ -106,7 +106,7 @@ mod tests { #[test] fn parser_grouping_short() { - let mut lexer = Lexer::from_memory("{ :; }", Source::Unknown); + let mut lexer = Lexer::with_code("{ :; }"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -118,7 +118,7 @@ mod tests { #[test] fn parser_grouping_long() { - let mut lexer = Lexer::from_memory("{ foo; bar& }", Source::Unknown); + let mut lexer = Lexer::with_code("{ foo; bar& }"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -130,7 +130,7 @@ mod tests { #[test] fn parser_grouping_unclosed() { - let mut lexer = Lexer::from_memory(" { oh no ", Source::Unknown); + let mut lexer = Lexer::with_code(" { oh no "); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -150,7 +150,7 @@ mod tests { #[test] fn parser_grouping_empty_posix() { - let mut lexer = Lexer::from_memory("{ }", Source::Unknown); + let mut lexer = Lexer::with_code("{ }"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -164,7 +164,7 @@ mod tests { #[test] fn parser_grouping_aliasing() { - let mut lexer = Lexer::from_memory(" { :; end ", Source::Unknown); + let mut lexer = Lexer::with_code(" { :; end "); #[allow(clippy::mutable_key_type)] let mut aliases = AliasSet::new(); let origin = Location::dummy(""); @@ -197,7 +197,7 @@ mod tests { #[test] fn parser_subshell_short() { - let mut lexer = Lexer::from_memory("(:)", Source::Unknown); + let mut lexer = Lexer::with_code("(:)"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -213,7 +213,7 @@ mod tests { #[test] fn parser_subshell_long() { - let mut lexer = Lexer::from_memory("( foo& bar; )", Source::Unknown); + let mut lexer = Lexer::with_code("( foo& bar; )"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -229,7 +229,7 @@ mod tests { #[test] fn parser_subshell_unclosed() { - let mut lexer = Lexer::from_memory(" ( oh no", Source::Unknown); + let mut lexer = Lexer::with_code(" ( oh no"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -249,7 +249,7 @@ mod tests { #[test] fn parser_subshell_empty_posix() { - let mut lexer = Lexer::from_memory("( )", Source::Unknown); + let mut lexer = Lexer::with_code("( )"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); diff --git a/yash-syntax/src/parser/if.rs b/yash-syntax/src/parser/if.rs index 25826a4f..b29ff046 100644 --- a/yash-syntax/src/parser/if.rs +++ b/yash-syntax/src/parser/if.rs @@ -146,7 +146,7 @@ mod tests { #[test] fn parser_if_command_minimum() { - let mut lexer = Lexer::from_memory("if a; then b; fi", Source::Unknown); + let mut lexer = Lexer::with_code("if a; then b; fi"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -164,10 +164,7 @@ mod tests { #[test] fn parser_if_command_one_elif() { - let mut lexer = Lexer::from_memory( - "if\ntrue\nthen\nfalse\n\nelif x; then y& fi", - Source::Unknown, - ); + let mut lexer = Lexer::with_code("if\ntrue\nthen\nfalse\n\nelif x; then y& fi"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -186,9 +183,8 @@ mod tests { #[test] fn parser_if_command_many_elifs() { - let mut lexer = Lexer::from_memory( + let mut lexer = Lexer::with_code( "if a; then b; elif c; then d; elif e 1; e 2& then f 1; f 2& elif g; then h; fi", - Source::Unknown, ); let mut parser = Parser::new(&mut lexer); @@ -210,7 +206,7 @@ mod tests { #[test] fn parser_if_command_else() { - let mut lexer = Lexer::from_memory("if a; then b; else c; d; fi", Source::Unknown); + let mut lexer = Lexer::with_code("if a; then b; else c; d; fi"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -228,8 +224,7 @@ mod tests { #[test] fn parser_if_command_elif_and_else() { - let mut lexer = - Lexer::from_memory("if 1; then 2; elif 3; then 4; else 5; fi", Source::Unknown); + let mut lexer = Lexer::with_code("if 1; then 2; elif 3; then 4; else 5; fi"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -248,7 +243,7 @@ mod tests { #[test] fn parser_if_command_without_then_after_if() { - let mut lexer = Lexer::from_memory(" if :; fi", Source::Unknown); + let mut lexer = Lexer::with_code(" if :; fi"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -267,7 +262,7 @@ mod tests { #[test] fn parser_if_command_without_then_after_elif() { - let mut lexer = Lexer::from_memory("if a; then b; elif c; fi", Source::Unknown); + let mut lexer = Lexer::with_code("if a; then b; elif c; fi"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -287,7 +282,7 @@ mod tests { #[test] fn parser_if_command_without_fi() { - let mut lexer = Lexer::from_memory(" if :; then :; }", Source::Unknown); + let mut lexer = Lexer::with_code(" if :; then :; }"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -307,7 +302,7 @@ mod tests { #[test] fn parser_if_command_empty_condition() { - let mut lexer = Lexer::from_memory(" if then :; fi", Source::Unknown); + let mut lexer = Lexer::with_code(" if then :; fi"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -321,7 +316,7 @@ mod tests { #[test] fn parser_if_command_empty_body() { - let mut lexer = Lexer::from_memory("if :; then fi", Source::Unknown); + let mut lexer = Lexer::with_code("if :; then fi"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -335,7 +330,7 @@ mod tests { #[test] fn parser_if_command_empty_elif_condition() { - let mut lexer = Lexer::from_memory("if :; then :; elif then :; fi", Source::Unknown); + let mut lexer = Lexer::with_code("if :; then :; elif then :; fi"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -352,7 +347,7 @@ mod tests { #[test] fn parser_if_command_empty_elif_body() { - let mut lexer = Lexer::from_memory("if :; then :; elif :; then fi", Source::Unknown); + let mut lexer = Lexer::with_code("if :; then :; elif :; then fi"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); @@ -369,7 +364,7 @@ mod tests { #[test] fn parser_if_command_empty_else() { - let mut lexer = Lexer::from_memory("if :; then :; else fi", Source::Unknown); + let mut lexer = Lexer::with_code("if :; then :; else fi"); let mut parser = Parser::new(&mut lexer); let result = parser.compound_command().now_or_never().unwrap(); diff --git a/yash-syntax/src/parser/lex/arith.rs b/yash-syntax/src/parser/lex/arith.rs index ac398332..2ab85a35 100644 --- a/yash-syntax/src/parser/lex/arith.rs +++ b/yash-syntax/src/parser/lex/arith.rs @@ -95,7 +95,7 @@ mod tests { #[test] fn lexer_arithmetic_expansion_empty() { - let mut lexer = Lexer::from_memory("$(());", Source::Unknown); + let mut lexer = Lexer::with_code("$(());"); lexer.peek_char().now_or_never().unwrap().unwrap(); lexer.consume_char(); @@ -114,7 +114,7 @@ mod tests { #[test] fn lexer_arithmetic_expansion_none() { - let mut lexer = Lexer::from_memory("$( foo bar )baz", Source::Unknown); + let mut lexer = Lexer::with_code("$( foo bar )baz"); lexer.peek_char().now_or_never().unwrap().unwrap(); lexer.consume_char(); assert_eq!( @@ -126,7 +126,7 @@ mod tests { #[test] fn lexer_arithmetic_expansion_line_continuations() { - let mut lexer = Lexer::from_memory("$(\\\n\\\n(\\\n)\\\n\\\n);", Source::Unknown); + let mut lexer = Lexer::with_code("$(\\\n\\\n(\\\n)\\\n\\\n);"); lexer.peek_char().now_or_never().unwrap().unwrap(); lexer.consume_char(); @@ -145,7 +145,7 @@ mod tests { #[test] fn lexer_arithmetic_expansion_escapes() { - let mut lexer = Lexer::from_memory(r#".$((\\\"\`\$));"#, Source::Unknown); + let mut lexer = Lexer::with_code(r#".$((\\\"\`\$));"#); lexer.peek_char().now_or_never().unwrap().unwrap(); lexer.consume_char(); lexer.peek_char().now_or_never().unwrap().unwrap(); @@ -175,7 +175,7 @@ mod tests { #[test] fn lexer_arithmetic_expansion_unclosed_first() { - let mut lexer = Lexer::from_memory("$((1", Source::Unknown); + let mut lexer = Lexer::with_code("$((1"); lexer.peek_char().now_or_never().unwrap().unwrap(); lexer.consume_char(); @@ -196,7 +196,7 @@ mod tests { #[test] fn lexer_arithmetic_expansion_unclosed_second() { - let mut lexer = Lexer::from_memory("$((1)", Source::Unknown); + let mut lexer = Lexer::with_code("$((1)"); lexer.peek_char().now_or_never().unwrap().unwrap(); lexer.consume_char(); @@ -217,7 +217,7 @@ mod tests { #[test] fn lexer_arithmetic_expansion_unclosed_but_maybe_command_substitution() { - let mut lexer = Lexer::from_memory("$((1) ", Source::Unknown); + let mut lexer = Lexer::with_code("$((1) "); lexer.peek_char().now_or_never().unwrap().unwrap(); lexer.consume_char(); assert_eq!( diff --git a/yash-syntax/src/parser/lex/backquote.rs b/yash-syntax/src/parser/lex/backquote.rs index c74ebc9f..14fdf289 100644 --- a/yash-syntax/src/parser/lex/backquote.rs +++ b/yash-syntax/src/parser/lex/backquote.rs @@ -92,7 +92,7 @@ mod tests { #[test] fn lexer_backquote_not_backquote() { - let mut lexer = Lexer::from_memory("X", Source::Unknown); + let mut lexer = Lexer::with_code("X"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -103,7 +103,7 @@ mod tests { #[test] fn lexer_backquote_empty() { - let mut lexer = Lexer::from_memory("``", Source::Unknown); + let mut lexer = Lexer::with_code("``"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -119,7 +119,7 @@ mod tests { #[test] fn lexer_backquote_literals() { - let mut lexer = Lexer::from_memory("`echo`", Source::Unknown); + let mut lexer = Lexer::with_code("`echo`"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -143,7 +143,7 @@ mod tests { #[test] fn lexer_backquote_with_escapes_double_quote_escapable() { - let mut lexer = Lexer::from_memory(r#"`a\a\$\`\\\"\'`"#, Source::Unknown); + let mut lexer = Lexer::with_code(r#"`a\a\$\`\\\"\'`"#); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Text, @@ -172,7 +172,7 @@ mod tests { #[test] fn lexer_backquote_with_escapes_double_quote_not_escapable() { - let mut lexer = Lexer::from_memory(r#"`a\a\$\`\\\"\'`"#, Source::Unknown); + let mut lexer = Lexer::with_code(r#"`a\a\$\`\\\"\'`"#); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -202,7 +202,7 @@ mod tests { #[test] fn lexer_backquote_line_continuation() { - let mut lexer = Lexer::from_memory("`\\\na\\\n\\\nb\\\n`", Source::Unknown); + let mut lexer = Lexer::with_code("`\\\na\\\n\\\nb\\\n`"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -221,7 +221,7 @@ mod tests { #[test] fn lexer_backquote_unclosed_empty() { - let mut lexer = Lexer::from_memory("`", Source::Unknown); + let mut lexer = Lexer::with_code("`"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -242,7 +242,7 @@ mod tests { #[test] fn lexer_backquote_unclosed_nonempty() { - let mut lexer = Lexer::from_memory("`foo", Source::Unknown); + let mut lexer = Lexer::with_code("`foo"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, diff --git a/yash-syntax/src/parser/lex/braced_param.rs b/yash-syntax/src/parser/lex/braced_param.rs index a54b5297..3f66e192 100644 --- a/yash-syntax/src/parser/lex/braced_param.rs +++ b/yash-syntax/src/parser/lex/braced_param.rs @@ -204,7 +204,7 @@ mod tests { #[test] fn lexer_braced_param_none() { - let mut lexer = Lexer::from_memory("$foo", Source::Unknown); + let mut lexer = Lexer::with_code("$foo"); lexer.peek_char().now_or_never().unwrap().unwrap(); lexer.consume_char(); let mut lexer = WordLexer { @@ -217,7 +217,7 @@ mod tests { #[test] fn lexer_braced_param_minimum() { - let mut lexer = Lexer::from_memory("${@};", Source::Unknown); + let mut lexer = Lexer::with_code("${@};"); lexer.peek_char().now_or_never().unwrap().unwrap(); lexer.consume_char(); let mut lexer = WordLexer { @@ -240,7 +240,7 @@ mod tests { #[test] fn lexer_braced_param_alphanumeric_name() { - let mut lexer = Lexer::from_memory("X${foo_123}<", Source::Unknown); + let mut lexer = Lexer::with_code("X${foo_123}<"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -265,7 +265,7 @@ mod tests { #[test] fn lexer_braced_param_positional() { - let mut lexer = Lexer::from_memory("${123}<", Source::Unknown); + let mut lexer = Lexer::with_code("${123}<"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -290,7 +290,7 @@ mod tests { /// parameter with the index 0. Compare [`lexer_braced_param_special_zero`]. #[test] fn lexer_braced_param_positional_zero() { - let mut lexer = Lexer::from_memory("${00}<", Source::Unknown); + let mut lexer = Lexer::with_code("${00}<"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -316,10 +316,7 @@ mod tests { fn lexer_braced_param_positional_overflow() { // This overflow is reported at the execution time of the script, not at // the parsing time. - let mut lexer = Lexer::from_memory( - "${9999999999999999999999999999999999999999}", - Source::Unknown, - ); + let mut lexer = Lexer::with_code("${9999999999999999999999999999999999999999}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -334,7 +331,7 @@ mod tests { #[test] fn lexer_braced_param_invalid_param() { - let mut lexer = Lexer::from_memory("${0_0}", Source::Unknown); + let mut lexer = Lexer::with_code("${0_0}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -354,7 +351,7 @@ mod tests { /// parameter `0`. Compare [`lexer_braced_param_positional_zero`]. #[test] fn lexer_braced_param_special_zero() { - let mut lexer = Lexer::from_memory("${0}<", Source::Unknown); + let mut lexer = Lexer::with_code("${0}<"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -378,7 +375,7 @@ mod tests { #[test] fn lexer_braced_param_special_hash() { - let mut lexer = Lexer::from_memory("${#}<", Source::Unknown); + let mut lexer = Lexer::with_code("${#}<"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -401,7 +398,7 @@ mod tests { #[test] fn lexer_braced_param_missing_name() { - let mut lexer = Lexer::from_memory("${};", Source::Unknown); + let mut lexer = Lexer::with_code("${};"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -419,7 +416,7 @@ mod tests { #[test] fn lexer_braced_param_unclosed_without_name() { - let mut lexer = Lexer::from_memory("${;", Source::Unknown); + let mut lexer = Lexer::with_code("${;"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -437,7 +434,7 @@ mod tests { #[test] fn lexer_braced_param_unclosed_with_name() { - let mut lexer = Lexer::from_memory("${_;", Source::Unknown); + let mut lexer = Lexer::with_code("${_;"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -461,7 +458,7 @@ mod tests { #[test] fn lexer_braced_param_length_alphanumeric_name() { - let mut lexer = Lexer::from_memory("${#foo_123}<", Source::Unknown); + let mut lexer = Lexer::with_code("${#foo_123}<"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -484,7 +481,7 @@ mod tests { #[test] fn lexer_braced_param_length_hash() { - let mut lexer = Lexer::from_memory("${##}<", Source::Unknown); + let mut lexer = Lexer::with_code("${##}<"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -507,7 +504,7 @@ mod tests { #[test] fn lexer_braced_param_length_question() { - let mut lexer = Lexer::from_memory("${#?}<", Source::Unknown); + let mut lexer = Lexer::with_code("${#?}<"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -530,7 +527,7 @@ mod tests { #[test] fn lexer_braced_param_length_hyphen() { - let mut lexer = Lexer::from_memory("${#-}<", Source::Unknown); + let mut lexer = Lexer::with_code("${#-}<"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -553,7 +550,7 @@ mod tests { #[test] fn lexer_braced_param_switch_minimum() { - let mut lexer = Lexer::from_memory("${x+})", Source::Unknown); + let mut lexer = Lexer::with_code("${x+})"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -580,7 +577,7 @@ mod tests { #[test] fn lexer_braced_param_switch_full() { - let mut lexer = Lexer::from_memory("${foo:?'!'})", Source::Unknown); + let mut lexer = Lexer::with_code("${foo:?'!'})"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -607,7 +604,7 @@ mod tests { #[test] fn lexer_braced_param_hash_suffix_alter() { - let mut lexer = Lexer::from_memory("${#+?}<", Source::Unknown); + let mut lexer = Lexer::with_code("${#+?}<"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -634,7 +631,7 @@ mod tests { #[test] fn lexer_braced_param_hash_suffix_default() { - let mut lexer = Lexer::from_memory("${#--}<", Source::Unknown); + let mut lexer = Lexer::with_code("${#--}<"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -661,7 +658,7 @@ mod tests { #[test] fn lexer_braced_param_hash_suffix_assign() { - let mut lexer = Lexer::from_memory("${#=?}<", Source::Unknown); + let mut lexer = Lexer::with_code("${#=?}<"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -688,7 +685,7 @@ mod tests { #[test] fn lexer_braced_param_hash_suffix_error() { - let mut lexer = Lexer::from_memory("${#??}<", Source::Unknown); + let mut lexer = Lexer::with_code("${#??}<"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -715,7 +712,7 @@ mod tests { #[test] fn lexer_braced_param_hash_suffix_with_colon() { - let mut lexer = Lexer::from_memory("${#:-}<", Source::Unknown); + let mut lexer = Lexer::with_code("${#:-}<"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -742,7 +739,7 @@ mod tests { #[test] fn lexer_braced_param_hash_with_longest_prefix_trim() { - let mut lexer = Lexer::from_memory("${###};", Source::Unknown); + let mut lexer = Lexer::with_code("${###};"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -769,7 +766,7 @@ mod tests { #[test] fn lexer_braced_param_hash_with_suffix_trim() { - let mut lexer = Lexer::from_memory("${#%};", Source::Unknown); + let mut lexer = Lexer::with_code("${#%};"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -796,7 +793,7 @@ mod tests { #[test] fn lexer_braced_param_multiple_modifier() { - let mut lexer = Lexer::from_memory("${#x+};", Source::Unknown); + let mut lexer = Lexer::with_code("${#x+};"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -812,7 +809,7 @@ mod tests { #[test] fn lexer_braced_param_line_continuations() { - let mut lexer = Lexer::from_memory("${\\\n#\\\n\\\na_\\\n1\\\n\\\n}z", Source::Unknown); + let mut lexer = Lexer::with_code("${\\\n#\\\n\\\na_\\\n1\\\n\\\n}z"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -838,7 +835,7 @@ mod tests { #[test] fn lexer_braced_param_line_continuations_hash() { - let mut lexer = Lexer::from_memory("${#\\\n\\\n}z", Source::Unknown); + let mut lexer = Lexer::with_code("${#\\\n\\\n}z"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, diff --git a/yash-syntax/src/parser/lex/command_subst.rs b/yash-syntax/src/parser/lex/command_subst.rs index 87706153..12cbafec 100644 --- a/yash-syntax/src/parser/lex/command_subst.rs +++ b/yash-syntax/src/parser/lex/command_subst.rs @@ -65,7 +65,7 @@ mod tests { #[test] fn lexer_command_substitution_success() { - let mut lexer = Lexer::from_memory("$( foo bar )baz", Source::Unknown); + let mut lexer = Lexer::with_code("$( foo bar )baz"); lexer.peek_char().now_or_never().unwrap().unwrap(); lexer.consume_char(); @@ -88,7 +88,7 @@ mod tests { #[test] fn lexer_command_substitution_none() { - let mut lexer = Lexer::from_memory("$ foo bar )baz", Source::Unknown); + let mut lexer = Lexer::with_code("$ foo bar )baz"); lexer.peek_char().now_or_never().unwrap().unwrap(); lexer.consume_char(); @@ -105,7 +105,7 @@ mod tests { #[test] fn lexer_command_substitution_unclosed() { - let mut lexer = Lexer::from_memory("$( foo bar baz", Source::Unknown); + let mut lexer = Lexer::with_code("$( foo bar baz"); lexer.peek_char().now_or_never().unwrap().unwrap(); lexer.consume_char(); diff --git a/yash-syntax/src/parser/lex/core.rs b/yash-syntax/src/parser/lex/core.rs index 09b3f8ec..657ce89c 100644 --- a/yash-syntax/src/parser/lex/core.rs +++ b/yash-syntax/src/parser/lex/core.rs @@ -559,7 +559,27 @@ impl<'a> Lexer<'a> { /// Creates a new lexer with a fixed source code. /// /// This is a convenience function that creates a lexer that reads from a - /// string using a [`Memory`] input function. The line number starts from 1. + /// string using [`Memory`] with the default configuration. + /// + /// This function is best used for testing or for simple cases where you + /// don't need to customize the lexer. For practical use, it is recommended + /// to use the [`config`](Self::config) function to create a configuration + /// and provide it with supplementary information, especially + /// [`source`](Config::source), before creating a lexer. + pub fn with_code(code: &'a str) -> Lexer<'a> { + Self::new(Box::new(Memory::new(code))) + } + + /// Creates a new lexer with a fixed source code. + /// + /// This is a convenience function that creates a lexer that reads from a + /// string using [`Memory`] with the specified source starting from line + /// number 1. + /// + /// This function is soft-deprecated. Use [`with_code`](Self::with_code) + /// instead if the source is `Unknown`. Otherwise, use + /// [`config`](Self::config) to set the source and [`input`](Config::input) + /// to create a lexer, which is more descriptive. pub fn from_memory>>(code: &'a str, source: S) -> Lexer<'a> { fn inner(code: &str, source: Rc) -> Lexer { let mut config = Lexer::config(); @@ -676,15 +696,14 @@ impl<'a> Lexer<'a> { /// /// ``` /// # use yash_syntax::parser::lex::Lexer; - /// # use yash_syntax::source::Source; - /// futures_executor::block_on(async { - /// let mut lexer = Lexer::from_memory("abc", Source::Unknown); - /// assert_eq!(lexer.index(), 0); - /// let _ = lexer.peek_char().await; - /// assert_eq!(lexer.index(), 0); - /// lexer.consume_char(); - /// assert_eq!(lexer.index(), 1); - /// }) + /// # futures_executor::block_on(async { + /// let mut lexer = Lexer::with_code("abc"); + /// assert_eq!(lexer.index(), 0); + /// let _ = lexer.peek_char().await; + /// assert_eq!(lexer.index(), 0); + /// lexer.consume_char(); + /// assert_eq!(lexer.index(), 1); + /// # }) /// ``` #[must_use] pub fn index(&self) -> usize { @@ -699,16 +718,15 @@ impl<'a> Lexer<'a> { /// /// ``` /// # use yash_syntax::parser::lex::Lexer; - /// # use yash_syntax::source::Source; - /// futures_executor::block_on(async { - /// let mut lexer = Lexer::from_memory("abc", Source::Unknown); - /// let saved_index = lexer.index(); - /// assert_eq!(lexer.peek_char().await, Ok(Some('a'))); - /// lexer.consume_char(); - /// assert_eq!(lexer.peek_char().await, Ok(Some('b'))); - /// lexer.rewind(saved_index); - /// assert_eq!(lexer.peek_char().await, Ok(Some('a'))); - /// }) + /// # futures_executor::block_on(async { + /// let mut lexer = Lexer::with_code("abc"); + /// let saved_index = lexer.index(); + /// assert_eq!(lexer.peek_char().await, Ok(Some('a'))); + /// lexer.consume_char(); + /// assert_eq!(lexer.peek_char().await, Ok(Some('b'))); + /// lexer.rewind(saved_index); + /// assert_eq!(lexer.peek_char().await, Ok(Some('a'))); + /// # }) /// ``` pub fn rewind(&mut self, index: usize) { self.core.rewind(index) @@ -1494,27 +1512,27 @@ mod tests { #[test] fn lexer_with_empty_source() { - let mut lexer = Lexer::from_memory("", Source::Unknown); + let mut lexer = Lexer::with_code(""); assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(None)); } #[test] fn lexer_peek_char_with_line_continuation_enabled_stopping_on_non_backslash() { - let mut lexer = Lexer::from_memory("\\\n\n\\", Source::Unknown); + let mut lexer = Lexer::with_code("\\\n\n\\"); assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(Some('\n'))); assert_eq!(lexer.index(), 2); } #[test] fn lexer_peek_char_with_line_continuation_enabled_stopping_on_non_newline() { - let mut lexer = Lexer::from_memory("\\\n\\\n\\\n\\\\", Source::Unknown); + let mut lexer = Lexer::with_code("\\\n\\\n\\\n\\\\"); assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(Some('\\'))); assert_eq!(lexer.index(), 6); } #[test] fn lexer_peek_char_with_line_continuation_disabled() { - let mut lexer = Lexer::from_memory("\\\n\\\n\\\\", Source::Unknown); + let mut lexer = Lexer::with_code("\\\n\\\n\\\\"); let mut lexer = lexer.disable_line_continuation(); assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(Some('\\'))); assert_eq!(lexer.index(), 0); @@ -1522,7 +1540,7 @@ mod tests { #[test] fn lexer_flush() { - let mut lexer = Lexer::from_memory(" \n\n\t\n", Source::Unknown); + let mut lexer = Lexer::with_code(" \n\n\t\n"); let location_1 = lexer.location().now_or_never().unwrap().unwrap().clone(); assert_eq!(*location_1.code.value.borrow(), " \n"); @@ -1549,7 +1567,7 @@ mod tests { #[test] fn lexer_consume_char_if() { - let mut lexer = Lexer::from_memory("word\n", Source::Unknown); + let mut lexer = Lexer::with_code("word\n"); let mut called = 0; let c = lexer @@ -1651,7 +1669,7 @@ mod tests { #[test] fn lexer_location_range_with_empty_range() { - let mut lexer = Lexer::from_memory("", Source::Unknown); + let mut lexer = Lexer::with_code(""); lexer.peek_char().now_or_never().unwrap().unwrap(); let location = lexer.location_range(0..0); assert_eq!(*location.code.value.borrow(), ""); @@ -1695,20 +1713,20 @@ mod tests { #[test] #[should_panic] fn lexer_location_range_with_unconsumed_code() { - let lexer = Lexer::from_memory("echo ok", Source::Unknown); + let lexer = Lexer::with_code("echo ok"); let _ = lexer.location_range(0..0); } #[test] #[should_panic(expected = "The index 1 must not be larger than the current index 0")] fn lexer_location_range_with_range_out_of_bounds() { - let lexer = Lexer::from_memory("", Source::Unknown); + let lexer = Lexer::with_code(""); let _ = lexer.location_range(1..2); } #[test] fn lexer_location_range_with_alias_substitution() { - let mut lexer = Lexer::from_memory(" a;", Source::Unknown); + let mut lexer = Lexer::with_code(" a;"); let alias_def = Rc::new(Alias { name: "a".to_string(), replacement: "abc".to_string(), @@ -1740,14 +1758,14 @@ mod tests { #[test] fn lexer_inner_program_success() { - let mut lexer = Lexer::from_memory("x y )", Source::Unknown); + let mut lexer = Lexer::with_code("x y )"); let source = lexer.inner_program().now_or_never().unwrap().unwrap(); assert_eq!(source, "x y "); } #[test] fn lexer_inner_program_failure() { - let mut lexer = Lexer::from_memory("<< )", Source::Unknown); + let mut lexer = Lexer::with_code("<< )"); let e = lexer.inner_program().now_or_never().unwrap().unwrap_err(); assert_eq!( e.cause, diff --git a/yash-syntax/src/parser/lex/dollar.rs b/yash-syntax/src/parser/lex/dollar.rs index 4973ddf9..24daf4b0 100644 --- a/yash-syntax/src/parser/lex/dollar.rs +++ b/yash-syntax/src/parser/lex/dollar.rs @@ -71,7 +71,7 @@ mod tests { #[test] fn lexer_dollar_unit_no_dollar() { - let mut lexer = Lexer::from_memory("foo", Source::Unknown); + let mut lexer = Lexer::with_code("foo"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -79,7 +79,7 @@ mod tests { let result = lexer.dollar_unit().now_or_never().unwrap().unwrap(); assert_eq!(result, None); - let mut lexer = Lexer::from_memory("()", Source::Unknown); + let mut lexer = Lexer::with_code("()"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -88,7 +88,7 @@ mod tests { assert_eq!(result, None); assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(Some('('))); - let mut lexer = Lexer::from_memory("", Source::Unknown); + let mut lexer = Lexer::with_code(""); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -99,7 +99,7 @@ mod tests { #[test] fn lexer_dollar_unit_dollar_followed_by_non_special() { - let mut lexer = Lexer::from_memory("$;", Source::Unknown); + let mut lexer = Lexer::with_code("$;"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -108,7 +108,7 @@ mod tests { assert_eq!(result, None); assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(Some('$'))); - let mut lexer = Lexer::from_memory("$&", Source::Unknown); + let mut lexer = Lexer::with_code("$&"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -119,7 +119,7 @@ mod tests { #[test] fn lexer_dollar_unit_raw_special_parameter() { - let mut lexer = Lexer::from_memory("$0", Source::Unknown); + let mut lexer = Lexer::with_code("$0"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -138,7 +138,7 @@ mod tests { #[test] fn lexer_dollar_unit_command_substitution() { - let mut lexer = Lexer::from_memory("$()", Source::Unknown); + let mut lexer = Lexer::with_code("$()"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -154,7 +154,7 @@ mod tests { }); assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(None)); - let mut lexer = Lexer::from_memory("$( foo bar )", Source::Unknown); + let mut lexer = Lexer::with_code("$( foo bar )"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -173,7 +173,7 @@ mod tests { #[test] fn lexer_dollar_unit_arithmetic_expansion() { - let mut lexer = Lexer::from_memory("$((1))", Source::Unknown); + let mut lexer = Lexer::with_code("$((1))"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -192,7 +192,7 @@ mod tests { #[test] fn lexer_dollar_unit_line_continuation() { - let mut lexer = Lexer::from_memory("$\\\n\\\n0", Source::Unknown); + let mut lexer = Lexer::with_code("$\\\n\\\n0"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, diff --git a/yash-syntax/src/parser/lex/escape.rs b/yash-syntax/src/parser/lex/escape.rs index 85772f31..bd992290 100644 --- a/yash-syntax/src/parser/lex/escape.rs +++ b/yash-syntax/src/parser/lex/escape.rs @@ -278,7 +278,7 @@ mod tests { #[test] fn escape_unit_literal() { - let mut lexer = Lexer::from_memory("bar", Source::Unknown); + let mut lexer = Lexer::with_code("bar"); let result = lexer.escape_unit().now_or_never().unwrap().unwrap(); assert_eq!(result, Some(Literal('b'))); assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(Some('a'))); @@ -286,7 +286,7 @@ mod tests { #[test] fn escape_unit_named_escapes() { - let mut lexer = Lexer::from_memory(r#"\""\'\\\?\a\b\e\E\f\n\r\t\v"#, Source::Unknown); + let mut lexer = Lexer::with_code(r#"\""\'\\\?\a\b\e\E\f\n\r\t\v"#); let result = lexer.escape_unit().now_or_never().unwrap().unwrap(); assert_eq!(result, Some(DoubleQuote)); let result = lexer.escape_unit().now_or_never().unwrap().unwrap(); @@ -320,7 +320,7 @@ mod tests { #[test] fn escape_unit_incomplete_escapes() { - let mut lexer = Lexer::from_memory(r"\", Source::Unknown); + let mut lexer = Lexer::with_code(r"\"); let error = lexer.escape_unit().now_or_never().unwrap().unwrap_err(); assert_matches!( error.cause, @@ -334,7 +334,7 @@ mod tests { #[test] fn escape_unit_control_escapes() { - let mut lexer = Lexer::from_memory(r"\cA\cz\c^\c?\c\\", Source::Unknown); + let mut lexer = Lexer::with_code(r"\cA\cz\c^\c?\c\\"); let result = lexer.escape_unit().now_or_never().unwrap().unwrap(); assert_eq!(result, Some(Control(0x01))); let result = lexer.escape_unit().now_or_never().unwrap().unwrap(); @@ -350,7 +350,7 @@ mod tests { #[test] fn escape_unit_incomplete_control_escape() { - let mut lexer = Lexer::from_memory(r"\c", Source::Unknown); + let mut lexer = Lexer::with_code(r"\c"); let error = lexer.escape_unit().now_or_never().unwrap().unwrap_err(); assert_matches!( error.cause, @@ -364,7 +364,7 @@ mod tests { #[test] fn escape_unit_incomplete_control_backslash_escapes() { - let mut lexer = Lexer::from_memory(r"\c\", Source::Unknown); + let mut lexer = Lexer::with_code(r"\c\"); let error = lexer.escape_unit().now_or_never().unwrap().unwrap_err(); assert_matches!( error.cause, @@ -375,7 +375,7 @@ mod tests { assert_eq!(*error.location.code.source, Source::Unknown); assert_eq!(error.location.range, 3..3); - let mut lexer = Lexer::from_memory(r"\c\a", Source::Unknown); + let mut lexer = Lexer::with_code(r"\c\a"); let error = lexer.escape_unit().now_or_never().unwrap().unwrap_err(); assert_matches!( error.cause, @@ -389,7 +389,7 @@ mod tests { #[test] fn escape_unit_unknown_control_escape() { - let mut lexer = Lexer::from_memory(r"\c!`", Source::Unknown); + let mut lexer = Lexer::with_code(r"\c!`"); let error = lexer.escape_unit().now_or_never().unwrap().unwrap_err(); assert_matches!( error.cause, @@ -403,7 +403,7 @@ mod tests { #[test] fn escape_unit_octal_escapes() { - let mut lexer = Lexer::from_memory(r"\0\07\234\0123", Source::Unknown); + let mut lexer = Lexer::with_code(r"\0\07\234\0123"); let result = lexer.escape_unit().now_or_never().unwrap().unwrap(); assert_eq!(result, Some(Octal(0o0))); let result = lexer.escape_unit().now_or_never().unwrap().unwrap(); @@ -415,12 +415,12 @@ mod tests { // At most 3 octal digits are consumed assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(Some('3'))); - let mut lexer = Lexer::from_memory(r"\787", Source::Unknown); + let mut lexer = Lexer::with_code(r"\787"); let result = lexer.escape_unit().now_or_never().unwrap().unwrap(); // '8' is not an octal digit assert_eq!(result, Some(Octal(0o7))); - let mut lexer = Lexer::from_memory(r"\12", Source::Unknown); + let mut lexer = Lexer::with_code(r"\12"); let result = lexer.escape_unit().now_or_never().unwrap().unwrap(); // Reaching the end of the input is okay assert_eq!(result, Some(Octal(0o12))); @@ -428,7 +428,7 @@ mod tests { #[test] fn escape_unit_non_byte_octal_escape() { - let mut lexer = Lexer::from_memory(r"\400", Source::Unknown); + let mut lexer = Lexer::with_code(r"\400"); let error = lexer.escape_unit().now_or_never().unwrap().unwrap_err(); assert_matches!( error.cause, @@ -442,7 +442,7 @@ mod tests { #[test] fn escape_unit_hexadecimal_escapes() { - let mut lexer = Lexer::from_memory(r"\x0\x7F\xd4A", Source::Unknown); + let mut lexer = Lexer::with_code(r"\x0\x7F\xd4A"); let result = lexer.escape_unit().now_or_never().unwrap().unwrap(); assert_eq!(result, Some(Hex(0x0))); let result = lexer.escape_unit().now_or_never().unwrap().unwrap(); @@ -452,7 +452,7 @@ mod tests { assert_eq!(result, Some(Hex(0xD4))); assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(Some('A'))); - let mut lexer = Lexer::from_memory(r"\xb", Source::Unknown); + let mut lexer = Lexer::with_code(r"\xb"); let result = lexer.escape_unit().now_or_never().unwrap().unwrap(); // Reaching the end of the input is okay assert_eq!(result, Some(Hex(0xB))); @@ -460,7 +460,7 @@ mod tests { #[test] fn escape_unit_incomplete_hexadecimal_escape() { - let mut lexer = Lexer::from_memory(r"\x", Source::Unknown); + let mut lexer = Lexer::with_code(r"\x"); let error = lexer.escape_unit().now_or_never().unwrap().unwrap_err(); assert_matches!( error.cause, @@ -474,7 +474,7 @@ mod tests { #[test] fn escape_unit_unicode_escapes() { - let mut lexer = Lexer::from_memory(r"\u20\u4B9d0", Source::Unknown); + let mut lexer = Lexer::with_code(r"\u20\u4B9d0"); let result = lexer.escape_unit().now_or_never().unwrap().unwrap(); assert_eq!(result, Some(Unicode('\u{20}'))); let result = lexer.escape_unit().now_or_never().unwrap().unwrap(); @@ -482,7 +482,7 @@ mod tests { // At most 4 hexadecimal digits are consumed assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(Some('0'))); - let mut lexer = Lexer::from_memory(r"\U42\U0001f4A9b", Source::Unknown); + let mut lexer = Lexer::with_code(r"\U42\U0001f4A9b"); let result = lexer.escape_unit().now_or_never().unwrap().unwrap(); assert_eq!(result, Some(Unicode('\u{42}'))); let result = lexer.escape_unit().now_or_never().unwrap().unwrap(); @@ -493,7 +493,7 @@ mod tests { #[test] fn escape_unit_incomplete_unicode_escapes() { - let mut lexer = Lexer::from_memory(r"\u", Source::Unknown); + let mut lexer = Lexer::with_code(r"\u"); let error = lexer.escape_unit().now_or_never().unwrap().unwrap_err(); assert_matches!( error.cause, @@ -504,7 +504,7 @@ mod tests { assert_eq!(*error.location.code.source, Source::Unknown); assert_eq!(error.location.range, 2..2); - let mut lexer = Lexer::from_memory(r"\U", Source::Unknown); + let mut lexer = Lexer::with_code(r"\U"); let error = lexer.escape_unit().now_or_never().unwrap().unwrap_err(); assert_matches!( error.cause, @@ -519,7 +519,7 @@ mod tests { #[test] fn escape_unit_invalid_unicode_escapes() { // U+D800 is not a valid Unicode scalar value - let mut lexer = Lexer::from_memory(r"\uD800", Source::Unknown); + let mut lexer = Lexer::with_code(r"\uD800"); let error = lexer.escape_unit().now_or_never().unwrap().unwrap_err(); assert_matches!( error.cause, @@ -533,7 +533,7 @@ mod tests { #[test] fn escape_unit_unknown_escape() { - let mut lexer = Lexer::from_memory(r"\!", Source::Unknown); + let mut lexer = Lexer::with_code(r"\!"); let error = lexer.escape_unit().now_or_never().unwrap().unwrap_err(); assert_matches!(error.cause, ErrorCause::Syntax(SyntaxError::InvalidEscape)); assert_eq!(*error.location.code.value.borrow(), r"\!"); @@ -546,7 +546,7 @@ mod tests { #[test] fn escaped_string_literals() { - let mut lexer = Lexer::from_memory("foo", Source::Unknown); + let mut lexer = Lexer::with_code("foo"); let EscapedString(content) = lexer .escaped_string(|_| false) .now_or_never() @@ -557,7 +557,7 @@ mod tests { #[test] fn escaped_string_mixed() { - let mut lexer = Lexer::from_memory(r"foo\bar", Source::Unknown); + let mut lexer = Lexer::with_code(r"foo\bar"); let EscapedString(content) = lexer .escaped_string(|_| false) .now_or_never() @@ -578,7 +578,7 @@ mod tests { #[test] fn no_line_continuations_in_escaped_string() { - let mut lexer = Lexer::from_memory("\\\\\n", Source::Unknown); + let mut lexer = Lexer::with_code("\\\\\n"); let EscapedString(content) = lexer .escaped_string(|_| false) .now_or_never() @@ -586,7 +586,7 @@ mod tests { .unwrap(); assert_eq!(content, [Backslash, Literal('\n')]); - let mut lexer = Lexer::from_memory("\\\n", Source::Unknown); + let mut lexer = Lexer::with_code("\\\n"); let error = lexer .escaped_string(|_| false) .now_or_never() @@ -601,7 +601,7 @@ mod tests { #[test] fn single_quoted_escaped_string_empty() { - let mut lexer = Lexer::from_memory("''", Source::Unknown); + let mut lexer = Lexer::with_code("''"); let result = lexer .single_quoted_escaped_string() .now_or_never() @@ -612,7 +612,7 @@ mod tests { #[test] fn single_quoted_escaped_string_nonempty() { - let mut lexer = Lexer::from_memory(r"'foo\e'x", Source::Unknown); + let mut lexer = Lexer::with_code(r"'foo\e'x"); let result = lexer .single_quoted_escaped_string() .now_or_never() @@ -634,7 +634,7 @@ mod tests { #[test] fn single_quoted_escaped_string_unclosed() { - let mut lexer = Lexer::from_memory("'foo", Source::Unknown); + let mut lexer = Lexer::with_code("'foo"); let error = lexer .single_quoted_escaped_string() .now_or_never() diff --git a/yash-syntax/src/parser/lex/heredoc.rs b/yash-syntax/src/parser/lex/heredoc.rs index 16e091c9..234e124b 100644 --- a/yash-syntax/src/parser/lex/heredoc.rs +++ b/yash-syntax/src/parser/lex/heredoc.rs @@ -134,11 +134,11 @@ mod tests { #[test] fn lexer_line() { - let mut lexer = Lexer::from_memory("\n", Source::Unknown); + let mut lexer = Lexer::with_code("\n"); let line = lexer.line().now_or_never().unwrap().unwrap(); assert_eq!(line, ""); - let mut lexer = Lexer::from_memory("foo\n", Source::Unknown); + let mut lexer = Lexer::with_code("foo\n"); let line = lexer.line().now_or_never().unwrap().unwrap(); assert_eq!(line, "foo"); let next = lexer.peek_char().now_or_never().unwrap().unwrap().unwrap(); @@ -157,7 +157,7 @@ mod tests { fn lexer_here_doc_content_empty_content() { let heredoc = here_doc_operator("END", false); - let mut lexer = Lexer::from_memory("END\nX", Source::Unknown); + let mut lexer = Lexer::with_code("END\nX"); lexer .here_doc_content(&heredoc) .now_or_never() @@ -177,7 +177,7 @@ mod tests { fn lexer_here_doc_content_one_line_content() { let heredoc = here_doc_operator("FOO", false); - let mut lexer = Lexer::from_memory("content\nFOO\nX", Source::Unknown); + let mut lexer = Lexer::with_code("content\nFOO\nX"); lexer .here_doc_content(&heredoc) .now_or_never() @@ -197,7 +197,7 @@ mod tests { fn lexer_here_doc_content_long_content() { let heredoc = here_doc_operator("BAR", false); - let mut lexer = Lexer::from_memory("foo\n\tBAR\n\nbaz\nBAR\nX", Source::Unknown); + let mut lexer = Lexer::with_code("foo\n\tBAR\n\nbaz\nBAR\nX"); lexer .here_doc_content(&heredoc) .now_or_never() @@ -220,12 +220,11 @@ mod tests { fn lexer_here_doc_content_escapes_with_unquoted_delimiter() { let heredoc = here_doc_operator("END", false); - let mut lexer = Lexer::from_memory( + let mut lexer = Lexer::with_code( r#"\a\$\"\'\`\\\ X END "#, - Source::Unknown, ); lexer .here_doc_content(&heredoc) @@ -254,12 +253,11 @@ END fn lexer_here_doc_content_escapes_with_quoted_delimiter() { let heredoc = here_doc_operator(r"\END", false); - let mut lexer = Lexer::from_memory( + let mut lexer = Lexer::with_code( r#"\a\$\"\'\`\\\ X END "#, - Source::Unknown, ); lexer .here_doc_content(&heredoc) @@ -293,7 +291,7 @@ END fn lexer_here_doc_content_with_tabs_removed() { let heredoc = here_doc_operator("BAR", true); - let mut lexer = Lexer::from_memory("\t\t\tfoo\n\tBAR\n\nbaz\nBAR\nX", Source::Unknown); + let mut lexer = Lexer::with_code("\t\t\tfoo\n\tBAR\n\nbaz\nBAR\nX"); lexer .here_doc_content(&heredoc) .now_or_never() @@ -313,7 +311,7 @@ END fn lexer_here_doc_content_unclosed() { let heredoc = here_doc_operator("END", false); - let mut lexer = Lexer::from_memory("", Source::Unknown); + let mut lexer = Lexer::with_code(""); let e = lexer .here_doc_content(&heredoc) .now_or_never() diff --git a/yash-syntax/src/parser/lex/misc.rs b/yash-syntax/src/parser/lex/misc.rs index 6fc294dc..7369c4c6 100644 --- a/yash-syntax/src/parser/lex/misc.rs +++ b/yash-syntax/src/parser/lex/misc.rs @@ -67,12 +67,11 @@ impl Lexer<'_> { #[cfg(test)] mod tests { use super::*; - use crate::source::Source; use futures_util::FutureExt; #[test] fn lexer_skip_blanks() { - let mut lexer = Lexer::from_memory(" \t w", Source::Unknown); + let mut lexer = Lexer::with_code(" \t w"); let c = async { lexer.skip_blanks().await?; @@ -94,14 +93,14 @@ mod tests { #[test] fn lexer_skip_blanks_does_not_skip_newline() { - let mut lexer = Lexer::from_memory("\n", Source::Unknown); + let mut lexer = Lexer::with_code("\n"); lexer.skip_blanks().now_or_never().unwrap().unwrap(); assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(Some('\n'))); } #[test] fn lexer_skip_blanks_skips_line_continuations() { - let mut lexer = Lexer::from_memory("\\\n \\\n\\\n\\\n \\\nX", Source::Unknown); + let mut lexer = Lexer::with_code("\\\n \\\n\\\n\\\n \\\nX"); let c = async { lexer.skip_blanks().await?; lexer.peek_char().await @@ -110,7 +109,7 @@ mod tests { .unwrap(); assert_eq!(c, Ok(Some('X'))); - let mut lexer = Lexer::from_memory(" \\\n\\\n \\\n Y", Source::Unknown); + let mut lexer = Lexer::with_code(" \\\n\\\n \\\n Y"); let c = async { lexer.skip_blanks().await?; lexer.peek_char().await @@ -122,14 +121,14 @@ mod tests { #[test] fn lexer_skip_comment_no_comment() { - let mut lexer = Lexer::from_memory("\n", Source::Unknown); + let mut lexer = Lexer::with_code("\n"); lexer.skip_comment().now_or_never().unwrap().unwrap(); assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(Some('\n'))); } #[test] fn lexer_skip_comment_empty_comment() { - let mut lexer = Lexer::from_memory("#\n", Source::Unknown); + let mut lexer = Lexer::with_code("#\n"); let c = async { lexer.skip_comment().await?; @@ -151,7 +150,7 @@ mod tests { #[test] fn lexer_skip_comment_non_empty_comment() { - let mut lexer = Lexer::from_memory("\\\n### foo bar\\\n", Source::Unknown); + let mut lexer = Lexer::with_code("\\\n### foo bar\\\n"); let c = async { lexer.skip_comment().await?; @@ -175,7 +174,7 @@ mod tests { #[test] fn lexer_skip_comment_not_ending_with_newline() { - let mut lexer = Lexer::from_memory("#comment", Source::Unknown); + let mut lexer = Lexer::with_code("#comment"); let c = async { lexer.skip_comment().await?; diff --git a/yash-syntax/src/parser/lex/modifier.rs b/yash-syntax/src/parser/lex/modifier.rs index 80aa387c..fe96b56c 100644 --- a/yash-syntax/src/parser/lex/modifier.rs +++ b/yash-syntax/src/parser/lex/modifier.rs @@ -149,7 +149,6 @@ impl WordLexer<'_, '_> { mod tests { use super::*; use crate::parser::error::ErrorCause; - use crate::source::Source; use crate::syntax::Text; use crate::syntax::TextUnit; use crate::syntax::WordUnit; @@ -158,7 +157,7 @@ mod tests { #[test] fn lexer_suffix_modifier_eof() { - let mut lexer = Lexer::from_memory("", Source::Unknown); + let mut lexer = Lexer::with_code(""); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -170,7 +169,7 @@ mod tests { #[test] fn lexer_suffix_modifier_none() { - let mut lexer = Lexer::from_memory("}", Source::Unknown); + let mut lexer = Lexer::with_code("}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -184,7 +183,7 @@ mod tests { #[test] fn lexer_suffix_modifier_alter_empty() { - let mut lexer = Lexer::from_memory("+}", Source::Unknown); + let mut lexer = Lexer::with_code("+}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -204,7 +203,7 @@ mod tests { #[test] fn lexer_suffix_modifier_alter_word() { - let mut lexer = Lexer::from_memory(r"+a z}", Source::Unknown); + let mut lexer = Lexer::with_code(r"+a z}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -232,7 +231,7 @@ mod tests { #[test] fn lexer_suffix_modifier_colon_alter_empty() { - let mut lexer = Lexer::from_memory(":+}", Source::Unknown); + let mut lexer = Lexer::with_code(":+}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -252,7 +251,7 @@ mod tests { #[test] fn lexer_suffix_modifier_default_empty() { - let mut lexer = Lexer::from_memory("-}", Source::Unknown); + let mut lexer = Lexer::with_code("-}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -272,7 +271,7 @@ mod tests { #[test] fn lexer_suffix_modifier_colon_default_word() { - let mut lexer = Lexer::from_memory(r":-cool}", Source::Unknown); + let mut lexer = Lexer::with_code(r":-cool}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -300,7 +299,7 @@ mod tests { #[test] fn lexer_suffix_modifier_colon_assign_empty() { - let mut lexer = Lexer::from_memory(":=}", Source::Unknown); + let mut lexer = Lexer::with_code(":=}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -320,7 +319,7 @@ mod tests { #[test] fn lexer_suffix_modifier_assign_word() { - let mut lexer = Lexer::from_memory(r"=Yes}", Source::Unknown); + let mut lexer = Lexer::with_code(r"=Yes}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -347,7 +346,7 @@ mod tests { #[test] fn lexer_suffix_modifier_error_empty() { - let mut lexer = Lexer::from_memory("?}", Source::Unknown); + let mut lexer = Lexer::with_code("?}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -367,7 +366,7 @@ mod tests { #[test] fn lexer_suffix_modifier_colon_error_word() { - let mut lexer = Lexer::from_memory(r":?No}", Source::Unknown); + let mut lexer = Lexer::with_code(r":?No}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -393,7 +392,7 @@ mod tests { #[test] fn lexer_suffix_modifier_tilde_expansion_in_switch_word_in_word_context() { - let mut lexer = Lexer::from_memory(r"-~}", Source::Unknown); + let mut lexer = Lexer::with_code(r"-~}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -407,7 +406,7 @@ mod tests { #[test] fn lexer_suffix_modifier_tilde_expansion_in_switch_word_in_text_context() { - let mut lexer = Lexer::from_memory(r"-~}", Source::Unknown); + let mut lexer = Lexer::with_code(r"-~}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Text, @@ -424,7 +423,7 @@ mod tests { #[test] fn lexer_suffix_modifier_trim_shortest_prefix_in_word_context() { - let mut lexer = Lexer::from_memory("#'*'}", Source::Unknown); + let mut lexer = Lexer::with_code("#'*'}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -444,7 +443,7 @@ mod tests { #[test] fn lexer_suffix_modifier_trim_shortest_prefix_in_text_context() { - let mut lexer = Lexer::from_memory("#'*'}", Source::Unknown); + let mut lexer = Lexer::with_code("#'*'}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Text, @@ -464,7 +463,7 @@ mod tests { #[test] fn lexer_suffix_modifier_trim_longest_prefix() { - let mut lexer = Lexer::from_memory(r#"##"?"}"#, Source::Unknown); + let mut lexer = Lexer::with_code(r#"##"?"}"#); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -487,7 +486,7 @@ mod tests { #[test] fn lexer_suffix_modifier_trim_shortest_suffix() { - let mut lexer = Lexer::from_memory(r"%\%}", Source::Unknown); + let mut lexer = Lexer::with_code(r"%\%}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -510,7 +509,7 @@ mod tests { #[test] fn lexer_suffix_modifier_trim_longest_suffix() { - let mut lexer = Lexer::from_memory("%%%}", Source::Unknown); + let mut lexer = Lexer::with_code("%%%}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -533,7 +532,7 @@ mod tests { #[test] fn lexer_suffix_modifier_tilde_expansion_in_trim_word() { - let mut lexer = Lexer::from_memory(r"#~}", Source::Unknown); + let mut lexer = Lexer::with_code(r"#~}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -547,7 +546,7 @@ mod tests { #[test] fn lexer_suffix_modifier_orphan_colon_eof() { - let mut lexer = Lexer::from_memory(r":", Source::Unknown); + let mut lexer = Lexer::with_code(r":"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -561,7 +560,7 @@ mod tests { #[test] fn lexer_suffix_modifier_orphan_colon_followed_by_letter() { - let mut lexer = Lexer::from_memory(r":x}", Source::Unknown); + let mut lexer = Lexer::with_code(r":x}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -575,7 +574,7 @@ mod tests { #[test] fn lexer_suffix_modifier_orphan_colon_followed_by_symbol() { - let mut lexer = Lexer::from_memory(r":#}", Source::Unknown); + let mut lexer = Lexer::with_code(r":#}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, diff --git a/yash-syntax/src/parser/lex/op.rs b/yash-syntax/src/parser/lex/op.rs index 569ee650..4c2c88ae 100644 --- a/yash-syntax/src/parser/lex/op.rs +++ b/yash-syntax/src/parser/lex/op.rs @@ -466,7 +466,7 @@ mod tests { #[test] fn lexer_operator_longest_match() { - let mut lexer = Lexer::from_memory("<<-", Source::Unknown); + let mut lexer = Lexer::with_code("<<-"); let t = lexer.operator().now_or_never().unwrap().unwrap().unwrap(); assert_eq!(t.word.units.len(), 3); @@ -484,7 +484,7 @@ mod tests { #[test] fn lexer_operator_delimited_by_another_operator() { - let mut lexer = Lexer::from_memory("<<>", Source::Unknown); + let mut lexer = Lexer::with_code("<<>"); let t = lexer.operator().now_or_never().unwrap().unwrap().unwrap(); assert_eq!(t.word.units.len(), 2); @@ -504,7 +504,7 @@ mod tests { #[test] fn lexer_operator_delimited_by_eof() { - let mut lexer = Lexer::from_memory("<<", Source::Unknown); + let mut lexer = Lexer::with_code("<<"); let t = lexer.operator().now_or_never().unwrap().unwrap().unwrap(); assert_eq!(t.word.units.len(), 2); @@ -521,7 +521,7 @@ mod tests { #[test] fn lexer_operator_containing_line_continuations() { - let mut lexer = Lexer::from_memory("\\\n\\\n<\\\n<\\\n>", Source::Unknown); + let mut lexer = Lexer::with_code("\\\n\\\n<\\\n<\\\n>"); let t = lexer.operator().now_or_never().unwrap().unwrap().unwrap(); assert_eq!(t.word.units.len(), 2); @@ -538,7 +538,7 @@ mod tests { #[test] fn lexer_operator_none() { - let mut lexer = Lexer::from_memory("\\\n ", Source::Unknown); + let mut lexer = Lexer::with_code("\\\n "); let r = lexer.operator().now_or_never().unwrap().unwrap(); assert!(r.is_none(), "unexpected success: {r:?}"); diff --git a/yash-syntax/src/parser/lex/raw_param.rs b/yash-syntax/src/parser/lex/raw_param.rs index e0dae012..3e8e0fb0 100644 --- a/yash-syntax/src/parser/lex/raw_param.rs +++ b/yash-syntax/src/parser/lex/raw_param.rs @@ -98,7 +98,7 @@ mod tests { #[test] fn lexer_raw_param_special_parameter() { - let mut lexer = Lexer::from_memory("$@;", Source::Unknown); + let mut lexer = Lexer::with_code("$@;"); lexer.peek_char().now_or_never().unwrap().unwrap(); lexer.consume_char(); @@ -116,7 +116,7 @@ mod tests { #[test] fn lexer_raw_param_digit() { - let mut lexer = Lexer::from_memory("$12", Source::Unknown); + let mut lexer = Lexer::with_code("$12"); lexer.peek_char().now_or_never().unwrap().unwrap(); lexer.consume_char(); @@ -134,7 +134,7 @@ mod tests { #[test] fn lexer_raw_param_posix_name() { - let mut lexer = Lexer::from_memory("$az_AZ_019<", Source::Unknown); + let mut lexer = Lexer::with_code("$az_AZ_019<"); lexer.peek_char().now_or_never().unwrap().unwrap(); lexer.consume_char(); @@ -152,7 +152,7 @@ mod tests { #[test] fn lexer_raw_param_posix_name_line_continuations() { - let mut lexer = Lexer::from_memory("$a\\\n\\\nb\\\n\\\nc\\\n>", Source::Unknown); + let mut lexer = Lexer::with_code("$a\\\n\\\nb\\\n\\\nc\\\n>"); lexer.peek_char().now_or_never().unwrap().unwrap(); lexer.consume_char(); @@ -170,7 +170,7 @@ mod tests { #[test] fn lexer_raw_param_not_parameter() { - let mut lexer = Lexer::from_memory("$;", Source::Unknown); + let mut lexer = Lexer::with_code("$;"); lexer.peek_char().now_or_never().unwrap().unwrap(); lexer.consume_char(); assert_eq!(lexer.raw_param(0).now_or_never().unwrap(), Ok(None)); diff --git a/yash-syntax/src/parser/lex/text.rs b/yash-syntax/src/parser/lex/text.rs index 2c245df0..1af39d17 100644 --- a/yash-syntax/src/parser/lex/text.rs +++ b/yash-syntax/src/parser/lex/text.rs @@ -226,7 +226,7 @@ mod tests { #[test] fn lexer_text_unit_literal_accepted() { - let mut lexer = Lexer::from_memory("X", Source::Unknown); + let mut lexer = Lexer::with_code("X"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -253,7 +253,7 @@ mod tests { #[test] fn lexer_text_unit_literal_rejected() { - let mut lexer = Lexer::from_memory(";", Source::Unknown); + let mut lexer = Lexer::with_code(";"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -279,7 +279,7 @@ mod tests { #[test] fn lexer_text_unit_backslash_accepted() { - let mut lexer = Lexer::from_memory(r"\#", Source::Unknown); + let mut lexer = Lexer::with_code(r"\#"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -306,7 +306,7 @@ mod tests { #[test] fn lexer_text_unit_backslash_eof() { - let mut lexer = Lexer::from_memory(r"\", Source::Unknown); + let mut lexer = Lexer::with_code(r"\"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -327,7 +327,7 @@ mod tests { #[test] fn lexer_text_unit_backslash_line_continuation_not_recognized() { - let mut lexer = Lexer::from_memory("\\\\\n", Source::Unknown); + let mut lexer = Lexer::with_code("\\\\\n"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -354,7 +354,7 @@ mod tests { #[test] fn lexer_text_unit_dollar() { - let mut lexer = Lexer::from_memory("$()", Source::Unknown); + let mut lexer = Lexer::with_code("$()"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -378,7 +378,7 @@ mod tests { #[test] fn lexer_text_unit_backquote_double_quote_escapable() { - let mut lexer = Lexer::from_memory(r#"`\"`"#, Source::Unknown); + let mut lexer = Lexer::with_code(r#"`\"`"#); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Text, @@ -402,7 +402,7 @@ mod tests { #[test] fn lexer_text_unit_backquote_double_quote_not_escapable() { - let mut lexer = Lexer::from_memory(r#"`\"`"#, Source::Unknown); + let mut lexer = Lexer::with_code(r#"`\"`"#); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -429,7 +429,7 @@ mod tests { #[test] fn lexer_text_unit_line_continuations() { - let mut lexer = Lexer::from_memory("\\\n\\\nX", Source::Unknown); + let mut lexer = Lexer::with_code("\\\n\\\nX"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -450,7 +450,7 @@ mod tests { #[test] fn lexer_text_empty() { - let mut lexer = Lexer::from_memory("", Source::Unknown); + let mut lexer = Lexer::with_code(""); let Text(units) = lexer .text( |c| unreachable!("unexpected call to is_delimiter({:?})", c), @@ -464,7 +464,7 @@ mod tests { #[test] fn lexer_text_nonempty() { - let mut lexer = Lexer::from_memory("abc", Source::Unknown); + let mut lexer = Lexer::with_code("abc"); let mut called = 0; let Text(units) = lexer .text( @@ -489,7 +489,7 @@ mod tests { #[test] fn lexer_text_delimiter() { - let mut lexer = Lexer::from_memory("abc", Source::Unknown); + let mut lexer = Lexer::with_code("abc"); let mut called = 0; let Text(units) = lexer .text( @@ -514,7 +514,7 @@ mod tests { #[test] fn lexer_text_escaping() { - let mut lexer = Lexer::from_memory(r"a\b\c", Source::Unknown); + let mut lexer = Lexer::with_code(r"a\b\c"); let mut tested_chars = String::new(); let Text(units) = lexer .text( @@ -538,7 +538,7 @@ mod tests { #[test] fn lexer_text_with_parentheses_no_parentheses() { - let mut lexer = Lexer::from_memory("abc", Source::Unknown); + let mut lexer = Lexer::with_code("abc"); let Text(units) = lexer .text_with_parentheses(|_| false, |_| false) .now_or_never() @@ -551,7 +551,7 @@ mod tests { #[test] fn lexer_text_with_parentheses_nest_1() { - let mut lexer = Lexer::from_memory("a(b)c)", Source::Unknown); + let mut lexer = Lexer::with_code("a(b)c)"); let Text(units) = lexer .text_with_parentheses(|c| c == 'b' || c == ')', |_| false) .now_or_never() @@ -573,7 +573,7 @@ mod tests { #[test] fn lexer_text_with_parentheses_nest_1_1() { - let mut lexer = Lexer::from_memory("ab(CD)ef(GH)ij;", Source::Unknown); + let mut lexer = Lexer::with_code("ab(CD)ef(GH)ij;"); let Text(units) = lexer .text_with_parentheses(|c| c.is_ascii_uppercase() || c == ';', |_| false) .now_or_never() @@ -604,7 +604,7 @@ mod tests { #[test] fn lexer_text_with_parentheses_nest_3() { - let mut lexer = Lexer::from_memory("a(B((C)D))e;", Source::Unknown); + let mut lexer = Lexer::with_code("a(B((C)D))e;"); let Text(units) = lexer .text_with_parentheses(|c| c.is_ascii_uppercase() || c == ';', |_| false) .now_or_never() @@ -632,7 +632,7 @@ mod tests { #[test] fn lexer_text_with_parentheses_unclosed() { - let mut lexer = Lexer::from_memory("x(()", Source::Unknown); + let mut lexer = Lexer::with_code("x(()"); let e = lexer .text_with_parentheses(|_| false, |_| false) .now_or_never() diff --git a/yash-syntax/src/parser/lex/token.rs b/yash-syntax/src/parser/lex/token.rs index decccd9b..582393a2 100644 --- a/yash-syntax/src/parser/lex/token.rs +++ b/yash-syntax/src/parser/lex/token.rs @@ -96,7 +96,7 @@ mod tests { #[test] fn lexer_token_empty() { // If there's no word unit that can be parsed, it is the end of input. - let mut lexer = Lexer::from_memory("", Source::Unknown); + let mut lexer = Lexer::with_code(""); let t = lexer.token().now_or_never().unwrap().unwrap(); assert_eq!(*t.word.location.code.value.borrow(), ""); @@ -109,7 +109,7 @@ mod tests { #[test] fn lexer_token_non_empty() { - let mut lexer = Lexer::from_memory("abc ", Source::Unknown); + let mut lexer = Lexer::with_code("abc "); let t = lexer.token().now_or_never().unwrap().unwrap(); assert_eq!(t.word.units.len(), 3); @@ -128,7 +128,7 @@ mod tests { #[test] fn lexer_token_tilde() { - let mut lexer = Lexer::from_memory("~a:~", Source::Unknown); + let mut lexer = Lexer::with_code("~a:~"); let t = lexer.token().now_or_never().unwrap().unwrap(); assert_eq!(t.word.units, [WordUnit::Tilde("a:~".to_string())]); @@ -136,7 +136,7 @@ mod tests { #[test] fn lexer_token_io_number_delimited_by_less() { - let mut lexer = Lexer::from_memory("12<", Source::Unknown); + let mut lexer = Lexer::with_code("12<"); let t = lexer.token().now_or_never().unwrap().unwrap(); assert_eq!(t.word.units.len(), 2); @@ -154,7 +154,7 @@ mod tests { #[test] fn lexer_token_io_number_delimited_by_greater() { - let mut lexer = Lexer::from_memory("0>>", Source::Unknown); + let mut lexer = Lexer::with_code("0>>"); let t = lexer.token().now_or_never().unwrap().unwrap(); assert_eq!(t.word.units.len(), 1); @@ -174,7 +174,7 @@ mod tests { #[test] fn lexer_token_after_blank() { - let mut lexer = Lexer::from_memory(" a ", Source::Unknown); + let mut lexer = Lexer::with_code(" a "); lexer.skip_blanks().now_or_never().unwrap().unwrap(); let t = lexer.token().now_or_never().unwrap().unwrap(); diff --git a/yash-syntax/src/parser/lex/word.rs b/yash-syntax/src/parser/lex/word.rs index d7c4a559..116e5d15 100644 --- a/yash-syntax/src/parser/lex/word.rs +++ b/yash-syntax/src/parser/lex/word.rs @@ -190,7 +190,7 @@ mod tests { #[test] fn lexer_word_unit_unquoted() { - let mut lexer = Lexer::from_memory("$()", Source::Unknown); + let mut lexer = Lexer::with_code("$()"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -212,7 +212,7 @@ mod tests { #[test] fn lexer_word_unit_unquoted_escapes_in_word_context() { // Any characters can be escaped in this context. - let mut lexer = Lexer::from_memory(r#"\a\$\`\"\\\'\#\{\}"#, Source::Unknown); + let mut lexer = Lexer::with_code(r#"\a\$\`\"\\\'\#\{\}"#); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -270,7 +270,7 @@ mod tests { #[test] fn lexer_word_unit_unquoted_escapes_in_text_context() { // $, `, " and \ can be escaped as well as delimiters - let mut lexer = Lexer::from_memory(r#"\a\$\`\"\\\'\#\{\}"#, Source::Unknown); + let mut lexer = Lexer::with_code(r#"\a\$\`\"\\\'\#\{\}"#); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Text, @@ -309,7 +309,7 @@ mod tests { #[test] fn lexer_word_unit_orphan_dollar_is_literal() { - let mut lexer = Lexer::from_memory("$", Source::Unknown); + let mut lexer = Lexer::with_code("$"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -330,7 +330,7 @@ mod tests { #[test] fn lexer_word_unit_single_quote_empty() { - let mut lexer = Lexer::from_memory("''", Source::Unknown); + let mut lexer = Lexer::with_code("''"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -348,7 +348,7 @@ mod tests { #[test] fn lexer_word_unit_single_quote_nonempty() { - let mut lexer = Lexer::from_memory("'abc\\\n$def\\'", Source::Unknown); + let mut lexer = Lexer::with_code("'abc\\\n$def\\'"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -366,7 +366,7 @@ mod tests { #[test] fn lexer_word_unit_single_quote_unclosed() { - let mut lexer = Lexer::from_memory("'abc\ndef\\", Source::Unknown); + let mut lexer = Lexer::with_code("'abc\ndef\\"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -392,7 +392,7 @@ mod tests { #[test] fn lexer_word_unit_not_single_quote_in_text_context() { - let mut lexer = Lexer::from_memory("'", Source::Unknown); + let mut lexer = Lexer::with_code("'"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Text, @@ -414,7 +414,7 @@ mod tests { #[test] fn lexer_word_unit_dollar_single_quote_empty() { - let mut lexer = Lexer::from_memory("$''", Source::Unknown); + let mut lexer = Lexer::with_code("$''"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -435,7 +435,7 @@ mod tests { #[test] fn lexer_word_unit_dollar_single_quote_nonempty() { - let mut lexer = Lexer::from_memory(r"$'foo'", Source::Unknown); + let mut lexer = Lexer::with_code(r"$'foo'"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -465,7 +465,7 @@ mod tests { #[test] fn lexer_word_unit_not_dollar_single_quote_in_text_context() { - let mut lexer = Lexer::from_memory("$''", Source::Unknown); + let mut lexer = Lexer::with_code("$''"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Text, @@ -484,7 +484,7 @@ mod tests { #[test] fn lexer_word_unit_double_quote_empty() { - let mut lexer = Lexer::from_memory("\"\"", Source::Unknown); + let mut lexer = Lexer::with_code("\"\""); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -502,7 +502,7 @@ mod tests { #[test] fn lexer_word_unit_double_quote_non_empty() { - let mut lexer = Lexer::from_memory("\"abc\"", Source::Unknown); + let mut lexer = Lexer::with_code("\"abc\""); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -523,7 +523,7 @@ mod tests { #[test] fn lexer_word_unit_double_quote_escapes() { // Only the following can be escaped in this context: $ ` " \ - let mut lexer = Lexer::from_memory(r#""\a\$\`\"\\\'\#""#, Source::Unknown); + let mut lexer = Lexer::with_code(r#""\a\$\`\"\\\'\#""#); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -560,7 +560,7 @@ mod tests { #[test] fn lexer_word_unit_double_quote_unclosed() { - let mut lexer = Lexer::from_memory("\"abc\ndef", Source::Unknown); + let mut lexer = Lexer::with_code("\"abc\ndef"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -586,7 +586,7 @@ mod tests { #[test] fn lexer_word_nonempty() { - let mut lexer = Lexer::from_memory(r"0$(:)X\#", Source::Unknown); + let mut lexer = Lexer::with_code(r"0$(:)X\#"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -614,7 +614,7 @@ mod tests { #[test] fn lexer_word_empty() { - let mut lexer = Lexer::from_memory("", Source::Unknown); + let mut lexer = Lexer::with_code(""); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -633,7 +633,7 @@ mod tests { #[test] fn lexer_word_with_switch_in_word_context() { - let mut lexer = Lexer::from_memory(r"${x-~}", Source::Unknown); + let mut lexer = Lexer::with_code(r"${x-~}"); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, @@ -656,7 +656,7 @@ mod tests { #[test] fn lexer_word_with_switch_in_text_context() { - let mut lexer = Lexer::from_memory(r#""${x-~}""#, Source::Unknown); + let mut lexer = Lexer::with_code(r#""${x-~}""#); let mut lexer = WordLexer { lexer: &mut lexer, context: WordContext::Word, diff --git a/yash-syntax/src/parser/list.rs b/yash-syntax/src/parser/list.rs index 030de057..eff70e95 100644 --- a/yash-syntax/src/parser/list.rs +++ b/yash-syntax/src/parser/list.rs @@ -217,7 +217,7 @@ mod tests { #[test] fn parser_list_eof() { - let mut lexer = Lexer::from_memory("", Source::Unknown); + let mut lexer = Lexer::with_code(""); let mut parser = Parser::new(&mut lexer); let list = parser.list().now_or_never().unwrap().unwrap().unwrap(); @@ -226,7 +226,7 @@ mod tests { #[test] fn parser_list_one_item_without_last_semicolon() { - let mut lexer = Lexer::from_memory("foo", Source::Unknown); + let mut lexer = Lexer::with_code("foo"); let mut parser = Parser::new(&mut lexer); let list = parser.list().now_or_never().unwrap().unwrap().unwrap(); @@ -237,7 +237,7 @@ mod tests { #[test] fn parser_list_one_item_with_last_semicolon() { - let mut lexer = Lexer::from_memory("foo;", Source::Unknown); + let mut lexer = Lexer::with_code("foo;"); let mut parser = Parser::new(&mut lexer); let list = parser.list().now_or_never().unwrap().unwrap().unwrap(); @@ -248,7 +248,7 @@ mod tests { #[test] fn parser_list_many_items() { - let mut lexer = Lexer::from_memory("foo & bar ; baz&", Source::Unknown); + let mut lexer = Lexer::with_code("foo & bar ; baz&"); let mut parser = Parser::new(&mut lexer); let list = parser.list().now_or_never().unwrap().unwrap().unwrap(); @@ -274,7 +274,7 @@ mod tests { #[test] fn parser_command_line_eof() { - let mut lexer = Lexer::from_memory("", Source::Unknown); + let mut lexer = Lexer::with_code(""); let mut parser = Parser::new(&mut lexer); let result = parser.command_line().now_or_never().unwrap().unwrap(); @@ -283,7 +283,7 @@ mod tests { #[test] fn parser_command_line_command_and_newline() { - let mut lexer = Lexer::from_memory("< /dev/null\n", Source::Unknown); + let mut lexer = Lexer::with_code("<> /dev/null\n"); let mut parser = Parser::new(&mut lexer); let result = parser.redirection().now_or_never().unwrap(); @@ -178,7 +178,7 @@ mod tests { #[test] fn parser_redirection_greater() { - let mut lexer = Lexer::from_memory(">/dev/null\n", Source::Unknown); + let mut lexer = Lexer::with_code(">/dev/null\n"); let mut parser = Parser::new(&mut lexer); let result = parser.redirection().now_or_never().unwrap(); @@ -192,7 +192,7 @@ mod tests { #[test] fn parser_redirection_greater_greater() { - let mut lexer = Lexer::from_memory(" >> /dev/null\n", Source::Unknown); + let mut lexer = Lexer::with_code(" >> /dev/null\n"); let mut parser = Parser::new(&mut lexer); let result = parser.redirection().now_or_never().unwrap(); @@ -206,7 +206,7 @@ mod tests { #[test] fn parser_redirection_greater_bar() { - let mut lexer = Lexer::from_memory(">| /dev/null\n", Source::Unknown); + let mut lexer = Lexer::with_code(">| /dev/null\n"); let mut parser = Parser::new(&mut lexer); let result = parser.redirection().now_or_never().unwrap(); @@ -220,7 +220,7 @@ mod tests { #[test] fn parser_redirection_less_and() { - let mut lexer = Lexer::from_memory("<& -\n", Source::Unknown); + let mut lexer = Lexer::with_code("<& -\n"); let mut parser = Parser::new(&mut lexer); let result = parser.redirection().now_or_never().unwrap(); @@ -234,7 +234,7 @@ mod tests { #[test] fn parser_redirection_greater_and() { - let mut lexer = Lexer::from_memory(">& 3\n", Source::Unknown); + let mut lexer = Lexer::with_code(">& 3\n"); let mut parser = Parser::new(&mut lexer); let result = parser.redirection().now_or_never().unwrap(); @@ -248,7 +248,7 @@ mod tests { #[test] fn parser_redirection_greater_greater_bar() { - let mut lexer = Lexer::from_memory(">>| 3\n", Source::Unknown); + let mut lexer = Lexer::with_code(">>| 3\n"); let mut parser = Parser::new(&mut lexer); let result = parser.redirection().now_or_never().unwrap(); @@ -262,7 +262,7 @@ mod tests { #[test] fn parser_redirection_less_less_less() { - let mut lexer = Lexer::from_memory("<<< foo\n", Source::Unknown); + let mut lexer = Lexer::with_code("<<< foo\n"); let mut parser = Parser::new(&mut lexer); let result = parser.redirection().now_or_never().unwrap(); @@ -276,7 +276,7 @@ mod tests { #[test] fn parser_redirection_less_less() { - let mut lexer = Lexer::from_memory("<", Source::Unknown); + let mut lexer = Lexer::with_code(" < >"); let mut parser = Parser::new(&mut lexer); let e = parser.redirection().now_or_never().unwrap().unwrap_err(); @@ -377,7 +374,7 @@ mod tests { #[test] fn parser_redirection_eof_operand() { - let mut lexer = Lexer::from_memory(" < ", Source::Unknown); + let mut lexer = Lexer::with_code(" < "); let mut parser = Parser::new(&mut lexer); let e = parser.redirection().now_or_never().unwrap().unwrap_err(); @@ -393,7 +390,7 @@ mod tests { #[test] fn parser_redirection_not_heredoc_delimiter() { - let mut lexer = Lexer::from_memory("<< <<", Source::Unknown); + let mut lexer = Lexer::with_code("<< <<"); let mut parser = Parser::new(&mut lexer); let e = parser.redirection().now_or_never().unwrap().unwrap_err(); @@ -409,7 +406,7 @@ mod tests { #[test] fn parser_redirection_eof_heredoc_delimiter() { - let mut lexer = Lexer::from_memory("<<", Source::Unknown); + let mut lexer = Lexer::with_code("<<"); let mut parser = Parser::new(&mut lexer); let e = parser.redirection().now_or_never().unwrap().unwrap_err(); diff --git a/yash-syntax/src/parser/simple_command.rs b/yash-syntax/src/parser/simple_command.rs index 6a6cc9fd..dce91b1d 100644 --- a/yash-syntax/src/parser/simple_command.rs +++ b/yash-syntax/src/parser/simple_command.rs @@ -260,7 +260,7 @@ mod tests { #[test] fn parser_array_values_no_open_parenthesis() { - let mut lexer = Lexer::from_memory(")", Source::Unknown); + let mut lexer = Lexer::with_code(")"); let mut parser = Parser::new(&mut lexer); let result = parser.array_values().now_or_never().unwrap().unwrap(); assert_eq!(result, None); @@ -268,7 +268,7 @@ mod tests { #[test] fn parser_array_values_empty() { - let mut lexer = Lexer::from_memory("()", Source::Unknown); + let mut lexer = Lexer::with_code("()"); let mut parser = Parser::new(&mut lexer); let result = parser.array_values().now_or_never().unwrap(); let words = result.unwrap().unwrap(); @@ -280,7 +280,7 @@ mod tests { #[test] fn parser_array_values_many() { - let mut lexer = Lexer::from_memory("(a b c)", Source::Unknown); + let mut lexer = Lexer::with_code("(a b c)"); let mut parser = Parser::new(&mut lexer); let result = parser.array_values().now_or_never().unwrap(); let words = result.unwrap().unwrap(); @@ -292,12 +292,11 @@ mod tests { #[test] fn parser_array_values_newlines_and_comments() { - let mut lexer = Lexer::from_memory( + let mut lexer = Lexer::with_code( "( a # b c d )", - Source::Unknown, ); let mut parser = Parser::new(&mut lexer); let result = parser.array_values().now_or_never().unwrap(); @@ -310,7 +309,7 @@ mod tests { #[test] fn parser_array_values_unclosed() { - let mut lexer = Lexer::from_memory("(a b", Source::Unknown); + let mut lexer = Lexer::with_code("(a b"); let mut parser = Parser::new(&mut lexer); let e = parser.array_values().now_or_never().unwrap().unwrap_err(); assert_matches!(e.cause, @@ -328,7 +327,7 @@ mod tests { #[test] fn parser_array_values_invalid_word() { - let mut lexer = Lexer::from_memory("(a;b)", Source::Unknown); + let mut lexer = Lexer::with_code("(a;b)"); let mut parser = Parser::new(&mut lexer); let e = parser.array_values().now_or_never().unwrap().unwrap_err(); assert_matches!(e.cause, @@ -346,7 +345,7 @@ mod tests { #[test] fn parser_simple_command_eof() { - let mut lexer = Lexer::from_memory("", Source::Unknown); + let mut lexer = Lexer::with_code(""); let mut parser = Parser::new(&mut lexer); let result = parser.simple_command().now_or_never().unwrap(); @@ -355,7 +354,7 @@ mod tests { #[test] fn parser_simple_command_keyword() { - let mut lexer = Lexer::from_memory("then", Source::Unknown); + let mut lexer = Lexer::with_code("then"); let mut parser = Parser::new(&mut lexer); let result = parser.simple_command().now_or_never().unwrap(); @@ -364,7 +363,7 @@ mod tests { #[test] fn parser_simple_command_one_assignment() { - let mut lexer = Lexer::from_memory("my=assignment", Source::Unknown); + let mut lexer = Lexer::with_code("my=assignment"); let mut parser = Parser::new(&mut lexer); let result = parser.simple_command().now_or_never().unwrap(); @@ -382,7 +381,7 @@ mod tests { #[test] fn parser_simple_command_many_assignments() { - let mut lexer = Lexer::from_memory("a= b=! c=X", Source::Unknown); + let mut lexer = Lexer::with_code("a= b=! c=X"); let mut parser = Parser::new(&mut lexer); let result = parser.simple_command().now_or_never().unwrap(); @@ -412,7 +411,7 @@ mod tests { #[test] fn parser_simple_command_one_word() { - let mut lexer = Lexer::from_memory("word", Source::Unknown); + let mut lexer = Lexer::with_code("word"); let mut parser = Parser::new(&mut lexer); let result = parser.simple_command().now_or_never().unwrap(); @@ -426,7 +425,7 @@ mod tests { #[test] fn parser_simple_command_many_words() { - let mut lexer = Lexer::from_memory(": if then", Source::Unknown); + let mut lexer = Lexer::with_code(": if then"); let mut parser = Parser::new(&mut lexer); let result = parser.simple_command().now_or_never().unwrap(); @@ -444,7 +443,7 @@ mod tests { #[test] fn parser_simple_command_one_redirection() { - let mut lexer = Lexer::from_memory("two >>three", Source::Unknown); + let mut lexer = Lexer::with_code("two >>three"); let mut parser = Parser::new(&mut lexer); let result = parser.simple_command().now_or_never().unwrap(); @@ -494,7 +493,7 @@ mod tests { #[test] fn parser_simple_command_assignment_word() { - let mut lexer = Lexer::from_memory("if=then else", Source::Unknown); + let mut lexer = Lexer::with_code("if=then else"); let mut parser = Parser::new(&mut lexer); let result = parser.simple_command().now_or_never().unwrap(); @@ -510,7 +509,7 @@ mod tests { #[test] fn parser_simple_command_word_redirection() { - let mut lexer = Lexer::from_memory("word Date: Mon, 30 Dec 2024 22:56:42 +0900 Subject: [PATCH 7/9] Remove trailing periods after noun phrases in comments --- yash-syntax/src/alias.rs | 20 ++++++------- yash-syntax/src/input.rs | 8 +++--- yash-syntax/src/lib.rs | 2 +- yash-syntax/src/parser.rs | 2 +- yash-syntax/src/parser/core.rs | 12 ++++---- yash-syntax/src/parser/error.rs | 12 ++++---- yash-syntax/src/parser/from_str.rs | 2 +- yash-syntax/src/parser/lex.rs | 2 +- yash-syntax/src/parser/lex/arith.rs | 2 +- yash-syntax/src/parser/lex/backquote.rs | 2 +- yash-syntax/src/parser/lex/braced_param.rs | 2 +- yash-syntax/src/parser/lex/command_subst.rs | 2 +- yash-syntax/src/parser/lex/core.rs | 28 +++++++++--------- yash-syntax/src/parser/lex/dollar.rs | 2 +- yash-syntax/src/parser/lex/keyword.rs | 4 +-- yash-syntax/src/parser/lex/misc.rs | 2 +- yash-syntax/src/parser/lex/modifier.rs | 2 +- yash-syntax/src/parser/lex/op.rs | 32 ++++++++++----------- yash-syntax/src/parser/lex/raw_param.rs | 2 +- yash-syntax/src/parser/lex/text.rs | 8 +++--- yash-syntax/src/parser/lex/token.rs | 2 +- yash-syntax/src/parser/lex/word.rs | 4 +-- yash-syntax/src/parser/simple_command.rs | 2 +- yash-syntax/src/source.rs | 14 ++++----- yash-syntax/src/syntax.rs | 4 +-- 25 files changed, 87 insertions(+), 87 deletions(-) diff --git a/yash-syntax/src/alias.rs b/yash-syntax/src/alias.rs index 2ac870ba..c69b82c1 100644 --- a/yash-syntax/src/alias.rs +++ b/yash-syntax/src/alias.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Defining aliases. +//! Defining aliases //! //! This module provides data structures for defining aliases in the shell //! execution environment. @@ -28,21 +28,21 @@ use std::hash::Hash; use std::hash::Hasher; use std::rc::Rc; -/// Name-value pair that defines an alias. +/// Name-value pair that defines an alias #[derive(Clone, Debug, Eq, PartialEq)] pub struct Alias { - /// Name of the alias that is matched against a command word by the syntax parser. + /// Name of the alias that is matched against a command word by the syntax parser pub name: String, - /// String that substitutes part of the source code when it is found to match the alias name. + /// String that substitutes part of the source code when it is found to match the alias name pub replacement: String, - /// Whether this alias is a global alias or not. + /// Whether this alias is a global alias or not pub global: bool, /// Location of the word in the simple command that invoked the alias built-in to define this - /// alias. + /// alias pub origin: Location, } -/// Wrapper of [`Alias`] for inserting into a hash set. +/// Wrapper of [`Alias`] for inserting into a hash set /// /// A `HashEntry` wraps an `Alias` in `Rc` so that the alias definition can be referred to even /// after the definition is removed. The `Hash` and `PartialEq` implementation for `HashEntry` @@ -94,10 +94,10 @@ impl Borrow for HashEntry { } } -/// Collection of aliases. +/// Collection of aliases pub type AliasSet = HashSet; -/// Interface used by the parser to look up aliases. +/// Interface used by the parser to look up aliases /// /// This trait is an abstract interface that represents an immutable collection /// of aliases. The parser uses this trait to look up aliases when it encounters @@ -137,7 +137,7 @@ impl Glossary for AliasSet { } } -/// Empty glossary that does not contain any aliases. +/// Empty glossary that does not contain any aliases #[derive(Clone, Debug, Default, Eq, Hash, PartialEq)] pub struct EmptyGlossary; diff --git a/yash-syntax/src/input.rs b/yash-syntax/src/input.rs index 8d26f8ee..8fe0f3e4 100644 --- a/yash-syntax/src/input.rs +++ b/yash-syntax/src/input.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Methods about passing [source](crate::source) code to the [parser](crate::parser). +//! Methods about passing [source](crate::source) code to the [parser](crate::parser) use std::future::Future; use std::ops::DerefMut; @@ -56,10 +56,10 @@ impl Context { } } -/// Error returned by the [Input] function. +/// Error returned by the [Input] function pub type Error = std::io::Error; -/// Result of the [Input] function. +/// Result of the [Input] function pub type Result = std::result::Result; /// Line-oriented source code reader @@ -115,7 +115,7 @@ impl InputObject for T { } } -/// Input function that reads from a string in memory. +/// Input function that reads from a string in memory pub struct Memory<'a> { lines: std::str::SplitInclusive<'a, char>, } diff --git a/yash-syntax/src/lib.rs b/yash-syntax/src/lib.rs index 2e042332..c5ec5370 100644 --- a/yash-syntax/src/lib.rs +++ b/yash-syntax/src/lib.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Shell language syntax and parser. +//! Shell language syntax and parser //! //! This crate defines data types for constructing abstract syntax trees (AST) //! of the shell language. See the [`syntax`] module for details. diff --git a/yash-syntax/src/parser.rs b/yash-syntax/src/parser.rs index 32e6b1cf..bf40c334 100644 --- a/yash-syntax/src/parser.rs +++ b/yash-syntax/src/parser.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Syntax parser for the shell language. +//! Syntax parser for the shell language //! //! The shell language parsing system has two important components: the lexical //! analyzer and the syntax parser. diff --git a/yash-syntax/src/parser/core.rs b/yash-syntax/src/parser/core.rs index 434c9501..e7de5138 100644 --- a/yash-syntax/src/parser/core.rs +++ b/yash-syntax/src/parser/core.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Fundamentals for implementing the parser. +//! Fundamentals for implementing the parser //! //! This module includes common types that are used as building blocks for constructing the syntax //! parser. @@ -32,11 +32,11 @@ use crate::syntax::MaybeLiteral; use crate::syntax::Word; use std::rc::Rc; -/// Entire result of parsing. +/// Entire result of parsing pub type Result = std::result::Result; /// Modifier that makes a result of parsing optional in order to trigger the parser to restart -/// parsing after alias substitution. +/// parsing after alias substitution /// /// `Rec` stands for "recursion", as it is used to make the parser work recursively. /// @@ -55,9 +55,9 @@ pub type Result = std::result::Result; /// variant. The caller then continues the remaining parse. #[derive(Copy, Clone, Debug, Eq, PartialEq)] pub enum Rec { - /// Result of alias substitution. + /// Result of alias substitution AliasSubstituted, - /// Successful parse result. + /// Successful parse result Parsed(T), } @@ -94,7 +94,7 @@ impl Rec { } } -/// Set of parameters for constructing a [parser](Parser). +/// Set of parameters for constructing a [parser](Parser) /// /// `Config` is a builder for constructing a parser. A [new](Self::new) /// configuration starts with default settings. You can customize them by diff --git a/yash-syntax/src/parser/error.rs b/yash-syntax/src/parser/error.rs index 5beb7b4e..c008f867 100644 --- a/yash-syntax/src/parser/error.rs +++ b/yash-syntax/src/parser/error.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Definition of errors that happen in the parser. +//! Definition of errors that happen in the parser use crate::source::pretty::Annotation; use crate::source::pretty::AnnotationType; @@ -25,7 +25,7 @@ use std::borrow::Cow; use std::rc::Rc; use thiserror::Error; -/// Types of syntax errors. +/// Types of syntax errors #[derive(Clone, Debug, Eq, Error, PartialEq)] #[error("{}", self.message())] #[non_exhaustive] @@ -409,13 +409,13 @@ impl SyntaxError { } } -/// Types of errors that may happen in parsing. +/// Types of errors that may happen in parsing #[derive(Clone, Debug, Error)] #[error("{}", self.message())] pub enum ErrorCause { - /// Error in an underlying input function. + /// Error in an underlying input function Io(#[from] Rc), - /// Syntax error. + /// Syntax error Syntax(#[from] SyntaxError), } @@ -467,7 +467,7 @@ impl From for ErrorCause { } } -/// Explanation of a failure in parsing. +/// Explanation of a failure in parsing #[derive(Clone, Debug, Error, PartialEq)] #[error("{cause}")] pub struct Error { diff --git a/yash-syntax/src/parser/from_str.rs b/yash-syntax/src/parser/from_str.rs index 0df32b89..46dd4c19 100644 --- a/yash-syntax/src/parser/from_str.rs +++ b/yash-syntax/src/parser/from_str.rs @@ -49,7 +49,7 @@ async fn reject_redundant_token(parser: &mut Parser<'_, '_>) -> Result<(), Error } } -/// Helper for implementing FromStr. +/// Helper for implementing FromStr trait Shift { type Output; fn shift(self) -> Self::Output; diff --git a/yash-syntax/src/parser/lex.rs b/yash-syntax/src/parser/lex.rs index 3bfffa46..cd1cf230 100644 --- a/yash-syntax/src/parser/lex.rs +++ b/yash-syntax/src/parser/lex.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Lexical analyzer. +//! Lexical analyzer //! //! See the [parent module](super)'s documentation to learn how to use the //! [lexer](Lexer). diff --git a/yash-syntax/src/parser/lex/arith.rs b/yash-syntax/src/parser/lex/arith.rs index 2ab85a35..160ea6e2 100644 --- a/yash-syntax/src/parser/lex/arith.rs +++ b/yash-syntax/src/parser/lex/arith.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Part of the lexer that parses arithmetic expansions. +//! Part of the lexer that parses arithmetic expansions use super::core::Lexer; use crate::parser::core::Result; diff --git a/yash-syntax/src/parser/lex/backquote.rs b/yash-syntax/src/parser/lex/backquote.rs index 14fdf289..0e7815f4 100644 --- a/yash-syntax/src/parser/lex/backquote.rs +++ b/yash-syntax/src/parser/lex/backquote.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Part of the lexer that parses backquotes. +//! Part of the lexer that parses backquotes use super::core::WordContext; use super::core::WordLexer; diff --git a/yash-syntax/src/parser/lex/braced_param.rs b/yash-syntax/src/parser/lex/braced_param.rs index 3f66e192..6be59f17 100644 --- a/yash-syntax/src/parser/lex/braced_param.rs +++ b/yash-syntax/src/parser/lex/braced_param.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Part of the lexer that parses braced parameter expansion. +//! Part of the lexer that parses braced parameter expansion use super::core::WordLexer; use super::raw_param::is_portable_name_char; diff --git a/yash-syntax/src/parser/lex/command_subst.rs b/yash-syntax/src/parser/lex/command_subst.rs index 12cbafec..3359c18f 100644 --- a/yash-syntax/src/parser/lex/command_subst.rs +++ b/yash-syntax/src/parser/lex/command_subst.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Part of the lexer that parses command substitutions. +//! Part of the lexer that parses command substitutions use super::core::Lexer; use crate::parser::core::Result; diff --git a/yash-syntax/src/parser/lex/core.rs b/yash-syntax/src/parser/lex/core.rs index 657ce89c..4070f160 100644 --- a/yash-syntax/src/parser/lex/core.rs +++ b/yash-syntax/src/parser/lex/core.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Fundamental building blocks for the lexical analyzer. +//! Fundamental building blocks for the lexical analyzer use super::keyword::Keyword; use super::op::Operator; @@ -46,7 +46,7 @@ pub fn is_blank(c: char) -> bool { c != '\n' && c.is_whitespace() } -/// Result of [`LexerCore::peek_char`]. +/// Result of [`LexerCore::peek_char`] #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum PeekChar<'a> { Char(&'a SourceChar), @@ -64,7 +64,7 @@ impl<'a> PeekChar<'a> { } } -/// Token identifier, or classification of tokens. +/// Token identifier, or classification of tokens /// /// This enum classifies a token as defined in POSIX XCU 2.10.1 Shell Grammar Lexical /// Conventions, but does not exactly reflect further distinction defined in @@ -85,7 +85,7 @@ pub enum TokenId { Operator(Operator), /// `IO_NUMBER` IoNumber, - /// Imaginary token identifier for the end of input. + /// Imaginary token identifier for the end of input EndOfInput, } @@ -108,18 +108,18 @@ impl TokenId { } } -/// Result of lexical analysis produced by the [`Lexer`]. +/// Result of lexical analysis produced by the [`Lexer`] #[derive(Debug)] pub struct Token { - /// Content of the token. + /// Content of the token /// /// The word value contains at least one [unit](crate::syntax::WordUnit), /// regardless of whether the token is an operator. The only exception is /// when `id` is `EndOfInput`, in which case the word is empty. pub word: Word, - /// Token identifier. + /// Token identifier pub id: TokenId, - /// Position of the first character of the word. + /// Position of the first character of the word pub index: usize, } @@ -129,7 +129,7 @@ impl fmt::Display for Token { } } -/// State of the input function in a lexer. +/// State of the input function in a lexer #[derive(Clone, Debug)] enum InputState { Alive, @@ -151,7 +151,7 @@ fn ex>(i: I) -> impl Iterator { input: Box, state: InputState, @@ -493,7 +493,7 @@ impl Default for Config { } } -/// Lexical analyzer. +/// Lexical analyzer /// /// A lexer reads lines using an input function and parses the characters into tokens. It has an /// internal buffer containing the characters that have been read and the position (or the @@ -876,7 +876,7 @@ impl<'a> Lexer<'a> { } } -/// Reference to [`Lexer`] with line continuation disabled. +/// Reference to [`Lexer`] with line continuation disabled /// /// This struct implements the RAII pattern for temporarily disabling line /// continuation. When you disable the line continuation of a lexer, you get an @@ -908,7 +908,7 @@ impl Drop for PlainLexer<'_, '_> { } } -/// Context in which a [word](crate::syntax::Word) is parsed. +/// Context in which a [word](crate::syntax::Word) is parsed /// /// The parse of the word of a [switch](crate::syntax::Switch) depends on /// whether the parameter expansion containing the switch is part of a text or a @@ -925,7 +925,7 @@ pub enum WordContext { } /// Lexer with additional information for parsing [texts](crate::syntax::Text) -/// and [words](crate::syntax::Word). +/// and [words](crate::syntax::Word) #[derive(Debug)] pub struct WordLexer<'a, 'b> { pub lexer: &'a mut Lexer<'b>, diff --git a/yash-syntax/src/parser/lex/dollar.rs b/yash-syntax/src/parser/lex/dollar.rs index 24daf4b0..99e61860 100644 --- a/yash-syntax/src/parser/lex/dollar.rs +++ b/yash-syntax/src/parser/lex/dollar.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Part of the lexer that parses dollar units. +//! Part of the lexer that parses dollar units //! //! Note that the detail lexer for each type of dollar units in another //! dedicated module. diff --git a/yash-syntax/src/parser/lex/keyword.rs b/yash-syntax/src/parser/lex/keyword.rs index 7fae3911..cb33e13b 100644 --- a/yash-syntax/src/parser/lex/keyword.rs +++ b/yash-syntax/src/parser/lex/keyword.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Types and functions for parsing reserved words. +//! Types and functions for parsing reserved words use std::fmt; use std::str::FromStr; @@ -33,7 +33,7 @@ impl fmt::Display for ParseKeywordError { } } -/// Token identifier for reserved words. +/// Token identifier for reserved words #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum Keyword { Bang, diff --git a/yash-syntax/src/parser/lex/misc.rs b/yash-syntax/src/parser/lex/misc.rs index 7369c4c6..98c88912 100644 --- a/yash-syntax/src/parser/lex/misc.rs +++ b/yash-syntax/src/parser/lex/misc.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Extension of the core for implementing the rest of the lexer. +//! Extension of the core for implementing the rest of the lexer use super::core::is_blank; use super::core::Lexer; diff --git a/yash-syntax/src/parser/lex/modifier.rs b/yash-syntax/src/parser/lex/modifier.rs index fe96b56c..0dbd14ba 100644 --- a/yash-syntax/src/parser/lex/modifier.rs +++ b/yash-syntax/src/parser/lex/modifier.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Part of the lexer that parses suffix modifiers. +//! Part of the lexer that parses suffix modifiers use super::core::Lexer; use super::core::WordContext; diff --git a/yash-syntax/src/parser/lex/op.rs b/yash-syntax/src/parser/lex/op.rs index 4c2c88ae..b52f92ad 100644 --- a/yash-syntax/src/parser/lex/op.rs +++ b/yash-syntax/src/parser/lex/op.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Part of the lexer that parses operators. +//! Part of the lexer that parses operators use super::core::Lexer; use super::core::Token; @@ -28,7 +28,7 @@ use std::future::Future; use std::pin::Pin; use thiserror::Error; -/// Operator token identifier. +/// Operator token identifier #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum Operator { /// Newline @@ -157,15 +157,15 @@ impl fmt::Display for Operator { #[derive(Copy, Clone, Debug)] pub struct Trie(&'static [Edge]); -/// Edge of a [`Trie`]. +/// Edge of a [`Trie`] #[derive(Copy, Clone, Debug)] pub struct Edge { - /// Character value of this edge. + /// Character value of this edge pub key: char, /// Final operator token that is delimited after taking this edge if there are no longer - /// matches. + /// matches pub value: Option, - /// Sub-trie containing values for keys that have the common prefix. + /// Sub-trie containing values for keys that have the common prefix pub next: Trie, } @@ -184,7 +184,7 @@ impl Trie { } } -/// Trie containing all the operators. +/// Trie containing all the operators pub const OPERATORS: Trie = Trie(&[ Edge { key: '\n', @@ -228,14 +228,14 @@ pub const OPERATORS: Trie = Trie(&[ }, ]); -/// Trie of the operators that start with `&`. +/// Trie of the operators that start with `&` const AND: Trie = Trie(&[Edge { key: '&', value: Some(Operator::AndAnd), next: NONE, }]); -/// Trie of the operators that start with `;`. +/// Trie of the operators that start with `;` const SEMICOLON: Trie = Trie(&[ Edge { key: '&', @@ -254,14 +254,14 @@ const SEMICOLON: Trie = Trie(&[ }, ]); -/// Trie of the operators that start with `;;`. +/// Trie of the operators that start with `;;` const SEMICOLON_SEMICOLON: Trie = Trie(&[Edge { key: '&', value: Some(Operator::SemicolonSemicolonAnd), next: NONE, }]); -/// Trie of the operators that start with `<`. +/// Trie of the operators that start with `<` const LESS: Trie = Trie(&[ Edge { key: '&', @@ -285,7 +285,7 @@ const LESS: Trie = Trie(&[ }, ]); -/// Trie of the operators that start with `<<`. +/// Trie of the operators that start with `<<` const LESS_LESS: Trie = Trie(&[ Edge { key: '-', @@ -299,7 +299,7 @@ const LESS_LESS: Trie = Trie(&[ }, ]); -/// Trie of the operators that start with `>`. +/// Trie of the operators that start with `>` const GREATER: Trie = Trie(&[ Edge { key: '&', @@ -323,21 +323,21 @@ const GREATER: Trie = Trie(&[ }, ]); -/// Trie of the operators that start with `>>`. +/// Trie of the operators that start with `>>` const GREATER_GREATER: Trie = Trie(&[Edge { key: '|', value: Some(Operator::GreaterGreaterBar), next: NONE, }]); -/// Trie of the operators that start with `|`. +/// Trie of the operators that start with `|` const BAR: Trie = Trie(&[Edge { key: '|', value: Some(Operator::BarBar), next: NONE, }]); -/// Trie containing nothing. +/// Trie containing nothing const NONE: Trie = Trie(&[]); /// Tests whether the given character is the first character of an operator. diff --git a/yash-syntax/src/parser/lex/raw_param.rs b/yash-syntax/src/parser/lex/raw_param.rs index 3e8e0fb0..21fd3488 100644 --- a/yash-syntax/src/parser/lex/raw_param.rs +++ b/yash-syntax/src/parser/lex/raw_param.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Part of the lexer that parses raw parameter expansion. +//! Part of the lexer that parses raw parameter expansion use super::core::Lexer; use crate::parser::core::Result; diff --git a/yash-syntax/src/parser/lex/text.rs b/yash-syntax/src/parser/lex/text.rs index 1af39d17..d01a05f2 100644 --- a/yash-syntax/src/parser/lex/text.rs +++ b/yash-syntax/src/parser/lex/text.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Part of the lexer that parses texts. +//! Part of the lexer that parses texts use super::core::Lexer; use super::core::WordContext; @@ -58,7 +58,7 @@ impl WordLexer<'_, '_> { .await } - /// Dynamic version of [`Self::text_unit`]. + /// Dynamic version of [`Self::text_unit`] async fn text_unit_dyn( &mut self, is_delimiter: &mut dyn FnMut(char) -> bool, @@ -124,7 +124,7 @@ impl Lexer<'_> { self.text_dyn(&mut is_delimiter, &mut is_escapable).await } - /// Dynamic version of [`Self::text`]. + /// Dynamic version of [`Self::text`] async fn text_dyn( &mut self, is_delimiter: &mut dyn FnMut(char) -> bool, @@ -168,7 +168,7 @@ impl Lexer<'_> { .await } - /// Dynamic version of [`Self::text_with_parentheses`]. + /// Dynamic version of [`Self::text_with_parentheses`] async fn text_with_parentheses_dyn( &mut self, is_delimiter: &mut dyn FnMut(char) -> bool, diff --git a/yash-syntax/src/parser/lex/token.rs b/yash-syntax/src/parser/lex/token.rs index 582393a2..a37873f7 100644 --- a/yash-syntax/src/parser/lex/token.rs +++ b/yash-syntax/src/parser/lex/token.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Part of the lexer that parses backquotes. +//! Part of the lexer that parses backquotes use super::core::is_blank; use super::core::Lexer; diff --git a/yash-syntax/src/parser/lex/word.rs b/yash-syntax/src/parser/lex/word.rs index 116e5d15..e5b8008e 100644 --- a/yash-syntax/src/parser/lex/word.rs +++ b/yash-syntax/src/parser/lex/word.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Part of the lexer that parses words. +//! Part of the lexer that parses words use super::core::Lexer; use super::core::WordContext; @@ -102,7 +102,7 @@ impl WordLexer<'_, '_> { self.word_unit_dyn(&is_delimiter).await } - /// Dynamic version of [`Self::word_unit`]. + /// Dynamic version of [`Self::word_unit`] async fn word_unit_dyn( &mut self, is_delimiter: &dyn Fn(char) -> bool, diff --git a/yash-syntax/src/parser/simple_command.rs b/yash-syntax/src/parser/simple_command.rs index dce91b1d..9963ec39 100644 --- a/yash-syntax/src/parser/simple_command.rs +++ b/yash-syntax/src/parser/simple_command.rs @@ -55,7 +55,7 @@ fn determine_expansion_mode(word: Word) -> (Word, ExpansionMode) { (word, ExpansionMode::Multiple) } -/// Simple command builder. +/// Simple command builder #[derive(Default)] struct Builder { assigns: Vec, diff --git a/yash-syntax/src/source.rs b/yash-syntax/src/source.rs index ccf70eea..a7bd46a8 100644 --- a/yash-syntax/src/source.rs +++ b/yash-syntax/src/source.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Source code that is passed to the parser. +//! Source code that is passed to the parser //! //! This module contains items representing information about the source code //! from which ASTs originate. [`Source`] identifies the origin of source code @@ -263,13 +263,13 @@ pub fn source_chars<'a>( }) } -/// Position of source code. +/// Position of source code #[derive(Clone, Debug, Eq, PartialEq)] pub struct Location { - /// Code that contains the character. + /// Code that contains the character pub code: Rc, - /// Character position in the code, counted from 0. + /// Character position in the code, counted from 0 /// /// Characters are counted in the number of Unicode scalar values, not /// bytes. That means the index should be between 0 and @@ -300,12 +300,12 @@ impl Location { } } -/// Character with source description. +/// Character with source description #[derive(Clone, Debug, Eq, PartialEq)] pub struct SourceChar { - /// Character value. + /// Character value pub value: char, - /// Location of this character in source code. + /// Location of this character in source code pub location: Location, } diff --git a/yash-syntax/src/syntax.rs b/yash-syntax/src/syntax.rs index 9114695e..469db765 100644 --- a/yash-syntax/src/syntax.rs +++ b/yash-syntax/src/syntax.rs @@ -243,7 +243,7 @@ pub struct Trim { pub side: TrimSide, /// How long the pattern should match? pub length: TrimLength, - /// Pattern to be matched with the expanded value. + /// Pattern to be matched with the expanded value pub pattern: Word, } @@ -341,7 +341,7 @@ pub use TextUnit::*; #[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct Text(pub Vec); -/// Element of an [`EscapedString`]. +/// Element of an [`EscapedString`] #[derive(Clone, Debug, Eq, PartialEq)] pub enum EscapeUnit { /// Literal single character From 44e21f9d259bad1f26af349d4a06130e751c9464 Mon Sep 17 00:00:00 2001 From: WATANABE Yuki Date: Mon, 30 Dec 2024 23:11:07 +0900 Subject: [PATCH 8/9] Remove SourceInput and use Lexer directly in prepare_input The `SourceInput` struct was not very useful and was only used to pass the input object and the source description to the caller. This commit removes the `SourceInput` struct and instead returns the `Lexer` object directly from the `prepare_input` function. --- yash-cli/src/lib.rs | 9 ++----- yash-cli/src/startup/input.rs | 47 +++++++++++++++++++---------------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/yash-cli/src/lib.rs b/yash-cli/src/lib.rs index 39eec5b7..304270ef 100644 --- a/yash-cli/src/lib.rs +++ b/yash-cli/src/lib.rs @@ -29,7 +29,6 @@ pub mod startup; use self::startup::args::Parse; use self::startup::init_file::run_rcfile; use self::startup::input::prepare_input; -use self::startup::input::SourceInput; use std::cell::RefCell; use std::ops::ControlFlow::{Break, Continue}; use yash_env::option::{Interactive, On}; @@ -42,7 +41,6 @@ use yash_executor::Executor; use yash_semantics::trap::run_exit_trap; use yash_semantics::{interactive_read_eval_loop, read_eval_loop}; use yash_semantics::{Divert, ExitStatus}; -use yash_syntax::parser::lex::Lexer; async fn print_version(env: &mut Env) -> ExitStatus { let version = env!("CARGO_PKG_VERSION"); @@ -78,8 +76,8 @@ async fn parse_and_print(mut env: Env) -> ExitStatus { // Prepare the input for the main read-eval loop let ref_env = &RefCell::new(&mut env); - let SourceInput { input, source } = match prepare_input(ref_env, &work.source) { - Ok(input) => input, + let lexer = match prepare_input(ref_env, &work.source) { + Ok(lexer) => lexer, Err(e) => { let arg0 = std::env::args().next().unwrap_or_else(|| "yash".to_owned()); let message = format!("{}: {}\n", arg0, e); @@ -94,9 +92,6 @@ async fn parse_and_print(mut env: Env) -> ExitStatus { }; } }; - let mut config = Lexer::config(); - config.source = Some(source.into()); - let mut lexer = config.input(input); // Run the read-eval loop let result = if is_interactive { diff --git a/yash-cli/src/startup/input.rs b/yash-cli/src/startup/input.rs index 81e5ba16..0313e29d 100644 --- a/yash-cli/src/startup/input.rs +++ b/yash-cli/src/startup/input.rs @@ -14,14 +14,15 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Preparing input object +//! Preparing input for the parser //! -//! This module implements the [`prepare_input`] function that prepares the input -//! object for the shell. The input object is constructed from the given source -//! and decorated with the [`Echo`] and [`Prompter`] decorators as necessary. +//! This module implements the [`prepare_input`] function that prepares the +//! input for the shell syntax parser. The input is constructed from the given +//! source and decorated with the [`Echo`] and [`Prompter`] decorators as +//! necessary. //! -//! The [`SourceInput`] and [`PrepareInputError`] types define the return value -//! of the function. +//! [`PrepareInputError`] defines the error that may occur when preparing the +//! input. use super::args::Source; use std::cell::RefCell; @@ -44,17 +45,10 @@ use yash_env::System; use yash_prompt::Prompter; use yash_syntax::input::InputObject; use yash_syntax::input::Memory; +use yash_syntax::parser::lex::Lexer; use yash_syntax::source::Source as SyntaxSource; -/// Result of [`prepare_input`]. -pub struct SourceInput<'a> { - /// Input to be passed to the lexer - pub input: Box, - /// Description of the source - pub source: SyntaxSource, -} - -/// Error returned by [`prepare_input`]. +/// Error returned by [`prepare_input`] #[derive(Clone, Debug, Eq, Error, PartialEq)] #[error("cannot open script file '{path}': {errno}")] pub struct PrepareInputError<'a> { @@ -64,10 +58,10 @@ pub struct PrepareInputError<'a> { pub path: &'a str, } -/// Prepares the input for the shell. +/// Prepares the input for the shell syntax parser. /// -/// This function constructs an input object from the given source with the -/// following decorators: +/// This function constructs a lexer from the given source with the +/// following decorators applied to the input object: /// /// - If the source is read with a file descriptor, the [`Echo`] decorator is /// applied to the input to implement the [`Verbose`] shell option. @@ -89,7 +83,16 @@ pub struct PrepareInputError<'a> { pub fn prepare_input<'s: 'i + 'e, 'i, 'e>( env: &'i RefCell<&mut Env>, source: &'s Source, -) -> Result, PrepareInputError<'e>> { +) -> Result, PrepareInputError<'e>> { + fn lexer_with_input_and_source<'a>( + input: Box, + source: SyntaxSource, + ) -> Lexer<'a> { + let mut config = Lexer::config(); + config.source = Some(source.into()); + config.input(input) + } + match source { Source::Stdin => { let mut system = env.borrow().system.clone(); @@ -103,7 +106,7 @@ pub fn prepare_input<'s: 'i + 'e, 'i, 'e>( let input = prepare_fd_input(Fd::STDIN, env); let source = SyntaxSource::Stdin; - Ok(SourceInput { input, source }) + Ok(lexer_with_input_and_source(input, source)) } Source::File { path } => { @@ -126,7 +129,7 @@ pub fn prepare_input<'s: 'i + 'e, 'i, 'e>( let input = prepare_fd_input(fd, env); let path = path.to_owned(); let source = SyntaxSource::CommandFile { path }; - Ok(SourceInput { input, source }) + Ok(lexer_with_input_and_source(input, source)) } Source::String(command) => { @@ -139,7 +142,7 @@ pub fn prepare_input<'s: 'i + 'e, 'i, 'e>( Box::new(basic_input) }; let source = SyntaxSource::CommandString; - Ok(SourceInput { input, source }) + Ok(lexer_with_input_and_source(input, source)) } } } From d90248cf1cd72f692dab2343c3dfe49eede774d9 Mon Sep 17 00:00:00 2001 From: WATANABE Yuki Date: Mon, 30 Dec 2024 23:22:42 +0900 Subject: [PATCH 9/9] Add a note about LexerCore::input --- yash-syntax/src/parser/lex/core.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/yash-syntax/src/parser/lex/core.rs b/yash-syntax/src/parser/lex/core.rs index 4070f160..5d7f4e55 100644 --- a/yash-syntax/src/parser/lex/core.rs +++ b/yash-syntax/src/parser/lex/core.rs @@ -153,6 +153,10 @@ fn ex>(i: I) -> impl Iterator { + // The `input` field could be a `&'a mut dyn InputObject + 'a`, but it is + // `Box` to allow the lexer to take ownership of the + // input object. This is necessary for `Lexer::with_code` and similarly + // constructed lexers. input: Box, state: InputState, raw_code: Rc,