From 83f91f61b13eeff1dc055290e53392f15b9e8660 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Thu, 29 Jun 2023 11:12:48 +0200 Subject: [PATCH 01/24] Infect mbe crate with generic span type parameter --- crates/mbe/src/benchmark.rs | 45 ++++--- crates/mbe/src/expander.rs | 41 ++++--- crates/mbe/src/expander/matcher.rs | 150 +++++++++++++---------- crates/mbe/src/expander/transcriber.rs | 99 ++++++++-------- crates/mbe/src/lib.rs | 31 ++--- crates/mbe/src/parser.rs | 68 ++++++----- crates/mbe/src/syntax_bridge.rs | 54 ++++----- crates/mbe/src/to_parser_input.rs | 4 +- crates/mbe/src/tt_iter.rs | 35 +++--- crates/tt/src/lib.rs | 158 +++++++++++++------------ 10 files changed, 360 insertions(+), 325 deletions(-) diff --git a/crates/mbe/src/benchmark.rs b/crates/mbe/src/benchmark.rs index 9d43e130457d..19cb20354b74 100644 --- a/crates/mbe/src/benchmark.rs +++ b/crates/mbe/src/benchmark.rs @@ -6,10 +6,11 @@ use syntax::{ AstNode, SmolStr, }; use test_utils::{bench, bench_fixture, skip_slow_tests}; +use tt::{Span, TokenId}; use crate::{ parser::{MetaVarKind, Op, RepeatKind, Separator}, - syntax_node_to_token_tree, tt, DeclarativeMacro, + syntax_node_to_token_tree, DeclarativeMacro, }; #[test] @@ -54,7 +55,7 @@ fn macro_rules_fixtures() -> FxHashMap { .collect() } -fn macro_rules_fixtures_tt() -> FxHashMap { +fn macro_rules_fixtures_tt() -> FxHashMap> { let fixture = bench_fixture::numerous_macro_rules(); let source_file = ast::SourceFile::parse(&fixture).ok().unwrap(); @@ -71,7 +72,9 @@ fn macro_rules_fixtures_tt() -> FxHashMap { } /// Generate random invocation fixtures from rules -fn invocation_fixtures(rules: &FxHashMap) -> Vec<(String, tt::Subtree)> { +fn invocation_fixtures( + rules: &FxHashMap, +) -> Vec<(String, tt::Subtree)> { let mut seed = 123456789; let mut res = Vec::new(); @@ -93,8 +96,8 @@ fn invocation_fixtures(rules: &FxHashMap) -> Vec<(Stri loop { let mut subtree = tt::Subtree { delimiter: tt::Delimiter { - open: tt::TokenId::UNSPECIFIED, - close: tt::TokenId::UNSPECIFIED, + open: tt::TokenId::DUMMY, + close: tt::TokenId::DUMMY, kind: tt::DelimiterKind::Invisible, }, token_trees: vec![], @@ -116,7 +119,7 @@ fn invocation_fixtures(rules: &FxHashMap) -> Vec<(Stri } return res; - fn collect_from_op(op: &Op, parent: &mut tt::Subtree, seed: &mut usize) { + fn collect_from_op(op: &Op, parent: &mut tt::Subtree, seed: &mut usize) { return match op { Op::Var { kind, .. } => match kind.as_ref() { Some(MetaVarKind::Ident) => parent.token_trees.push(make_ident("foo")), @@ -202,36 +205,30 @@ fn invocation_fixtures(rules: &FxHashMap) -> Vec<(Stri *seed = usize::wrapping_add(usize::wrapping_mul(*seed, a), c); *seed } - fn make_ident(ident: &str) -> tt::TokenTree { - tt::Leaf::Ident(tt::Ident { - span: tt::TokenId::unspecified(), - text: SmolStr::new(ident), - }) - .into() + fn make_ident(ident: &str) -> tt::TokenTree { + tt::Leaf::Ident(tt::Ident { span: tt::TokenId::DUMMY, text: SmolStr::new(ident) }) + .into() } - fn make_punct(char: char) -> tt::TokenTree { + fn make_punct(char: char) -> tt::TokenTree { tt::Leaf::Punct(tt::Punct { - span: tt::TokenId::unspecified(), + span: tt::TokenId::DUMMY, char, spacing: tt::Spacing::Alone, }) .into() } - fn make_literal(lit: &str) -> tt::TokenTree { - tt::Leaf::Literal(tt::Literal { - span: tt::TokenId::unspecified(), - text: SmolStr::new(lit), - }) - .into() + fn make_literal(lit: &str) -> tt::TokenTree { + tt::Leaf::Literal(tt::Literal { span: tt::TokenId::DUMMY, text: SmolStr::new(lit) }) + .into() } fn make_subtree( kind: tt::DelimiterKind, - token_trees: Option>, - ) -> tt::TokenTree { + token_trees: Option>>, + ) -> tt::TokenTree { tt::Subtree { delimiter: tt::Delimiter { - open: tt::TokenId::unspecified(), - close: tt::TokenId::unspecified(), + open: tt::TokenId::DUMMY, + close: tt::TokenId::DUMMY, kind, }, token_trees: token_trees.unwrap_or_default(), diff --git a/crates/mbe/src/expander.rs b/crates/mbe/src/expander.rs index 908048c99042..fac2b3375810 100644 --- a/crates/mbe/src/expander.rs +++ b/crates/mbe/src/expander.rs @@ -7,15 +7,16 @@ mod transcriber; use rustc_hash::FxHashMap; use syntax::SmolStr; +use tt::Span; -use crate::{parser::MetaVarKind, tt, ExpandError, ExpandResult}; +use crate::{parser::MetaVarKind, ExpandError, ExpandResult}; -pub(crate) fn expand_rules( - rules: &[crate::Rule], - input: &tt::Subtree, +pub(crate) fn expand_rules( + rules: &[crate::Rule], + input: &tt::Subtree, is_2021: bool, -) -> ExpandResult { - let mut match_: Option<(matcher::Match, &crate::Rule)> = None; +) -> ExpandResult> { + let mut match_: Option<(matcher::Match, &crate::Rule)> = None; for rule in rules { let new_match = matcher::match_(&rule.lhs, input, is_2021); @@ -47,7 +48,7 @@ pub(crate) fn expand_rules( ExpandResult { value, err: match_.err.or(transcribe_err) } } else { ExpandResult::new( - tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] }, + tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: vec![] }, ExpandError::NoMatchingRule, ) } @@ -98,23 +99,29 @@ pub(crate) fn expand_rules( /// In other words, `Bindings` is a *multi* mapping from `SmolStr` to /// `tt::TokenTree`, where the index to select a particular `TokenTree` among /// many is not a plain `usize`, but a `&[usize]`. -#[derive(Debug, Default, Clone, PartialEq, Eq)] -struct Bindings { - inner: FxHashMap, +#[derive(Debug, Clone, PartialEq, Eq)] +struct Bindings { + inner: FxHashMap>, +} + +impl Default for Bindings { + fn default() -> Self { + Self { inner: Default::default() } + } } #[derive(Debug, Clone, PartialEq, Eq)] -enum Binding { - Fragment(Fragment), - Nested(Vec), +enum Binding { + Fragment(Fragment), + Nested(Vec>), Empty, Missing(MetaVarKind), } #[derive(Debug, Clone, PartialEq, Eq)] -enum Fragment { +enum Fragment { /// token fragments are just copy-pasted into the output - Tokens(tt::TokenTree), + Tokens(tt::TokenTree), /// Expr ast fragments are surrounded with `()` on insertion to preserve /// precedence. Note that this impl is different from the one currently in /// `rustc` -- `rustc` doesn't translate fragments into token trees at all. @@ -122,7 +129,7 @@ enum Fragment { /// At one point in time, we tried to use "fake" delimiters here à la /// proc-macro delimiter=none. As we later discovered, "none" delimiters are /// tricky to handle in the parser, and rustc doesn't handle those either. - Expr(tt::TokenTree), + Expr(tt::TokenTree), /// There are roughly two types of paths: paths in expression context, where a /// separator `::` between an identifier and its following generic argument list /// is mandatory, and paths in type context, where `::` can be omitted. @@ -132,5 +139,5 @@ enum Fragment { /// and is trasncribed as an expression-context path, verbatim transcription /// would cause a syntax error. We need to fix it up just before transcribing; /// see `transcriber::fix_up_and_push_path_tt()`. - Path(tt::TokenTree), + Path(tt::TokenTree), } diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs index 1471af98b75b..796c9f2eb321 100644 --- a/crates/mbe/src/expander/matcher.rs +++ b/crates/mbe/src/expander/matcher.rs @@ -63,21 +63,20 @@ use std::rc::Rc; use smallvec::{smallvec, SmallVec}; use syntax::SmolStr; +use tt::Span; use crate::{ expander::{Binding, Bindings, ExpandResult, Fragment}, parser::{MetaVarKind, Op, RepeatKind, Separator}, - tt, tt_iter::TtIter, ExpandError, MetaTemplate, ValueResult, }; -impl Bindings { +impl Bindings { fn push_optional(&mut self, name: &SmolStr) { // FIXME: Do we have a better way to represent an empty token ? // Insert an empty subtree for empty token - let tt = - tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] }.into(); + let tt = tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: vec![] }.into(); self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt))); } @@ -85,14 +84,14 @@ impl Bindings { self.inner.insert(name.clone(), Binding::Empty); } - fn bindings(&self) -> impl Iterator { + fn bindings(&self) -> impl Iterator> { self.inner.values() } } -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub(super) struct Match { - pub(super) bindings: Bindings, +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct Match { + pub(super) bindings: Bindings, /// We currently just keep the first error and count the rest to compare matches. pub(super) err: Option, pub(super) err_count: usize, @@ -102,7 +101,19 @@ pub(super) struct Match { pub(super) bound_count: usize, } -impl Match { +impl Default for Match { + fn default() -> Self { + Self { + bindings: Default::default(), + err: Default::default(), + err_count: Default::default(), + unmatched_tts: Default::default(), + bound_count: Default::default(), + } + } +} + +impl Match { fn add_err(&mut self, err: ExpandError) { let prev_err = self.err.take(); self.err = prev_err.or(Some(err)); @@ -111,12 +122,16 @@ impl Match { } /// Matching errors are added to the `Match`. -pub(super) fn match_(pattern: &MetaTemplate, input: &tt::Subtree, is_2021: bool) -> Match { +pub(super) fn match_( + pattern: &MetaTemplate, + input: &tt::Subtree, + is_2021: bool, +) -> Match { let mut res = match_loop(pattern, input, is_2021); res.bound_count = count(res.bindings.bindings()); return res; - fn count<'a>(bindings: impl Iterator) -> usize { + fn count<'a, S: 'a>(bindings: impl Iterator>) -> usize { bindings .map(|it| match it { Binding::Fragment(_) => 1, @@ -129,10 +144,10 @@ pub(super) fn match_(pattern: &MetaTemplate, input: &tt::Subtree, is_2021: bool) } #[derive(Debug, Clone)] -enum BindingKind { +enum BindingKind { Empty(SmolStr), Optional(SmolStr), - Fragment(SmolStr, Fragment), + Fragment(SmolStr, Fragment), Missing(SmolStr, MetaVarKind), Nested(usize, usize), } @@ -146,13 +161,18 @@ enum LinkNode { Parent { idx: usize, len: usize }, } -#[derive(Default)] -struct BindingsBuilder { - nodes: Vec>>>, +struct BindingsBuilder { + nodes: Vec>>>>, nested: Vec>>, } -impl BindingsBuilder { +impl Default for BindingsBuilder { + fn default() -> Self { + Self { nodes: Default::default(), nested: Default::default() } + } +} + +impl BindingsBuilder { fn alloc(&mut self) -> BindingsIdx { let idx = self.nodes.len(); self.nodes.push(Vec::new()); @@ -189,7 +209,7 @@ impl BindingsBuilder { self.nodes[idx.0].push(LinkNode::Node(Rc::new(BindingKind::Optional(var.clone())))); } - fn push_fragment(&mut self, idx: &mut BindingsIdx, var: &SmolStr, fragment: Fragment) { + fn push_fragment(&mut self, idx: &mut BindingsIdx, var: &SmolStr, fragment: Fragment) { self.nodes[idx.0] .push(LinkNode::Node(Rc::new(BindingKind::Fragment(var.clone(), fragment)))); } @@ -210,11 +230,11 @@ impl BindingsBuilder { idx.0 = new_idx; } - fn build(self, idx: &BindingsIdx) -> Bindings { + fn build(self, idx: &BindingsIdx) -> Bindings { self.build_inner(&self.nodes[idx.0]) } - fn build_inner(&self, link_nodes: &[LinkNode>]) -> Bindings { + fn build_inner(&self, link_nodes: &[LinkNode>>]) -> Bindings { let mut bindings = Bindings::default(); let mut nodes = Vec::new(); self.collect_nodes(link_nodes, &mut nodes); @@ -264,7 +284,7 @@ impl BindingsBuilder { &'a self, id: usize, len: usize, - nested_refs: &mut Vec<&'a [LinkNode>]>, + nested_refs: &mut Vec<&'a [LinkNode>>]>, ) { self.nested[id].iter().take(len).for_each(|it| match it { LinkNode::Node(id) => nested_refs.push(&self.nodes[*id]), @@ -272,7 +292,7 @@ impl BindingsBuilder { }); } - fn collect_nested(&self, idx: usize, nested_idx: usize, nested: &mut Vec) { + fn collect_nested(&self, idx: usize, nested_idx: usize, nested: &mut Vec>) { let last = &self.nodes[idx]; let mut nested_refs: Vec<&[_]> = Vec::new(); self.nested[nested_idx].iter().for_each(|it| match *it { @@ -283,7 +303,7 @@ impl BindingsBuilder { nested.extend(nested_refs.into_iter().map(|iter| self.build_inner(iter))); } - fn collect_nodes_ref<'a>(&'a self, id: usize, len: usize, nodes: &mut Vec<&'a BindingKind>) { + fn collect_nodes_ref<'a>(&'a self, id: usize, len: usize, nodes: &mut Vec<&'a BindingKind>) { self.nodes[id].iter().take(len).for_each(|it| match it { LinkNode::Node(it) => nodes.push(it), LinkNode::Parent { idx, len } => self.collect_nodes_ref(*idx, *len, nodes), @@ -292,8 +312,8 @@ impl BindingsBuilder { fn collect_nodes<'a>( &'a self, - link_nodes: &'a [LinkNode>], - nodes: &mut Vec<&'a BindingKind>, + link_nodes: &'a [LinkNode>>], + nodes: &mut Vec<&'a BindingKind>, ) { link_nodes.iter().for_each(|it| match it { LinkNode::Node(it) => nodes.push(it), @@ -303,22 +323,22 @@ impl BindingsBuilder { } #[derive(Debug, Clone)] -struct MatchState<'t> { +struct MatchState<'t, S> { /// The position of the "dot" in this matcher - dot: OpDelimitedIter<'t>, + dot: OpDelimitedIter<'t, S>, /// Token subtree stack /// When matching against matchers with nested delimited submatchers (e.g., `pat ( pat ( .. ) /// pat ) pat`), we need to keep track of the matchers we are descending into. This stack does /// that where the bottom of the stack is the outermost matcher. - stack: SmallVec<[OpDelimitedIter<'t>; 4]>, + stack: SmallVec<[OpDelimitedIter<'t, S>; 4]>, /// The "parent" matcher position if we are in a repetition. That is, the matcher position just /// before we enter the repetition. - up: Option>>, + up: Option>>, /// The separator if we are in a repetition. - sep: Option, + sep: Option>, /// The KleeneOp of this sequence if we are in a repetition. sep_kind: Option, @@ -330,7 +350,7 @@ struct MatchState<'t> { bindings: BindingsIdx, /// Cached result of meta variable parsing - meta_result: Option<(TtIter<'t>, ExpandResult>)>, + meta_result: Option<(TtIter<'t, S>, ExpandResult>>)>, /// Is error occurred in this state, will `poised` to "parent" is_error: bool, @@ -355,16 +375,16 @@ struct MatchState<'t> { /// - `bb_items`: the set of items that are waiting for the black-box parser. /// - `error_items`: the set of items in errors, used for error-resilient parsing #[inline] -fn match_loop_inner<'t>( - src: TtIter<'t>, - stack: &[TtIter<'t>], - res: &mut Match, - bindings_builder: &mut BindingsBuilder, - cur_items: &mut SmallVec<[MatchState<'t>; 1]>, - bb_items: &mut SmallVec<[MatchState<'t>; 1]>, - next_items: &mut Vec>, - eof_items: &mut SmallVec<[MatchState<'t>; 1]>, - error_items: &mut SmallVec<[MatchState<'t>; 1]>, +fn match_loop_inner<'t, S: Span>( + src: TtIter<'t, S>, + stack: &[TtIter<'t, S>], + res: &mut Match, + bindings_builder: &mut BindingsBuilder, + cur_items: &mut SmallVec<[MatchState<'t, S>; 1]>, + bb_items: &mut SmallVec<[MatchState<'t, S>; 1]>, + next_items: &mut Vec>, + eof_items: &mut SmallVec<[MatchState<'t, S>; 1]>, + error_items: &mut SmallVec<[MatchState<'t, S>; 1]>, is_2021: bool, ) { macro_rules! try_push { @@ -468,7 +488,7 @@ fn match_loop_inner<'t>( if let Ok(subtree) = src.clone().expect_subtree() { if subtree.delimiter.kind == delimiter.kind { item.stack.push(item.dot); - item.dot = tokens.iter_delimited(Some(delimiter)); + item.dot = tokens.iter_delimited(Some(*delimiter)); cur_items.push(item); } } @@ -587,9 +607,9 @@ fn match_loop_inner<'t>( } } -fn match_loop(pattern: &MetaTemplate, src: &tt::Subtree, is_2021: bool) -> Match { +fn match_loop(pattern: &MetaTemplate, src: &tt::Subtree, is_2021: bool) -> Match { let mut src = TtIter::new(src); - let mut stack: SmallVec<[TtIter<'_>; 1]> = SmallVec::new(); + let mut stack: SmallVec<[TtIter<'_, S>; 1]> = SmallVec::new(); let mut res = Match::default(); let mut error_recover_item = None; @@ -736,11 +756,11 @@ fn match_loop(pattern: &MetaTemplate, src: &tt::Subtree, is_2021: bool) -> Match } } -fn match_meta_var( +fn match_meta_var( kind: MetaVarKind, - input: &mut TtIter<'_>, + input: &mut TtIter<'_, S>, is_2021: bool, -) -> ExpandResult> { +) -> ExpandResult>> { let fragment = match kind { MetaVarKind::Path => { return input @@ -811,7 +831,7 @@ fn match_meta_var( input.expect_fragment(fragment).map(|it| it.map(Fragment::Tokens)) } -fn collect_vars(collector_fun: &mut impl FnMut(SmolStr), pattern: &MetaTemplate) { +fn collect_vars(collector_fun: &mut impl FnMut(SmolStr), pattern: &MetaTemplate) { for op in pattern.iter() { match op { Op::Var { name, .. } => collector_fun(name.clone()), @@ -824,38 +844,38 @@ fn collect_vars(collector_fun: &mut impl FnMut(SmolStr), pattern: &MetaTemplate) } } } -impl MetaTemplate { - fn iter_delimited<'a>(&'a self, delimited: Option<&'a tt::Delimiter>) -> OpDelimitedIter<'a> { +impl MetaTemplate { + fn iter_delimited(&self, delimited: Option>) -> OpDelimitedIter<'_, S> { OpDelimitedIter { inner: &self.0, idx: 0, - delimited: delimited.unwrap_or(&tt::Delimiter::UNSPECIFIED), + delimited: delimited.unwrap_or(tt::Delimiter::UNSPECIFIED), } } } #[derive(Debug, Clone, Copy)] -enum OpDelimited<'a> { - Op(&'a Op), +enum OpDelimited<'a, S> { + Op(&'a Op), Open, Close, } #[derive(Debug, Clone, Copy)] -struct OpDelimitedIter<'a> { - inner: &'a [Op], - delimited: &'a tt::Delimiter, +struct OpDelimitedIter<'a, S> { + inner: &'a [Op], + delimited: tt::Delimiter, idx: usize, } -impl<'a> OpDelimitedIter<'a> { +impl<'a, S: Span> OpDelimitedIter<'a, S> { fn is_eof(&self) -> bool { let len = self.inner.len() + if self.delimited.kind != tt::DelimiterKind::Invisible { 2 } else { 0 }; self.idx >= len } - fn peek(&self) -> Option> { + fn peek(&self) -> Option> { match self.delimited.kind { tt::DelimiterKind::Invisible => self.inner.get(self.idx).map(OpDelimited::Op), _ => match self.idx { @@ -871,8 +891,8 @@ impl<'a> OpDelimitedIter<'a> { } } -impl<'a> Iterator for OpDelimitedIter<'a> { - type Item = OpDelimited<'a>; +impl<'a, S: Span> Iterator for OpDelimitedIter<'a, S> { + type Item = OpDelimited<'a, S>; fn next(&mut self) -> Option { let res = self.peek(); @@ -888,8 +908,8 @@ impl<'a> Iterator for OpDelimitedIter<'a> { } } -impl TtIter<'_> { - fn expect_separator(&mut self, separator: &Separator) -> bool { +impl TtIter<'_, S> { + fn expect_separator(&mut self, separator: &Separator) -> bool { let mut fork = self.clone(); let ok = match separator { Separator::Ident(lhs) => match fork.expect_ident_or_underscore() { @@ -919,7 +939,7 @@ impl TtIter<'_> { ok } - fn expect_tt(&mut self) -> Result { + fn expect_tt(&mut self) -> Result, ()> { if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = self.peek_n(0) { if punct.char == '\'' { self.expect_lifetime() @@ -936,7 +956,7 @@ impl TtIter<'_> { } } - fn expect_lifetime(&mut self) -> Result { + fn expect_lifetime(&mut self) -> Result, ()> { let punct = self.expect_single_punct()?; if punct.char != '\'' { return Err(()); @@ -953,7 +973,7 @@ impl TtIter<'_> { .into()) } - fn eat_char(&mut self, c: char) -> Option { + fn eat_char(&mut self, c: char) -> Option> { let mut fork = self.clone(); match fork.expect_char(c) { Ok(_) => { diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs index cdac2f1e3bb8..4f5cd0480c78 100644 --- a/crates/mbe/src/expander/transcriber.rs +++ b/crates/mbe/src/expander/transcriber.rs @@ -2,20 +2,20 @@ //! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}` use syntax::SmolStr; +use tt::{Delimiter, Span}; use crate::{ expander::{Binding, Bindings, Fragment}, parser::{MetaVarKind, Op, RepeatKind, Separator}, - tt::{self, Delimiter}, CountError, ExpandError, ExpandResult, MetaTemplate, }; -impl Bindings { +impl Bindings { fn contains(&self, name: &str) -> bool { self.inner.contains_key(name) } - fn get(&self, name: &str) -> Result<&Binding, ExpandError> { + fn get(&self, name: &str) -> Result<&Binding, ExpandError> { match self.inner.get(name) { Some(binding) => Ok(binding), None => Err(ExpandError::binding_error(format!("could not find binding `{name}`"))), @@ -26,7 +26,7 @@ impl Bindings { &self, name: &str, nesting: &mut [NestingState], - ) -> Result { + ) -> Result, ExpandError> { macro_rules! binding_err { ($($arg:tt)*) => { ExpandError::binding_error(format!($($arg)*)) }; } @@ -54,15 +54,15 @@ impl Bindings { Binding::Missing(it) => Ok(match it { MetaVarKind::Stmt => { Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { - span: tt::TokenId::unspecified(), + span: S::DUMMY, char: ';', spacing: tt::Spacing::Alone, }))) } MetaVarKind::Block => Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { delimiter: tt::Delimiter { - open: tt::TokenId::unspecified(), - close: tt::TokenId::unspecified(), + open: S::DUMMY, + close: S::DUMMY, kind: tt::DelimiterKind::Brace, }, token_trees: vec![], @@ -82,19 +82,19 @@ impl Bindings { | MetaVarKind::Ident => { Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: SmolStr::new_inline("missing"), - span: tt::TokenId::unspecified(), + span: S::DUMMY, }))) } MetaVarKind::Lifetime => { Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: SmolStr::new_inline("'missing"), - span: tt::TokenId::unspecified(), + span: S::DUMMY, }))) } MetaVarKind::Literal => { Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: SmolStr::new_inline("\"missing\""), - span: tt::TokenId::unspecified(), + span: S::DUMMY, }))) } }), @@ -108,12 +108,12 @@ impl Bindings { } } -pub(super) fn transcribe( - template: &MetaTemplate, - bindings: &Bindings, -) -> ExpandResult { +pub(super) fn transcribe( + template: &MetaTemplate, + bindings: &Bindings, +) -> ExpandResult> { let mut ctx = ExpandCtx { bindings, nesting: Vec::new() }; - let mut arena: Vec = Vec::new(); + let mut arena: Vec> = Vec::new(); expand_subtree(&mut ctx, template, None, &mut arena) } @@ -129,17 +129,17 @@ struct NestingState { } #[derive(Debug)] -struct ExpandCtx<'a> { - bindings: &'a Bindings, +struct ExpandCtx<'a, S> { + bindings: &'a Bindings, nesting: Vec, } -fn expand_subtree( - ctx: &mut ExpandCtx<'_>, - template: &MetaTemplate, - delimiter: Option, - arena: &mut Vec, -) -> ExpandResult { +fn expand_subtree( + ctx: &mut ExpandCtx<'_, S>, + template: &MetaTemplate, + delimiter: Option>, + arena: &mut Vec>, +) -> ExpandResult> { // remember how many elements are in the arena now - when returning, we want to drain exactly how many elements we added. This way, the recursive uses of the arena get their own "view" of the arena, but will reuse the allocation let start_elements = arena.len(); let mut err = None; @@ -180,7 +180,7 @@ fn expand_subtree( arena.push( tt::Leaf::Literal(tt::Literal { text: index.to_string().into(), - span: tt::TokenId::unspecified(), + span: S::DUMMY, }) .into(), ); @@ -237,11 +237,8 @@ fn expand_subtree( } }; arena.push( - tt::Leaf::Literal(tt::Literal { - text: c.to_string().into(), - span: tt::TokenId::unspecified(), - }) - .into(), + tt::Leaf::Literal(tt::Literal { text: c.to_string().into(), span: S::DUMMY }) + .into(), ); } } @@ -257,7 +254,11 @@ fn expand_subtree( } } -fn expand_var(ctx: &mut ExpandCtx<'_>, v: &SmolStr, id: tt::TokenId) -> ExpandResult { +fn expand_var( + ctx: &mut ExpandCtx<'_, S>, + v: &SmolStr, + id: S, +) -> ExpandResult> { // We already handle $crate case in mbe parser debug_assert!(v != "crate"); @@ -296,14 +297,14 @@ fn expand_var(ctx: &mut ExpandCtx<'_>, v: &SmolStr, id: tt::TokenId) -> ExpandRe } } -fn expand_repeat( - ctx: &mut ExpandCtx<'_>, - template: &MetaTemplate, +fn expand_repeat( + ctx: &mut ExpandCtx<'_, S>, + template: &MetaTemplate, kind: RepeatKind, - separator: &Option, - arena: &mut Vec, -) -> ExpandResult { - let mut buf: Vec = Vec::new(); + separator: &Option>, + arena: &mut Vec>, +) -> ExpandResult> { + let mut buf: Vec> = Vec::new(); ctx.nesting.push(NestingState { idx: 0, at_end: false, hit: false }); // Dirty hack to make macro-expansion terminate. // This should be replaced by a proper macro-by-example implementation @@ -342,7 +343,7 @@ fn expand_repeat( continue; } - t.delimiter = tt::Delimiter::unspecified(); + t.delimiter = tt::Delimiter::UNSPECIFIED; push_subtree(&mut buf, t); if let Some(sep) = separator { @@ -376,7 +377,7 @@ fn expand_repeat( // Check if it is a single token subtree without any delimiter // e.g {Delimiter:None> ['>'] /Delimiter:None>} - let tt = tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: buf }.into(); + let tt = tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: buf }.into(); if RepeatKind::OneOrMore == kind && counter == 0 { return ExpandResult { @@ -387,14 +388,14 @@ fn expand_repeat( ExpandResult { value: Fragment::Tokens(tt), err } } -fn push_fragment(buf: &mut Vec, fragment: Fragment) { +fn push_fragment(buf: &mut Vec>, fragment: Fragment) { match fragment { Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt), Fragment::Expr(tt::TokenTree::Subtree(mut tt)) => { if tt.delimiter.kind == tt::DelimiterKind::Invisible { tt.delimiter = tt::Delimiter { - open: tt::TokenId::UNSPECIFIED, - close: tt::TokenId::UNSPECIFIED, + open: S::DUMMY, + close: S::DUMMY, kind: tt::DelimiterKind::Parenthesis, }; } @@ -405,7 +406,7 @@ fn push_fragment(buf: &mut Vec, fragment: Fragment) { } } -fn push_subtree(buf: &mut Vec, tt: tt::Subtree) { +fn push_subtree(buf: &mut Vec>, tt: tt::Subtree) { match tt.delimiter.kind { tt::DelimiterKind::Invisible => buf.extend(tt.token_trees), _ => buf.push(tt.into()), @@ -415,7 +416,7 @@ fn push_subtree(buf: &mut Vec, tt: tt::Subtree) { /// Inserts the path separator `::` between an identifier and its following generic /// argument list, and then pushes into the buffer. See [`Fragment::Path`] for why /// we need this fixup. -fn fix_up_and_push_path_tt(buf: &mut Vec, subtree: tt::Subtree) { +fn fix_up_and_push_path_tt(buf: &mut Vec>, subtree: tt::Subtree) { stdx::always!(matches!(subtree.delimiter.kind, tt::DelimiterKind::Invisible)); let mut prev_was_ident = false; // Note that we only need to fix up the top-level `TokenTree`s because the @@ -432,7 +433,7 @@ fn fix_up_and_push_path_tt(buf: &mut Vec, subtree: tt::Subtree) { tt::Leaf::Punct(tt::Punct { char: ':', spacing: tt::Spacing::Joint, - span: tt::Span::unspecified(), + span: S::DUMMY, }) .into(), ); @@ -440,7 +441,7 @@ fn fix_up_and_push_path_tt(buf: &mut Vec, subtree: tt::Subtree) { tt::Leaf::Punct(tt::Punct { char: ':', spacing: tt::Spacing::Alone, - span: tt::Span::unspecified(), + span: S::DUMMY, }) .into(), ); @@ -453,9 +454,9 @@ fn fix_up_and_push_path_tt(buf: &mut Vec, subtree: tt::Subtree) { /// Handles `${count(t, depth)}`. `our_depth` is the recursion depth and `count_depth` is the depth /// defined by the metavar expression. -fn count( - ctx: &ExpandCtx<'_>, - binding: &Binding, +fn count( + ctx: &ExpandCtx<'_, S>, + binding: &Binding, our_depth: usize, count_depth: Option, ) -> Result { diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index a439c9c50d6c..46599802935e 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -18,8 +18,8 @@ mod to_parser_input; mod benchmark; mod token_map; -use ::tt::token_id as tt; use stdx::impl_from; +use tt::{Span, TokenId}; use std::fmt; @@ -28,8 +28,9 @@ use crate::{ tt_iter::TtIter, }; -pub use self::tt::{Delimiter, DelimiterKind, Punct}; +// FIXME: we probably should re-think `token_tree_to_syntax_node` interfaces pub use ::parser::TopEntryPoint; +pub use tt::{Delimiter, DelimiterKind, Punct}; pub use crate::{ syntax_bridge::{ @@ -125,7 +126,7 @@ impl fmt::Display for CountError { /// and `$()*` have special meaning (see `Var` and `Repeat` data structures) #[derive(Clone, Debug, PartialEq, Eq)] pub struct DeclarativeMacro { - rules: Box<[Rule]>, + rules: Box<[Rule]>, /// Highest id of the token we have in TokenMap shift: Shift, // This is used for correctly determining the behavior of the pat fragment @@ -135,23 +136,23 @@ pub struct DeclarativeMacro { } #[derive(Clone, Debug, PartialEq, Eq)] -struct Rule { - lhs: MetaTemplate, - rhs: MetaTemplate, +struct Rule { + lhs: MetaTemplate, + rhs: MetaTemplate, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct Shift(u32); impl Shift { - pub fn new(tt: &tt::Subtree) -> Shift { + pub fn new(tt: &tt::Subtree) -> Shift { // Note that TokenId is started from zero, // We have to add 1 to prevent duplication. let value = max_id(tt).map_or(0, |it| it + 1); return Shift(value); // Find the max token id inside a subtree - fn max_id(subtree: &tt::Subtree) -> Option { + fn max_id(subtree: &tt::Subtree) -> Option { let filter = |tt: &_| match tt { tt::TokenTree::Subtree(subtree) => { @@ -177,7 +178,7 @@ impl Shift { } /// Shift given TokenTree token id - pub fn shift_all(self, tt: &mut tt::Subtree) { + pub fn shift_all(self, tt: &mut tt::Subtree) { for t in &mut tt.token_trees { match t { tt::TokenTree::Leaf( @@ -224,7 +225,7 @@ impl DeclarativeMacro { } /// The old, `macro_rules! m {}` flavor. - pub fn parse_macro_rules(tt: &tt::Subtree, is_2021: bool) -> DeclarativeMacro { + pub fn parse_macro_rules(tt: &tt::Subtree, is_2021: bool) -> DeclarativeMacro { // Note: this parsing can be implemented using mbe machinery itself, by // matching against `$($lhs:tt => $rhs:tt);*` pattern, but implementing // manually seems easier. @@ -260,7 +261,7 @@ impl DeclarativeMacro { } /// The new, unstable `macro m {}` flavor. - pub fn parse_macro2(tt: &tt::Subtree, is_2021: bool) -> DeclarativeMacro { + pub fn parse_macro2(tt: &tt::Subtree, is_2021: bool) -> DeclarativeMacro { let mut src = TtIter::new(tt); let mut rules = Vec::new(); let mut err = None; @@ -310,7 +311,7 @@ impl DeclarativeMacro { DeclarativeMacro { rules: rules.into_boxed_slice(), shift: Shift::new(tt), is_2021, err } } - pub fn expand(&self, mut tt: tt::Subtree) -> ExpandResult { + pub fn expand(&self, mut tt: tt::Subtree) -> ExpandResult> { self.shift.shift_all(&mut tt); expander::expand_rules(&self.rules, &tt, self.is_2021) } @@ -335,8 +336,8 @@ impl DeclarativeMacro { } } -impl Rule { - fn parse(src: &mut TtIter<'_>, expect_arrow: bool) -> Result { +impl Rule { + fn parse(src: &mut TtIter<'_, S>, expect_arrow: bool) -> Result { let lhs = src.expect_subtree().map_err(|()| ParseError::expected("expected subtree"))?; if expect_arrow { src.expect_char('=').map_err(|()| ParseError::expected("expected `=`"))?; @@ -351,7 +352,7 @@ impl Rule { } } -fn validate(pattern: &MetaTemplate) -> Result<(), ParseError> { +fn validate(pattern: &MetaTemplate) -> Result<(), ParseError> { for op in pattern.iter() { match op { Op::Subtree { tokens, .. } => validate(tokens)?, diff --git a/crates/mbe/src/parser.rs b/crates/mbe/src/parser.rs index 7a143e7466a9..00ba35377a42 100644 --- a/crates/mbe/src/parser.rs +++ b/crates/mbe/src/parser.rs @@ -3,8 +3,9 @@ use smallvec::{smallvec, SmallVec}; use syntax::SmolStr; +use tt::Span; -use crate::{tt, tt_iter::TtIter, ParseError}; +use crate::{tt_iter::TtIter, ParseError}; /// Consider /// @@ -20,22 +21,22 @@ use crate::{tt, tt_iter::TtIter, ParseError}; /// Stuff to the right is a [`MetaTemplate`] template which is used to produce /// output. #[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct MetaTemplate(pub(crate) Box<[Op]>); +pub(crate) struct MetaTemplate(pub(crate) Box<[Op]>); -impl MetaTemplate { - pub(crate) fn parse_pattern(pattern: &tt::Subtree) -> Result { +impl MetaTemplate { + pub(crate) fn parse_pattern(pattern: &tt::Subtree) -> Result { MetaTemplate::parse(pattern, Mode::Pattern) } - pub(crate) fn parse_template(template: &tt::Subtree) -> Result { + pub(crate) fn parse_template(template: &tt::Subtree) -> Result { MetaTemplate::parse(template, Mode::Template) } - pub(crate) fn iter(&self) -> impl Iterator { + pub(crate) fn iter(&self) -> impl Iterator> { self.0.iter() } - fn parse(tt: &tt::Subtree, mode: Mode) -> Result { + fn parse(tt: &tt::Subtree, mode: Mode) -> Result { let mut src = TtIter::new(tt); let mut res = Vec::new(); @@ -49,16 +50,16 @@ impl MetaTemplate { } #[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) enum Op { - Var { name: SmolStr, kind: Option, id: tt::TokenId }, - Ignore { name: SmolStr, id: tt::TokenId }, +pub(crate) enum Op { + Var { name: SmolStr, kind: Option, id: S }, + Ignore { name: SmolStr, id: S }, Index { depth: usize }, Count { name: SmolStr, depth: Option }, - Repeat { tokens: MetaTemplate, kind: RepeatKind, separator: Option }, - Subtree { tokens: MetaTemplate, delimiter: tt::Delimiter }, - Literal(tt::Literal), - Punct(SmallVec<[tt::Punct; 3]>), - Ident(tt::Ident), + Repeat { tokens: MetaTemplate, kind: RepeatKind, separator: Option> }, + Subtree { tokens: MetaTemplate, delimiter: tt::Delimiter }, + Literal(tt::Literal), + Punct(SmallVec<[tt::Punct; 3]>), + Ident(tt::Ident), } #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -87,15 +88,15 @@ pub(crate) enum MetaVarKind { } #[derive(Clone, Debug, Eq)] -pub(crate) enum Separator { - Literal(tt::Literal), - Ident(tt::Ident), - Puncts(SmallVec<[tt::Punct; 3]>), +pub(crate) enum Separator { + Literal(tt::Literal), + Ident(tt::Ident), + Puncts(SmallVec<[tt::Punct; 3]>), } // Note that when we compare a Separator, we just care about its textual value. -impl PartialEq for Separator { - fn eq(&self, other: &Separator) -> bool { +impl PartialEq for Separator { + fn eq(&self, other: &Separator) -> bool { use Separator::*; match (self, other) { @@ -117,11 +118,11 @@ enum Mode { Template, } -fn next_op( - first_peeked: &tt::TokenTree, - src: &mut TtIter<'_>, +fn next_op( + first_peeked: &tt::TokenTree, + src: &mut TtIter<'_, S>, mode: Mode, -) -> Result { +) -> Result, ParseError> { let res = match first_peeked { tt::TokenTree::Leaf(tt::Leaf::Punct(p @ tt::Punct { char: '$', .. })) => { src.next().expect("first token already peeked"); @@ -212,7 +213,10 @@ fn next_op( Ok(res) } -fn eat_fragment_kind(src: &mut TtIter<'_>, mode: Mode) -> Result, ParseError> { +fn eat_fragment_kind( + src: &mut TtIter<'_, S>, + mode: Mode, +) -> Result, ParseError> { if let Mode::Pattern = mode { src.expect_char(':').map_err(|()| ParseError::unexpected("missing fragment specifier"))?; let ident = src @@ -240,11 +244,13 @@ fn eat_fragment_kind(src: &mut TtIter<'_>, mode: Mode) -> Result bool { +fn is_boolean_literal(lit: &tt::Literal) -> bool { matches!(lit.text.as_str(), "true" | "false") } -fn parse_repeat(src: &mut TtIter<'_>) -> Result<(Option, RepeatKind), ParseError> { +fn parse_repeat( + src: &mut TtIter<'_, S>, +) -> Result<(Option>, RepeatKind), ParseError> { let mut separator = Separator::Puncts(SmallVec::new()); for tt in src { let tt = match tt { @@ -281,7 +287,7 @@ fn parse_repeat(src: &mut TtIter<'_>) -> Result<(Option, RepeatKind), Err(ParseError::InvalidRepeat) } -fn parse_metavar_expr(src: &mut TtIter<'_>) -> Result { +fn parse_metavar_expr(src: &mut TtIter<'_, S>) -> Result, ()> { let func = src.expect_ident()?; let args = src.expect_subtree()?; @@ -314,7 +320,7 @@ fn parse_metavar_expr(src: &mut TtIter<'_>) -> Result { Ok(op) } -fn parse_depth(src: &mut TtIter<'_>) -> Result { +fn parse_depth(src: &mut TtIter<'_, S>) -> Result { if src.len() == 0 { Ok(0) } else if let tt::Leaf::Literal(lit) = src.expect_literal()? { @@ -325,7 +331,7 @@ fn parse_depth(src: &mut TtIter<'_>) -> Result { } } -fn try_eat_comma(src: &mut TtIter<'_>) -> bool { +fn try_eat_comma(src: &mut TtIter<'_, S>) -> bool { if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', .. }))) = src.peek_n(0) { let _ = src.next(); return true; diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 7b9bb61e696a..01aab6b659cb 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -8,23 +8,19 @@ use syntax::{ SyntaxKind::*, SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, WalkEvent, T, }; - -use crate::{ - to_parser_input::to_parser_input, - tt::{ - self, - buffer::{Cursor, TokenBuffer}, - }, - tt_iter::TtIter, - TokenMap, +use tt::{ + buffer::{Cursor, TokenBuffer}, + TokenId, }; +use crate::{to_parser_input::to_parser_input, tt_iter::TtIter, TokenMap}; + #[cfg(test)] mod tests; /// Convert the syntax node to a `TokenTree` (what macro /// will consume). -pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> (tt::Subtree, TokenMap) { +pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> (tt::Subtree, TokenMap) { let (subtree, token_map, _) = syntax_node_to_token_tree_with_modifications( node, Default::default(), @@ -43,7 +39,7 @@ pub fn syntax_node_to_token_tree_with_modifications( next_id: u32, replace: FxHashMap>, append: FxHashMap>, -) -> (tt::Subtree, TokenMap, u32) { +) -> (tt::Subtree, TokenMap, u32) { let global_offset = node.text_range().start(); let mut c = Converter::new(node, global_offset, existing_token_map, next_id, replace, append); let subtree = convert_tokens(&mut c); @@ -108,7 +104,7 @@ pub struct SyntheticToken { // * ForeignItems(SmallVec<[ast::ForeignItem; 1]> pub fn token_tree_to_syntax_node( - tt: &tt::Subtree, + tt: &tt::Subtree, entry_point: parser::TopEntryPoint, ) -> (Parse, TokenMap) { let buffer = match tt { @@ -138,7 +134,7 @@ pub fn token_tree_to_syntax_node( } /// Convert a string to a `TokenTree` -pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> { +pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> { let lexed = parser::LexedStr::new(text); if lexed.errors().next().is_some() { return None; @@ -159,7 +155,7 @@ pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> { } /// Split token tree with separate expr: $($e:expr)SEP* -pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec { +pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec> { if tt.token_trees.is_empty() { return Vec::new(); } @@ -195,9 +191,9 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec { res } -fn convert_tokens(conv: &mut C) -> tt::Subtree { +fn convert_tokens(conv: &mut C) -> tt::Subtree { struct StackEntry { - subtree: tt::Subtree, + subtree: tt::Subtree, idx: usize, open_range: TextRange, } @@ -296,7 +292,7 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { .into() }; } - let leaf: tt::Leaf = match kind { + let leaf: tt::Leaf = match kind { T![true] | T![false] => make_leaf!(Ident), IDENT => make_leaf!(Ident), UNDERSCORE => make_leaf!(Ident), @@ -335,7 +331,7 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { let parent = stack.last_mut(); conv.id_alloc().close_delim(entry.idx, None); - let leaf: tt::Leaf = tt::Punct { + let leaf: tt::Leaf = tt::Punct { span: conv.id_alloc().alloc(entry.open_range, None), char: match entry.subtree.delimiter.kind { tt::DelimiterKind::Parenthesis => '(', @@ -514,7 +510,7 @@ fn doc_comment_text(comment: &ast::Comment) -> SmolStr { fn convert_doc_comment( token: &syntax::SyntaxToken, span: tt::TokenId, -) -> Option> { +) -> Option>> { cov_mark::hit!(test_meta_doc_comments); let comment = ast::Comment::cast(token.clone())?; let doc = comment.kind().doc?; @@ -537,11 +533,11 @@ fn convert_doc_comment( return Some(token_trees); // Helper functions - fn mk_ident(s: &str, span: tt::TokenId) -> tt::TokenTree { + fn mk_ident(s: &str, span: tt::TokenId) -> tt::TokenTree { tt::TokenTree::from(tt::Leaf::from(tt::Ident { text: s.into(), span })) } - fn mk_punct(c: char, span: tt::TokenId) -> tt::TokenTree { + fn mk_punct(c: char, span: tt::TokenId) -> tt::TokenTree { tt::TokenTree::from(tt::Leaf::from(tt::Punct { char: c, spacing: tt::Spacing::Alone, @@ -549,7 +545,7 @@ fn convert_doc_comment( })) } - fn mk_doc_literal(comment: &ast::Comment, span: tt::TokenId) -> tt::TokenTree { + fn mk_doc_literal(comment: &ast::Comment, span: tt::TokenId) -> tt::TokenTree { let lit = tt::Literal { text: doc_comment_text(comment), span }; tt::TokenTree::from(tt::Leaf::from(lit)) @@ -636,7 +632,7 @@ trait TokenConverter: Sized { &self, token: &Self::Token, span: tt::TokenId, - ) -> Option>; + ) -> Option>>; fn bump(&mut self) -> Option<(Self::Token, TextRange)>; @@ -666,7 +662,11 @@ impl SrcToken> for usize { impl TokenConverter for RawConverter<'_> { type Token = usize; - fn convert_doc_comment(&self, &token: &usize, span: tt::TokenId) -> Option> { + fn convert_doc_comment( + &self, + &token: &usize, + span: tt::TokenId, + ) -> Option>> { let text = self.lexed.text(token); convert_doc_comment(&doc_comment(text), span) } @@ -819,7 +819,7 @@ impl TokenConverter for Converter { &self, token: &Self::Token, span: tt::TokenId, - ) -> Option> { + ) -> Option>> { convert_doc_comment(token.token()?, span) } @@ -899,7 +899,7 @@ impl TokenConverter for Converter { struct TtTreeSink<'a> { buf: String, - cursor: Cursor<'a>, + cursor: Cursor<'a, TokenId>, open_delims: FxHashMap, text_pos: TextSize, inner: SyntaxTreeBuilder, @@ -907,7 +907,7 @@ struct TtTreeSink<'a> { } impl<'a> TtTreeSink<'a> { - fn new(cursor: Cursor<'a>) -> Self { + fn new(cursor: Cursor<'a, TokenId>) -> Self { TtTreeSink { buf: String::new(), cursor, diff --git a/crates/mbe/src/to_parser_input.rs b/crates/mbe/src/to_parser_input.rs index 051e20b3a3f9..00a14f04686e 100644 --- a/crates/mbe/src/to_parser_input.rs +++ b/crates/mbe/src/to_parser_input.rs @@ -3,9 +3,9 @@ use syntax::{SyntaxKind, SyntaxKind::*, T}; -use crate::tt::buffer::TokenBuffer; +use tt::{buffer::TokenBuffer, Span}; -pub(crate) fn to_parser_input(buffer: &TokenBuffer<'_>) -> parser::Input { +pub(crate) fn to_parser_input(buffer: &TokenBuffer<'_, S>) -> parser::Input { let mut res = parser::Input::default(); let mut current = buffer.begin(); diff --git a/crates/mbe/src/tt_iter.rs b/crates/mbe/src/tt_iter.rs index 79ff8ca28e86..44fbbcfc2068 100644 --- a/crates/mbe/src/tt_iter.rs +++ b/crates/mbe/src/tt_iter.rs @@ -3,16 +3,17 @@ use smallvec::{smallvec, SmallVec}; use syntax::SyntaxKind; +use tt::Span; -use crate::{to_parser_input::to_parser_input, tt, ExpandError, ExpandResult}; +use crate::{to_parser_input::to_parser_input, ExpandError, ExpandResult}; #[derive(Debug, Clone)] -pub(crate) struct TtIter<'a> { - pub(crate) inner: std::slice::Iter<'a, tt::TokenTree>, +pub(crate) struct TtIter<'a, S> { + pub(crate) inner: std::slice::Iter<'a, tt::TokenTree>, } -impl<'a> TtIter<'a> { - pub(crate) fn new(subtree: &'a tt::Subtree) -> TtIter<'a> { +impl<'a, S: Span> TtIter<'a, S> { + pub(crate) fn new(subtree: &'a tt::Subtree) -> TtIter<'a, S> { TtIter { inner: subtree.token_trees.iter() } } @@ -36,35 +37,35 @@ impl<'a> TtIter<'a> { } } - pub(crate) fn expect_subtree(&mut self) -> Result<&'a tt::Subtree, ()> { + pub(crate) fn expect_subtree(&mut self) -> Result<&'a tt::Subtree, ()> { match self.next() { Some(tt::TokenTree::Subtree(it)) => Ok(it), _ => Err(()), } } - pub(crate) fn expect_leaf(&mut self) -> Result<&'a tt::Leaf, ()> { + pub(crate) fn expect_leaf(&mut self) -> Result<&'a tt::Leaf, ()> { match self.next() { Some(tt::TokenTree::Leaf(it)) => Ok(it), _ => Err(()), } } - pub(crate) fn expect_ident(&mut self) -> Result<&'a tt::Ident, ()> { + pub(crate) fn expect_ident(&mut self) -> Result<&'a tt::Ident, ()> { match self.expect_leaf()? { tt::Leaf::Ident(it) if it.text != "_" => Ok(it), _ => Err(()), } } - pub(crate) fn expect_ident_or_underscore(&mut self) -> Result<&'a tt::Ident, ()> { + pub(crate) fn expect_ident_or_underscore(&mut self) -> Result<&'a tt::Ident, ()> { match self.expect_leaf()? { tt::Leaf::Ident(it) => Ok(it), _ => Err(()), } } - pub(crate) fn expect_literal(&mut self) -> Result<&'a tt::Leaf, ()> { + pub(crate) fn expect_literal(&mut self) -> Result<&'a tt::Leaf, ()> { let it = self.expect_leaf()?; match it { tt::Leaf::Literal(_) => Ok(it), @@ -73,7 +74,7 @@ impl<'a> TtIter<'a> { } } - pub(crate) fn expect_single_punct(&mut self) -> Result<&'a tt::Punct, ()> { + pub(crate) fn expect_single_punct(&mut self) -> Result<&'a tt::Punct, ()> { match self.expect_leaf()? { tt::Leaf::Punct(it) => Ok(it), _ => Err(()), @@ -84,7 +85,7 @@ impl<'a> TtIter<'a> { /// /// This method currently may return a single quotation, which is part of lifetime ident and /// conceptually not a punct in the context of mbe. Callers should handle this. - pub(crate) fn expect_glued_punct(&mut self) -> Result, ()> { + pub(crate) fn expect_glued_punct(&mut self) -> Result; 3]>, ()> { let tt::TokenTree::Leaf(tt::Leaf::Punct(first)) = self.next().ok_or(())?.clone() else { return Err(()); }; @@ -126,7 +127,7 @@ impl<'a> TtIter<'a> { pub(crate) fn expect_fragment( &mut self, entry_point: parser::PrefixEntryPoint, - ) -> ExpandResult> { + ) -> ExpandResult>> { let buffer = tt::buffer::TokenBuffer::from_tokens(self.inner.as_slice()); let parser_input = to_parser_input(&buffer); let tree_traversal = entry_point.parse(&parser_input); @@ -181,13 +182,13 @@ impl<'a> TtIter<'a> { ExpandResult { value: res, err } } - pub(crate) fn peek_n(&self, n: usize) -> Option<&'a tt::TokenTree> { + pub(crate) fn peek_n(&self, n: usize) -> Option<&'a tt::TokenTree> { self.inner.as_slice().get(n) } } -impl<'a> Iterator for TtIter<'a> { - type Item = &'a tt::TokenTree; +impl<'a, S> Iterator for TtIter<'a, S> { + type Item = &'a tt::TokenTree; fn next(&mut self) -> Option { self.inner.next() } @@ -197,4 +198,4 @@ impl<'a> Iterator for TtIter<'a> { } } -impl std::iter::ExactSizeIterator for TtIter<'_> {} +impl std::iter::ExactSizeIterator for TtIter<'_, S> {} diff --git a/crates/tt/src/lib.rs b/crates/tt/src/lib.rs index b5a72bec079a..a4ffc328f217 100644 --- a/crates/tt/src/lib.rs +++ b/crates/tt/src/lib.rs @@ -47,70 +47,44 @@ pub mod token_id { pub type Cursor<'a> = crate::buffer::Cursor<'a, super::Span>; pub type TokenTreeRef<'a> = crate::buffer::TokenTreeRef<'a, super::Span>; } +} - impl Delimiter { - pub const UNSPECIFIED: Self = Self { - open: TokenId::UNSPECIFIED, - close: TokenId::UNSPECIFIED, - kind: DelimiterKind::Invisible, - }; - pub const fn unspecified() -> Self { - Self::UNSPECIFIED - } - } - impl Subtree { - pub const fn empty() -> Self { - Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] } - } - } - impl TokenTree { - pub const fn empty() -> Self { - Self::Subtree(Subtree::empty()) - } - } +pub trait Span: std::fmt::Debug + Copy + Sized { + const DUMMY: Self; + fn is_dummy(&self) -> bool; +} +impl Span for TokenId { + const DUMMY: Self = TokenId(!0); - impl Subtree { - pub fn visit_ids(&mut self, f: &mut impl FnMut(TokenId) -> TokenId) { - self.delimiter.open = f(self.delimiter.open); - self.delimiter.close = f(self.delimiter.close); - self.token_trees.iter_mut().for_each(|tt| match tt { - crate::TokenTree::Leaf(leaf) => match leaf { - crate::Leaf::Literal(it) => it.span = f(it.span), - crate::Leaf::Punct(it) => it.span = f(it.span), - crate::Leaf::Ident(it) => it.span = f(it.span), - }, - crate::TokenTree::Subtree(s) => s.visit_ids(f), - }) - } + fn is_dummy(&self) -> bool { + *self == Self::DUMMY } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct SyntaxContext(pub u32); -// #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -// pub struct Span { -// pub id: TokenId, -// pub ctx: SyntaxContext, -// } -// pub type Span = (TokenId, SyntaxContext); - #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum TokenTree { - Leaf(Leaf), - Subtree(Subtree), +pub enum TokenTree { + Leaf(Leaf), + Subtree(Subtree), +} +impl_from!(Leaf, Subtree for TokenTree); +impl TokenTree { + pub const fn empty() -> Self { + Self::Subtree(Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] }) + } } -impl_from!(Leaf, Subtree for TokenTree); #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum Leaf { - Literal(Literal), - Punct(Punct), - Ident(Ident), +pub enum Leaf { + Literal(Literal), + Punct(Punct), + Ident(Ident), } -impl Leaf { - pub fn span(&self) -> &Span { +impl Leaf { + pub fn span(&self) -> &S { match self { Leaf::Literal(it) => &it.span, Leaf::Punct(it) => &it.span, @@ -118,21 +92,49 @@ impl Leaf { } } } -impl_from!(Literal, Punct, Ident for Leaf); +impl_from!(Literal, Punct, Ident for Leaf); #[derive(Clone, PartialEq, Eq, Hash)] -pub struct Subtree { - pub delimiter: Delimiter, - pub token_trees: Vec>, +pub struct Subtree { + pub delimiter: Delimiter, + pub token_trees: Vec>, +} + +impl Subtree { + pub const fn empty() -> Self { + Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] } + } + + pub fn visit_ids(&mut self, f: &mut impl FnMut(S) -> S) { + self.delimiter.open = f(self.delimiter.open); + self.delimiter.close = f(self.delimiter.close); + self.token_trees.iter_mut().for_each(|tt| match tt { + crate::TokenTree::Leaf(leaf) => match leaf { + crate::Leaf::Literal(it) => it.span = f(it.span), + crate::Leaf::Punct(it) => it.span = f(it.span), + crate::Leaf::Ident(it) => it.span = f(it.span), + }, + crate::TokenTree::Subtree(s) => s.visit_ids(f), + }) + } } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct Delimiter { - pub open: Span, - pub close: Span, +pub struct Delimiter { + pub open: S, + pub close: S, pub kind: DelimiterKind, } +impl Delimiter { + pub const UNSPECIFIED: Self = + Self { open: S::DUMMY, close: S::DUMMY, kind: DelimiterKind::Invisible }; + pub const fn unspecified() -> Self { + Self::UNSPECIFIED + } +} + + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum DelimiterKind { Parenthesis, @@ -142,16 +144,16 @@ pub enum DelimiterKind { } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Literal { +pub struct Literal { pub text: SmolStr, - pub span: Span, + pub span: S, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct Punct { +pub struct Punct { pub char: char, pub spacing: Spacing, - pub span: Span, + pub span: S, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -162,9 +164,9 @@ pub enum Spacing { #[derive(Debug, Clone, PartialEq, Eq, Hash)] /// Identifier or keyword. Unlike rustc, we keep "r#" prefix when it represents a raw identifier. -pub struct Ident { +pub struct Ident { pub text: SmolStr, - pub span: Span, + pub span: S, } impl Ident { @@ -173,9 +175,9 @@ impl Ident { } } -fn print_debug_subtree( +fn print_debug_subtree( f: &mut fmt::Formatter<'_>, - subtree: &Subtree, + subtree: &Subtree, level: usize, ) -> fmt::Result { let align = " ".repeat(level); @@ -203,9 +205,9 @@ fn print_debug_subtree( Ok(()) } -fn print_debug_token( +fn print_debug_token( f: &mut fmt::Formatter<'_>, - tkn: &TokenTree, + tkn: &TokenTree, level: usize, ) -> fmt::Result { let align = " ".repeat(level); @@ -231,13 +233,13 @@ fn print_debug_token( Ok(()) } -impl fmt::Debug for Subtree { +impl fmt::Debug for Subtree { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { print_debug_subtree(f, self, 0) } } -impl fmt::Display for TokenTree { +impl fmt::Display for TokenTree { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { TokenTree::Leaf(it) => fmt::Display::fmt(it, f), @@ -246,7 +248,7 @@ impl fmt::Display for TokenTree { } } -impl fmt::Display for Subtree { +impl fmt::Display for Subtree { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let (l, r) = match self.delimiter.kind { DelimiterKind::Parenthesis => ("(", ")"), @@ -274,7 +276,7 @@ impl fmt::Display for Subtree { } } -impl fmt::Display for Leaf { +impl fmt::Display for Leaf { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Leaf::Ident(it) => fmt::Display::fmt(it, f), @@ -284,25 +286,25 @@ impl fmt::Display for Leaf { } } -impl fmt::Display for Ident { +impl fmt::Display for Ident { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&self.text, f) } } -impl fmt::Display for Literal { +impl fmt::Display for Literal { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&self.text, f) } } -impl fmt::Display for Punct { +impl fmt::Display for Punct { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&self.char, f) } } -impl Subtree { +impl Subtree { /// Count the number of tokens recursively pub fn count(&self) -> usize { let children_count = self @@ -318,7 +320,7 @@ impl Subtree { } } -impl Subtree { +impl Subtree { /// A simple line string used for debugging pub fn as_debug_string(&self) -> String { let delim = match self.delimiter.kind { @@ -366,8 +368,8 @@ impl Subtree { pub mod buffer; -pub fn pretty(tkns: &[TokenTree]) -> String { - fn tokentree_to_text(tkn: &TokenTree) -> String { +pub fn pretty(tkns: &[TokenTree]) -> String { + fn tokentree_to_text(tkn: &TokenTree) -> String { match tkn { TokenTree::Leaf(Leaf::Ident(ident)) => ident.text.clone().into(), TokenTree::Leaf(Leaf::Literal(literal)) => literal.text.clone().into(), From f79439caedee7ed00252aff8d4d4fae8fac0c597 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Thu, 29 Jun 2023 12:23:45 +0200 Subject: [PATCH 02/24] Infect proc-macro-api crate with generic span type parameter --- crates/proc-macro-api/src/lib.rs | 12 +- crates/proc-macro-api/src/msg.rs | 23 ++- crates/proc-macro-api/src/msg/flat.rs | 230 ++++++++++++++++++-------- 3 files changed, 180 insertions(+), 85 deletions(-) diff --git a/crates/proc-macro-api/src/lib.rs b/crates/proc-macro-api/src/lib.rs index 1603458f756e..7a3580f814e2 100644 --- a/crates/proc-macro-api/src/lib.rs +++ b/crates/proc-macro-api/src/lib.rs @@ -17,10 +17,8 @@ use triomphe::Arc; use serde::{Deserialize, Serialize}; -use ::tt::token_id as tt; - use crate::{ - msg::{ExpandMacro, FlatTree, PanicMessage}, + msg::{flat::SerializableSpan, ExpandMacro, FlatTree, PanicMessage}, process::ProcMacroProcessSrv, }; @@ -134,12 +132,12 @@ impl ProcMacro { self.kind } - pub fn expand( + pub fn expand>( &self, - subtree: &tt::Subtree, - attr: Option<&tt::Subtree>, + subtree: &tt::Subtree, + attr: Option<&tt::Subtree>, env: Vec<(String, String)>, - ) -> Result, ServerError> { + ) -> Result, PanicMessage>, ServerError> { let version = self.process.lock().unwrap_or_else(|e| e.into_inner()).version(); let current_dir = env .iter() diff --git a/crates/proc-macro-api/src/msg.rs b/crates/proc-macro-api/src/msg.rs index 4b01643c2a29..f0719777ab4d 100644 --- a/crates/proc-macro-api/src/msg.rs +++ b/crates/proc-macro-api/src/msg.rs @@ -16,8 +16,15 @@ pub use crate::msg::flat::FlatTree; pub const NO_VERSION_CHECK_VERSION: u32 = 0; pub const VERSION_CHECK_VERSION: u32 = 1; pub const ENCODE_CLOSE_SPAN_VERSION: u32 = 2; +/// This version changes how spans are encoded, kind of. Prior to this version, +/// spans were represented as a single u32 which effectively forced spans to be +/// token ids. Starting with this version, the span fields are still u32, +/// but if the size of the span is greater than 1 then the span data is encoded in +/// an additional vector where the span represents the offset into that vector. +/// This allows encoding bigger spans while supporting the previous versions. +pub const VARIABLE_SIZED_SPANS: u32 = 2; -pub const CURRENT_API_VERSION: u32 = ENCODE_CLOSE_SPAN_VERSION; +pub const CURRENT_API_VERSION: u32 = VARIABLE_SIZED_SPANS; #[derive(Debug, Serialize, Deserialize)] pub enum Request { @@ -115,10 +122,14 @@ fn write_json(out: &mut impl Write, msg: &str) -> io::Result<()> { #[cfg(test)] mod tests { + use tt::{ + Delimiter, DelimiterKind, Ident, Leaf, Literal, Punct, Spacing, Span, Subtree, TokenId, + TokenTree, + }; + use super::*; - use crate::tt::*; - fn fixture_token_tree() -> Subtree { + fn fixture_token_tree() -> Subtree { let mut subtree = Subtree { delimiter: Delimiter::unspecified(), token_trees: Vec::new() }; subtree .token_trees @@ -128,17 +139,17 @@ mod tests { .push(TokenTree::Leaf(Ident { text: "Foo".into(), span: TokenId(1) }.into())); subtree.token_trees.push(TokenTree::Leaf(Leaf::Literal(Literal { text: "Foo".into(), - span: TokenId::unspecified(), + span: TokenId::DUMMY, }))); subtree.token_trees.push(TokenTree::Leaf(Leaf::Punct(Punct { char: '@', - span: TokenId::unspecified(), + span: TokenId::DUMMY, spacing: Spacing::Joint, }))); subtree.token_trees.push(TokenTree::Subtree(Subtree { delimiter: Delimiter { open: TokenId(2), - close: TokenId::UNSPECIFIED, + close: TokenId::DUMMY, kind: DelimiterKind::Brace, }, token_trees: vec![], diff --git a/crates/proc-macro-api/src/msg/flat.rs b/crates/proc-macro-api/src/msg/flat.rs index 44245336f016..fe82b8d04542 100644 --- a/crates/proc-macro-api/src/msg/flat.rs +++ b/crates/proc-macro-api/src/msg/flat.rs @@ -38,11 +38,22 @@ use std::collections::{HashMap, VecDeque}; use serde::{Deserialize, Serialize}; +use tt::Span; -use crate::{ - msg::ENCODE_CLOSE_SPAN_VERSION, - tt::{self, TokenId}, -}; +use crate::msg::{ENCODE_CLOSE_SPAN_VERSION, VARIABLE_SIZED_SPANS}; + +pub trait SerializableSpan: Span { + fn into_u32(self) -> [u32; L]; + fn from_u32(input: [u32; L]) -> Self; +} +impl SerializableSpan<1> for tt::TokenId { + fn into_u32(self) -> [u32; 1] { + [self.0] + } + fn from_u32([input]: [u32; 1]) -> Self { + tt::TokenId(input) + } +} #[derive(Serialize, Deserialize, Debug)] pub struct FlatTree { @@ -52,33 +63,79 @@ pub struct FlatTree { ident: Vec, token_tree: Vec, text: Vec, + #[serde(skip_serializing_if = "SpanMap::do_serialize")] + #[serde(default)] + span_map: SpanMap, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct SpanMap { + #[serde(skip_serializing)] + serialize: bool, + span_size: u32, + spans: Vec, +} + +impl Default for SpanMap { + fn default() -> Self { + Self { serialize: false, span_size: 1, spans: Default::default() } + } } -struct SubtreeRepr { - open: tt::TokenId, - close: tt::TokenId, +impl SpanMap { + fn serialize_span>(&mut self, span: S) -> u32 { + let u32s = span.into_u32(); + if L == 1 { + u32s[0] + } else { + let offset = self.spans.len() as u32; + self.spans.extend(u32s); + offset + } + } + fn deserialize_span>(&self, offset: u32) -> S { + S::from_u32(if L == 1 { + [offset].as_ref().try_into().unwrap() + } else { + self.spans[offset as usize..][..L].try_into().unwrap() + }) + } +} + +impl SpanMap { + pub fn do_serialize(&self) -> bool { + self.serialize + } +} + +struct SubtreeRepr { + open: S, + close: S, kind: tt::DelimiterKind, tt: [u32; 2], } -struct LiteralRepr { - id: tt::TokenId, +struct LiteralRepr { + id: S, text: u32, } -struct PunctRepr { - id: tt::TokenId, +struct PunctRepr { + id: S, char: char, spacing: tt::Spacing, } -struct IdentRepr { - id: tt::TokenId, +struct IdentRepr { + id: S, text: u32, } impl FlatTree { - pub fn new(subtree: &tt::Subtree, version: u32) -> FlatTree { + pub fn new>( + subtree: &tt::Subtree, + version: u32, + ) -> FlatTree { let mut w = Writer { string_table: HashMap::new(), work: VecDeque::new(), @@ -91,60 +148,78 @@ impl FlatTree { text: Vec::new(), }; w.write(subtree); - + assert!(L == 1 || version >= VARIABLE_SIZED_SPANS); + let mut span_map = SpanMap { + serialize: version >= VARIABLE_SIZED_SPANS && L != 1, + span_size: L as u32, + spans: Vec::new(), + }; return FlatTree { subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - write_vec(w.subtree, SubtreeRepr::write_with_close_span) + write_vec(&mut span_map, w.subtree, SubtreeRepr::write_with_close_span) } else { - write_vec(w.subtree, SubtreeRepr::write) + write_vec(&mut span_map, w.subtree, SubtreeRepr::write) }, - literal: write_vec(w.literal, LiteralRepr::write), - punct: write_vec(w.punct, PunctRepr::write), - ident: write_vec(w.ident, IdentRepr::write), + literal: write_vec(&mut span_map, w.literal, LiteralRepr::write), + punct: write_vec(&mut span_map, w.punct, PunctRepr::write), + ident: write_vec(&mut span_map, w.ident, IdentRepr::write), token_tree: w.token_tree, text: w.text, + span_map, }; - fn write_vec [u32; N], const N: usize>(xs: Vec, f: F) -> Vec { - xs.into_iter().flat_map(f).collect() + fn write_vec [u32; N], const N: usize>( + map: &mut SpanMap, + xs: Vec, + f: F, + ) -> Vec { + xs.into_iter().flat_map(|it| f(it, map)).collect() } } - pub fn to_subtree(self, version: u32) -> tt::Subtree { + pub fn to_subtree>( + self, + version: u32, + ) -> tt::Subtree { + assert!((version >= VARIABLE_SIZED_SPANS || L == 1) && L as u32 == self.span_map.span_size); return Reader { subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - read_vec(self.subtree, SubtreeRepr::read_with_close_span) + read_vec(&self.span_map, self.subtree, SubtreeRepr::read_with_close_span) } else { - read_vec(self.subtree, SubtreeRepr::read) + read_vec(&self.span_map, self.subtree, SubtreeRepr::read) }, - literal: read_vec(self.literal, LiteralRepr::read), - punct: read_vec(self.punct, PunctRepr::read), - ident: read_vec(self.ident, IdentRepr::read), + literal: read_vec(&self.span_map, self.literal, LiteralRepr::read), + punct: read_vec(&self.span_map, self.punct, PunctRepr::read), + ident: read_vec(&self.span_map, self.ident, IdentRepr::read), token_tree: self.token_tree, text: self.text, } .read(); - fn read_vec T, const N: usize>(xs: Vec, f: F) -> Vec { + fn read_vec T, const N: usize>( + map: &SpanMap, + xs: Vec, + f: F, + ) -> Vec { let mut chunks = xs.chunks_exact(N); - let res = chunks.by_ref().map(|chunk| f(chunk.try_into().unwrap())).collect(); + let res = chunks.by_ref().map(|chunk| f(chunk.try_into().unwrap(), map)).collect(); assert!(chunks.remainder().is_empty()); res } } } -impl SubtreeRepr { - fn write(self) -> [u32; 4] { +impl> SubtreeRepr { + fn write(self, map: &mut SpanMap) -> [u32; 4] { let kind = match self.kind { tt::DelimiterKind::Invisible => 0, tt::DelimiterKind::Parenthesis => 1, tt::DelimiterKind::Brace => 2, tt::DelimiterKind::Bracket => 3, }; - [self.open.0, kind, self.tt[0], self.tt[1]] + [map.serialize_span(self.open), kind, self.tt[0], self.tt[1]] } - fn read([open, kind, lo, len]: [u32; 4]) -> SubtreeRepr { + fn read([open, kind, lo, len]: [u32; 4], map: &SpanMap) -> Self { let kind = match kind { 0 => tt::DelimiterKind::Invisible, 1 => tt::DelimiterKind::Parenthesis, @@ -152,18 +227,24 @@ impl SubtreeRepr { 3 => tt::DelimiterKind::Bracket, other => panic!("bad kind {other}"), }; - SubtreeRepr { open: TokenId(open), close: TokenId::UNSPECIFIED, kind, tt: [lo, len] } + SubtreeRepr { open: map.deserialize_span(open), close: S::DUMMY, kind, tt: [lo, len] } } - fn write_with_close_span(self) -> [u32; 5] { + fn write_with_close_span(self, map: &mut SpanMap) -> [u32; 5] { let kind = match self.kind { tt::DelimiterKind::Invisible => 0, tt::DelimiterKind::Parenthesis => 1, tt::DelimiterKind::Brace => 2, tt::DelimiterKind::Bracket => 3, }; - [self.open.0, self.close.0, kind, self.tt[0], self.tt[1]] + [ + map.serialize_span(self.open), + map.serialize_span(self.close), + kind, + self.tt[0], + self.tt[1], + ] } - fn read_with_close_span([open, close, kind, lo, len]: [u32; 5]) -> SubtreeRepr { + fn read_with_close_span([open, close, kind, lo, len]: [u32; 5], map: &SpanMap) -> Self { let kind = match kind { 0 => tt::DelimiterKind::Invisible, 1 => tt::DelimiterKind::Parenthesis, @@ -171,67 +252,72 @@ impl SubtreeRepr { 3 => tt::DelimiterKind::Bracket, other => panic!("bad kind {other}"), }; - SubtreeRepr { open: TokenId(open), close: TokenId(close), kind, tt: [lo, len] } + SubtreeRepr { + open: map.deserialize_span(open), + close: map.deserialize_span(close), + kind, + tt: [lo, len], + } } } -impl LiteralRepr { - fn write(self) -> [u32; 2] { - [self.id.0, self.text] +impl> LiteralRepr { + fn write(self, map: &mut SpanMap) -> [u32; 2] { + [map.serialize_span(self.id), self.text] } - fn read([id, text]: [u32; 2]) -> LiteralRepr { - LiteralRepr { id: TokenId(id), text } + fn read([id, text]: [u32; 2], map: &SpanMap) -> Self { + LiteralRepr { id: map.deserialize_span(id), text } } } -impl PunctRepr { - fn write(self) -> [u32; 3] { +impl> PunctRepr { + fn write(self, map: &mut SpanMap) -> [u32; 3] { let spacing = match self.spacing { tt::Spacing::Alone => 0, tt::Spacing::Joint => 1, }; - [self.id.0, self.char as u32, spacing] + [map.serialize_span(self.id), self.char as u32, spacing] } - fn read([id, char, spacing]: [u32; 3]) -> PunctRepr { + fn read([id, char, spacing]: [u32; 3], map: &SpanMap) -> Self { let spacing = match spacing { 0 => tt::Spacing::Alone, 1 => tt::Spacing::Joint, other => panic!("bad spacing {other}"), }; - PunctRepr { id: TokenId(id), char: char.try_into().unwrap(), spacing } + PunctRepr { id: map.deserialize_span(id), char: char.try_into().unwrap(), spacing } } } -impl IdentRepr { - fn write(self) -> [u32; 2] { - [self.id.0, self.text] +impl> IdentRepr { + fn write(self, map: &mut SpanMap) -> [u32; 2] { + [map.serialize_span(self.id), self.text] } - fn read(data: [u32; 2]) -> IdentRepr { - IdentRepr { id: TokenId(data[0]), text: data[1] } + fn read(data: [u32; 2], map: &SpanMap) -> Self { + IdentRepr { id: map.deserialize_span(data[0]), text: data[1] } } } -struct Writer<'a> { - work: VecDeque<(usize, &'a tt::Subtree)>, +struct Writer<'a, const L: usize, S> { + work: VecDeque<(usize, &'a tt::Subtree)>, string_table: HashMap<&'a str, u32>, - subtree: Vec, - literal: Vec, - punct: Vec, - ident: Vec, + subtree: Vec>, + literal: Vec>, + punct: Vec>, + ident: Vec>, token_tree: Vec, text: Vec, } -impl<'a> Writer<'a> { - fn write(&mut self, root: &'a tt::Subtree) { +impl<'a, const L: usize, S: Copy> Writer<'a, L, S> { + fn write(&mut self, root: &'a tt::Subtree) { self.enqueue(root); while let Some((idx, subtree)) = self.work.pop_front() { self.subtree(idx, subtree); } } - fn subtree(&mut self, idx: usize, subtree: &'a tt::Subtree) { + fn subtree(&mut self, idx: usize, subtree: &'a tt::Subtree) { let mut first_tt = self.token_tree.len(); let n_tt = subtree.token_trees.len(); self.token_tree.resize(first_tt + n_tt, !0); @@ -273,7 +359,7 @@ impl<'a> Writer<'a> { } } - fn enqueue(&mut self, subtree: &'a tt::Subtree) -> u32 { + fn enqueue(&mut self, subtree: &'a tt::Subtree) -> u32 { let idx = self.subtree.len(); let open = subtree.delimiter.open; let close = subtree.delimiter.close; @@ -293,18 +379,18 @@ impl<'a> Writer<'a> { } } -struct Reader { - subtree: Vec, - literal: Vec, - punct: Vec, - ident: Vec, +struct Reader { + subtree: Vec>, + literal: Vec>, + punct: Vec>, + ident: Vec>, token_tree: Vec, text: Vec, } -impl Reader { - pub(crate) fn read(self) -> tt::Subtree { - let mut res: Vec> = vec![None; self.subtree.len()]; +impl> Reader { + pub(crate) fn read(self) -> tt::Subtree { + let mut res: Vec>> = vec![None; self.subtree.len()]; for i in (0..self.subtree.len()).rev() { let repr = &self.subtree[i]; let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize]; From 890eb17b4e1659b22f8113ff99506a2b0eefe5ff Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Fri, 29 Sep 2023 12:37:57 +0200 Subject: [PATCH 03/24] Replace ID based TokenMap with proper relative text-ranges / spans --- Cargo.lock | 2 + crates/base-db/src/fixture.rs | 33 +- crates/base-db/src/input.rs | 9 +- crates/base-db/src/lib.rs | 1 + crates/base-db/src/span.rs | 166 +++++ crates/cfg/src/tests.rs | 39 +- crates/hir-def/src/attr.rs | 89 ++- crates/hir-def/src/attr/tests.rs | 11 +- crates/hir-def/src/data.rs | 2 +- crates/hir-def/src/expander.rs | 17 +- crates/hir-def/src/generics.rs | 18 +- crates/hir-def/src/item_tree.rs | 7 +- crates/hir-def/src/item_tree/lower.rs | 210 ++++-- crates/hir-def/src/lib.rs | 18 +- .../hir-def/src/macro_expansion_tests/mbe.rs | 69 +- .../hir-def/src/macro_expansion_tests/mod.rs | 97 +-- crates/hir-def/src/nameres/collector.rs | 9 +- crates/hir-def/src/nameres/mod_resolution.rs | 2 +- .../hir-def/src/nameres/tests/incremental.rs | 28 +- crates/hir-expand/src/ast_id_map.rs | 18 +- crates/hir-expand/src/attrs.rs | 45 +- crates/hir-expand/src/builtin_attr_macro.rs | 6 +- crates/hir-expand/src/builtin_derive_macro.rs | 147 ++-- crates/hir-expand/src/builtin_fn_macro.rs | 39 +- crates/hir-expand/src/db.rs | 509 ++++++------- crates/hir-expand/src/eager.rs | 171 +++-- crates/hir-expand/src/hygiene.rs | 132 ++-- crates/hir-expand/src/lib.rs | 395 ++++------ crates/hir-expand/src/quote.rs | 39 +- crates/hir-ty/src/diagnostics/decl_check.rs | 2 +- crates/hir-ty/src/mir/eval.rs | 2 +- crates/hir/src/db.rs | 4 +- crates/hir/src/lib.rs | 2 +- crates/hir/src/semantics.rs | 21 +- crates/hir/src/semantics/source_to_def.rs | 2 +- crates/hir/src/source_analyzer.rs | 2 +- .../src/handlers/extract_module.rs | 2 +- .../src/handlers/fix_visibility.rs | 4 +- .../src/handlers/generate_constant.rs | 2 +- .../src/handlers/generate_enum_variant.rs | 2 +- .../src/handlers/generate_function.rs | 5 +- .../src/handlers/remove_unused_imports.rs | 2 +- .../replace_derive_with_manual_impl.rs | 2 +- crates/ide-completion/src/completions.rs | 2 +- crates/ide-completion/src/completions/mod_.rs | 2 +- crates/ide-db/src/apply_change.rs | 2 +- crates/ide-db/src/lib.rs | 2 +- crates/ide-db/src/rename.rs | 2 +- crates/ide-db/src/search.rs | 3 +- .../ide-db/src/test_data/test_doc_alias.txt | 28 +- .../test_symbol_index_collection.txt | 118 +-- .../src/handlers/missing_fields.rs | 2 +- .../src/handlers/missing_unsafe.rs | 1 + .../src/handlers/no_such_field.rs | 2 +- .../replace_filter_map_next_with_find_map.rs | 2 +- .../src/handlers/type_mismatch.rs | 2 +- .../src/handlers/unresolved_module.rs | 2 +- crates/ide/src/call_hierarchy.rs | 4 +- crates/ide/src/expand_macro.rs | 2 +- crates/ide/src/goto_definition.rs | 8 +- crates/ide/src/rename.rs | 2 +- crates/ide/src/runnables.rs | 2 +- crates/ide/src/static_index.rs | 2 +- .../ide/src/syntax_highlighting/highlight.rs | 2 +- crates/load-cargo/src/lib.rs | 38 +- crates/mbe/src/benchmark.rs | 67 +- crates/mbe/src/lib.rs | 126 +--- crates/mbe/src/syntax_bridge.rs | 701 ++++++------------ crates/mbe/src/syntax_bridge/tests.rs | 24 +- crates/mbe/src/token_map.rs | 208 +++--- crates/proc-macro-api/Cargo.toml | 1 + crates/proc-macro-api/src/msg/flat.rs | 14 + crates/rust-analyzer/src/cargo_target_spec.rs | 11 +- .../rust-analyzer/src/cli/analysis_stats.rs | 2 +- crates/rust-analyzer/src/cli/diagnostics.rs | 2 +- crates/syntax/src/lib.rs | 46 +- crates/syntax/src/parsing/reparsing.rs | 12 +- crates/syntax/src/tests.rs | 2 +- crates/tt/Cargo.toml | 1 + crates/tt/src/lib.rs | 39 +- 80 files changed, 1819 insertions(+), 2049 deletions(-) create mode 100644 crates/base-db/src/span.rs diff --git a/Cargo.lock b/Cargo.lock index 5a8d971c3d4f..90ee0810fa6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1263,6 +1263,7 @@ dependencies = [ "serde_json", "snap", "stdx", + "text-size", "tracing", "triomphe", "tt", @@ -2010,6 +2011,7 @@ version = "0.0.0" dependencies = [ "smol_str", "stdx", + "text-size", ] [[package]] diff --git a/crates/base-db/src/fixture.rs b/crates/base-db/src/fixture.rs index 3da555a47acb..7236b56f6d47 100644 --- a/crates/base-db/src/fixture.rs +++ b/crates/base-db/src/fixture.rs @@ -8,11 +8,12 @@ use test_utils::{ ESCAPED_CURSOR_MARKER, }; use triomphe::Arc; -use tt::token_id::{Leaf, Subtree, TokenTree}; +use tt::{Leaf, Subtree, TokenTree}; use vfs::{file_set::FileSet, VfsPath}; use crate::{ input::{CrateName, CrateOrigin, LangCrateOrigin}, + span::SpanData, Change, CrateDisplayName, CrateGraph, CrateId, Dependency, DependencyKind, Edition, Env, FileId, FilePosition, FileRange, ProcMacro, ProcMacroExpander, ProcMacroExpansionError, ProcMacros, ReleaseChannel, SourceDatabaseExt, SourceRoot, SourceRootId, @@ -539,10 +540,10 @@ struct IdentityProcMacroExpander; impl ProcMacroExpander for IdentityProcMacroExpander { fn expand( &self, - subtree: &Subtree, - _: Option<&Subtree>, + subtree: &Subtree, + _: Option<&Subtree>, _: &Env, - ) -> Result { + ) -> Result, ProcMacroExpansionError> { Ok(subtree.clone()) } } @@ -553,10 +554,10 @@ struct AttributeInputReplaceProcMacroExpander; impl ProcMacroExpander for AttributeInputReplaceProcMacroExpander { fn expand( &self, - _: &Subtree, - attrs: Option<&Subtree>, + _: &Subtree, + attrs: Option<&Subtree>, _: &Env, - ) -> Result { + ) -> Result, ProcMacroExpansionError> { attrs .cloned() .ok_or_else(|| ProcMacroExpansionError::Panic("Expected attribute input".into())) @@ -568,11 +569,11 @@ struct MirrorProcMacroExpander; impl ProcMacroExpander for MirrorProcMacroExpander { fn expand( &self, - input: &Subtree, - _: Option<&Subtree>, + input: &Subtree, + _: Option<&Subtree>, _: &Env, - ) -> Result { - fn traverse(input: &Subtree) -> Subtree { + ) -> Result, ProcMacroExpansionError> { + fn traverse(input: &Subtree) -> Subtree { let mut token_trees = vec![]; for tt in input.token_trees.iter().rev() { let tt = match tt { @@ -595,13 +596,13 @@ struct ShortenProcMacroExpander; impl ProcMacroExpander for ShortenProcMacroExpander { fn expand( &self, - input: &Subtree, - _: Option<&Subtree>, + input: &Subtree, + _: Option<&Subtree>, _: &Env, - ) -> Result { + ) -> Result, ProcMacroExpansionError> { return Ok(traverse(input)); - fn traverse(input: &Subtree) -> Subtree { + fn traverse(input: &Subtree) -> Subtree { let token_trees = input .token_trees .iter() @@ -613,7 +614,7 @@ impl ProcMacroExpander for ShortenProcMacroExpander { Subtree { delimiter: input.delimiter, token_trees } } - fn modify_leaf(leaf: &Leaf) -> Leaf { + fn modify_leaf(leaf: &Leaf) -> Leaf { let mut leaf = leaf.clone(); match &mut leaf { Leaf::Literal(it) => { diff --git a/crates/base-db/src/input.rs b/crates/base-db/src/input.rs index e4f78321e215..2fa5c25c91e5 100644 --- a/crates/base-db/src/input.rs +++ b/crates/base-db/src/input.rs @@ -13,9 +13,10 @@ use la_arena::{Arena, Idx}; use rustc_hash::{FxHashMap, FxHashSet}; use syntax::SmolStr; use triomphe::Arc; -use tt::token_id::Subtree; use vfs::{file_set::FileSet, AbsPathBuf, AnchoredPath, FileId, VfsPath}; +use crate::span::SpanData; + // Map from crate id to the name of the crate and path of the proc-macro. If the value is `None`, // then the crate for the proc-macro hasn't been build yet as the build data is missing. pub type ProcMacroPaths = FxHashMap, AbsPathBuf), String>>; @@ -255,10 +256,10 @@ pub enum ProcMacroKind { pub trait ProcMacroExpander: fmt::Debug + Send + Sync + RefUnwindSafe { fn expand( &self, - subtree: &Subtree, - attrs: Option<&Subtree>, + subtree: &tt::Subtree, + attrs: Option<&tt::Subtree>, env: &Env, - ) -> Result; + ) -> Result, ProcMacroExpansionError>; } #[derive(Debug)] diff --git a/crates/base-db/src/lib.rs b/crates/base-db/src/lib.rs index 40cfab88afdb..6dc1629c3be9 100644 --- a/crates/base-db/src/lib.rs +++ b/crates/base-db/src/lib.rs @@ -5,6 +5,7 @@ mod input; mod change; pub mod fixture; +pub mod span; use std::panic; diff --git a/crates/base-db/src/span.rs b/crates/base-db/src/span.rs new file mode 100644 index 000000000000..1072d937a3bd --- /dev/null +++ b/crates/base-db/src/span.rs @@ -0,0 +1,166 @@ +use std::fmt; + +use salsa::InternId; +use vfs::FileId; + +pub type ErasedFileAstId = la_arena::Idx; + +// The first inde is always the root node's AstId +pub const ROOT_ERASED_FILE_AST_ID: ErasedFileAstId = + la_arena::Idx::from_raw(la_arena::RawIdx::from_u32(0)); + +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +pub struct SyntaxContext; + +pub type SpanData = tt::SpanData; + +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub struct SpanAnchor { + pub file_id: HirFileId, + pub ast_id: ErasedFileAstId, +} + +impl fmt::Debug for SpanAnchor { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("SpanAnchor").field(&self.file_id).field(&self.ast_id.into_raw()).finish() + } +} + +impl tt::Span for SpanAnchor { + const DUMMY: Self = SpanAnchor { file_id: HirFileId(0), ast_id: ROOT_ERASED_FILE_AST_ID }; +} + +/// Input to the analyzer is a set of files, where each file is identified by +/// `FileId` and contains source code. However, another source of source code in +/// Rust are macros: each macro can be thought of as producing a "temporary +/// file". To assign an id to such a file, we use the id of the macro call that +/// produced the file. So, a `HirFileId` is either a `FileId` (source code +/// written by user), or a `MacroCallId` (source code produced by macro). +/// +/// What is a `MacroCallId`? Simplifying, it's a `HirFileId` of a file +/// containing the call plus the offset of the macro call in the file. Note that +/// this is a recursive definition! However, the size_of of `HirFileId` is +/// finite (because everything bottoms out at the real `FileId`) and small +/// (`MacroCallId` uses the location interning. You can check details here: +/// ). +/// +/// The two variants are encoded in a single u32 which are differentiated by the MSB. +/// If the MSB is 0, the value represents a `FileId`, otherwise the remaining 31 bits represent a +/// `MacroCallId`. +#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct HirFileId(u32); + +impl From for u32 { + fn from(value: HirFileId) -> Self { + value.0 + } +} + +impl From for HirFileId { + fn from(value: u32) -> Self { + HirFileId(value) + } +} + +impl From for HirFileId { + fn from(value: MacroCallId) -> Self { + value.as_file() + } +} + +impl fmt::Debug for HirFileId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.repr().fmt(f) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct MacroFile { + pub macro_call_id: MacroCallId, +} + +/// `MacroCallId` identifies a particular macro invocation, like +/// `println!("Hello, {}", world)`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct MacroCallId(salsa::InternId); +crate::impl_intern_key!(MacroCallId); + +impl MacroCallId { + pub fn as_file(self) -> HirFileId { + MacroFile { macro_call_id: self }.into() + } + + pub fn as_macro_file(self) -> MacroFile { + MacroFile { macro_call_id: self } + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub enum HirFileIdRepr { + FileId(FileId), + MacroFile(MacroFile), +} + +impl fmt::Debug for HirFileIdRepr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::FileId(arg0) => f.debug_tuple("FileId").field(&arg0.0).finish(), + Self::MacroFile(arg0) => { + f.debug_tuple("MacroFile").field(&arg0.macro_call_id.0).finish() + } + } + } +} + +impl From for HirFileId { + fn from(FileId(id): FileId) -> Self { + assert!(id < Self::MAX_FILE_ID); + HirFileId(id) + } +} + +impl From for HirFileId { + fn from(MacroFile { macro_call_id: MacroCallId(id) }: MacroFile) -> Self { + let id = id.as_u32(); + assert!(id < Self::MAX_FILE_ID); + HirFileId(id | Self::MACRO_FILE_TAG_MASK) + } +} + +impl HirFileId { + const MAX_FILE_ID: u32 = u32::MAX ^ Self::MACRO_FILE_TAG_MASK; + const MACRO_FILE_TAG_MASK: u32 = 1 << 31; + + #[inline] + pub fn is_macro(self) -> bool { + self.0 & Self::MACRO_FILE_TAG_MASK != 0 + } + + #[inline] + pub fn macro_file(self) -> Option { + match self.0 & Self::MACRO_FILE_TAG_MASK { + 0 => None, + _ => Some(MacroFile { + macro_call_id: MacroCallId(InternId::from(self.0 ^ Self::MACRO_FILE_TAG_MASK)), + }), + } + } + + #[inline] + pub fn file_id(self) -> Option { + match self.0 & Self::MACRO_FILE_TAG_MASK { + 0 => Some(FileId(self.0)), + _ => None, + } + } + + #[inline] + pub fn repr(self) -> HirFileIdRepr { + match self.0 & Self::MACRO_FILE_TAG_MASK { + 0 => HirFileIdRepr::FileId(FileId(self.0)), + _ => HirFileIdRepr::MacroFile(MacroFile { + macro_call_id: MacroCallId(InternId::from(self.0 ^ Self::MACRO_FILE_TAG_MASK)), + }), + } + } +} diff --git a/crates/cfg/src/tests.rs b/crates/cfg/src/tests.rs index bdc3f854e086..242929c006a5 100644 --- a/crates/cfg/src/tests.rs +++ b/crates/cfg/src/tests.rs @@ -2,36 +2,37 @@ use arbitrary::{Arbitrary, Unstructured}; use expect_test::{expect, Expect}; use mbe::syntax_node_to_token_tree; use syntax::{ast, AstNode}; +use tt::Span; use crate::{CfgAtom, CfgExpr, CfgOptions, DnfExpr}; +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +struct DummyFile; +impl Span for DummyFile { + const DUMMY: Self = DummyFile; +} + fn assert_parse_result(input: &str, expected: CfgExpr) { - let (tt, _) = { - let source_file = ast::SourceFile::parse(input).ok().unwrap(); - let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - syntax_node_to_token_tree(tt.syntax()) - }; + let source_file = ast::SourceFile::parse(input).ok().unwrap(); + let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); + let tt = syntax_node_to_token_tree(tt.syntax(), DummyFile, 0.into(), &Default::default()); let cfg = CfgExpr::parse(&tt); assert_eq!(cfg, expected); } fn check_dnf(input: &str, expect: Expect) { - let (tt, _) = { - let source_file = ast::SourceFile::parse(input).ok().unwrap(); - let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - syntax_node_to_token_tree(tt.syntax()) - }; + let source_file = ast::SourceFile::parse(input).ok().unwrap(); + let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); + let tt = syntax_node_to_token_tree(tt.syntax(), DummyFile, 0.into(), &Default::default()); let cfg = CfgExpr::parse(&tt); let actual = format!("#![cfg({})]", DnfExpr::new(cfg)); expect.assert_eq(&actual); } fn check_why_inactive(input: &str, opts: &CfgOptions, expect: Expect) { - let (tt, _) = { - let source_file = ast::SourceFile::parse(input).ok().unwrap(); - let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - syntax_node_to_token_tree(tt.syntax()) - }; + let source_file = ast::SourceFile::parse(input).ok().unwrap(); + let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); + let tt = syntax_node_to_token_tree(tt.syntax(), DummyFile, 0.into(), &Default::default()); let cfg = CfgExpr::parse(&tt); let dnf = DnfExpr::new(cfg); let why_inactive = dnf.why_inactive(opts).unwrap().to_string(); @@ -40,11 +41,9 @@ fn check_why_inactive(input: &str, opts: &CfgOptions, expect: Expect) { #[track_caller] fn check_enable_hints(input: &str, opts: &CfgOptions, expected_hints: &[&str]) { - let (tt, _) = { - let source_file = ast::SourceFile::parse(input).ok().unwrap(); - let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - syntax_node_to_token_tree(tt.syntax()) - }; + let source_file = ast::SourceFile::parse(input).ok().unwrap(); + let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); + let tt = syntax_node_to_token_tree(tt.syntax(), DummyFile, 0.into(), &Default::default()); let cfg = CfgExpr::parse(&tt); let dnf = DnfExpr::new(cfg); let hints = dnf.compute_enable_hints(opts).map(|diff| diff.to_string()).collect::>(); diff --git a/crates/hir-def/src/attr.rs b/crates/hir-def/src/attr.rs index fa3025e0303d..35c9d63979a0 100644 --- a/crates/hir-def/src/attr.rs +++ b/crates/hir-def/src/attr.rs @@ -7,7 +7,10 @@ mod tests; use std::{hash::Hash, ops, slice::Iter as SliceIter}; -use base_db::CrateId; +use base_db::{ + span::{ErasedFileAstId, SpanAnchor}, + CrateId, +}; use cfg::{CfgExpr, CfgOptions}; use either::Either; use hir_expand::{ @@ -28,8 +31,8 @@ use crate::{ lang_item::LangItem, nameres::{ModuleOrigin, ModuleSource}, src::{HasChildSource, HasSource}, - AdtId, AssocItemLoc, AttrDefId, EnumId, GenericParamId, ItemLoc, LocalEnumVariantId, - LocalFieldId, Lookup, MacroId, VariantId, + AdtId, AssocItemLoc, AttrDefId, EnumId, GenericDefId, GenericParamId, ItemLoc, + LocalEnumVariantId, LocalFieldId, Lookup, MacroId, VariantId, }; #[derive(Default, Debug, Clone, PartialEq, Eq)] @@ -415,26 +418,48 @@ impl AttrsWithOwner { AttrDefId::StaticId(it) => attrs_from_item_tree_assoc(db, it), AttrDefId::FunctionId(it) => attrs_from_item_tree_assoc(db, it), AttrDefId::TypeAliasId(it) => attrs_from_item_tree_assoc(db, it), - AttrDefId::GenericParamId(it) => match it { - GenericParamId::ConstParamId(it) => { - let src = it.parent().child_source(db); - RawAttrs::from_attrs_owner( - db.upcast(), - src.with_value(&src.value[it.local_id()]), - ) - } - GenericParamId::TypeParamId(it) => { - let src = it.parent().child_source(db); - RawAttrs::from_attrs_owner( - db.upcast(), - src.with_value(&src.value[it.local_id()]), - ) - } - GenericParamId::LifetimeParamId(it) => { - let src = it.parent.child_source(db); - RawAttrs::from_attrs_owner(db.upcast(), src.with_value(&src.value[it.local_id])) + AttrDefId::GenericParamId(it) => { + let ast_id = |p| match p { + GenericDefId::AdtId(AdtId::StructId(it)) => { + erased_ast_id_from_item_tree(db, it) + } + GenericDefId::AdtId(AdtId::EnumId(it)) => erased_ast_id_from_item_tree(db, it), + GenericDefId::AdtId(AdtId::UnionId(it)) => erased_ast_id_from_item_tree(db, it), + GenericDefId::TraitId(it) => erased_ast_id_from_item_tree(db, it), + GenericDefId::TraitAliasId(it) => erased_ast_id_from_item_tree(db, it), + GenericDefId::ImplId(it) => erased_ast_id_from_item_tree(db, it), + GenericDefId::EnumVariantId(it) => erased_ast_id_from_item_tree(db, it.parent), + GenericDefId::TypeAliasId(it) => erased_ast_id_from_item_tree_assoc(db, it), + GenericDefId::FunctionId(it) => erased_ast_id_from_item_tree_assoc(db, it), + GenericDefId::ConstId(it) => erased_ast_id_from_item_tree_assoc(db, it), + }; + match it { + GenericParamId::ConstParamId(it) => { + let src = it.parent().child_source(db); + RawAttrs::from_attrs_owner( + db.upcast(), + SpanAnchor { file_id: src.file_id, ast_id: ast_id(it.parent()) }, + src.with_value(&src.value[it.local_id()]), + ) + } + GenericParamId::TypeParamId(it) => { + let src = it.parent().child_source(db); + RawAttrs::from_attrs_owner( + db.upcast(), + SpanAnchor { file_id: src.file_id, ast_id: ast_id(it.parent()) }, + src.with_value(&src.value[it.local_id()]), + ) + } + GenericParamId::LifetimeParamId(it) => { + let src = it.parent.child_source(db); + RawAttrs::from_attrs_owner( + db.upcast(), + SpanAnchor { file_id: src.file_id, ast_id: ast_id(it.parent) }, + src.with_value(&src.value[it.local_id]), + ) + } } - }, + } AttrDefId::ExternBlockId(it) => attrs_from_item_tree_loc(db, it), AttrDefId::ExternCrateId(it) => attrs_from_item_tree_loc(db, it), AttrDefId::UseId(it) => attrs_from_item_tree_loc(db, it), @@ -638,6 +663,26 @@ fn any_has_attrs( id.lookup(db).source(db).map(ast::AnyHasAttrs::new) } +fn erased_ast_id_from_item_tree( + db: &dyn DefDatabase, + lookup: impl Lookup>, +) -> ErasedFileAstId { + let id = lookup.lookup(db).id; + let tree = id.item_tree(db); + let mod_item = N::id_to_mod_item(id.value); + mod_item.ast_id(&tree).erase() +} + +fn erased_ast_id_from_item_tree_assoc( + db: &dyn DefDatabase, + lookup: impl Lookup>, +) -> ErasedFileAstId { + let id = lookup.lookup(db).id; + let tree = id.item_tree(db); + let mod_item = N::id_to_mod_item(id.value); + mod_item.ast_id(&tree).erase() +} + fn attrs_from_item_tree(db: &dyn DefDatabase, id: ItemTreeId) -> RawAttrs { let tree = id.item_tree(db); let mod_item = N::id_to_mod_item(id.value); diff --git a/crates/hir-def/src/attr/tests.rs b/crates/hir-def/src/attr/tests.rs index e4c8d446af7b..ad101e9bdf77 100644 --- a/crates/hir-def/src/attr/tests.rs +++ b/crates/hir-def/src/attr/tests.rs @@ -1,17 +1,18 @@ //! This module contains tests for doc-expression parsing. //! Currently, it tests `#[doc(hidden)]` and `#[doc(alias)]`. +use base_db::span::SpanAnchor; use mbe::syntax_node_to_token_tree; use syntax::{ast, AstNode}; +use tt::Span; use crate::attr::{DocAtom, DocExpr}; fn assert_parse_result(input: &str, expected: DocExpr) { - let (tt, _) = { - let source_file = ast::SourceFile::parse(input).ok().unwrap(); - let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - syntax_node_to_token_tree(tt.syntax()) - }; + let source_file = ast::SourceFile::parse(input).ok().unwrap(); + let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); + let tt = + syntax_node_to_token_tree(tt.syntax(), SpanAnchor::DUMMY, 0.into(), &Default::default()); let cfg = DocExpr::parse(&tt); assert_eq!(cfg, expected); } diff --git a/crates/hir-def/src/data.rs b/crates/hir-def/src/data.rs index 72ccc17486f0..4083d497917c 100644 --- a/crates/hir-def/src/data.rs +++ b/crates/hir-def/src/data.rs @@ -663,7 +663,7 @@ impl<'a> AssocItemCollector<'a> { self.module_id.local_id, MacroCallKind::Attr { ast_id, - attr_args: Arc::new((tt::Subtree::empty(), Default::default())), + attr_args: Arc::new(tt::Subtree::empty()), invoc_attr_index: attr.id, }, attr.path().clone(), diff --git a/crates/hir-def/src/expander.rs b/crates/hir-def/src/expander.rs index 6db8398bc986..b7cffb57625e 100644 --- a/crates/hir-def/src/expander.rs +++ b/crates/hir-def/src/expander.rs @@ -1,6 +1,9 @@ //! Macro expansion utilities. -use base_db::CrateId; +use base_db::{ + span::{SpanAnchor, ROOT_ERASED_FILE_AST_ID}, + CrateId, +}; use cfg::CfgOptions; use drop_bomb::DropBomb; use hir_expand::{ @@ -118,7 +121,17 @@ impl Expander { } pub(crate) fn parse_attrs(&self, db: &dyn DefDatabase, owner: &dyn ast::HasAttrs) -> Attrs { - Attrs::filter(db, self.krate, RawAttrs::new(db.upcast(), owner, &self.hygiene)) + Attrs::filter( + db, + self.krate, + RawAttrs::new( + db.upcast(), + // Usin `ROOT_ERASED_FILE_AST_ID` here is fine as this is only used for cfg checking + SpanAnchor { file_id: self.current_file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, + owner, + &self.hygiene, + ), + ) } pub(crate) fn cfg_options(&self) -> &CfgOptions { diff --git a/crates/hir-def/src/generics.rs b/crates/hir-def/src/generics.rs index fac90e663045..6d656bf9482f 100644 --- a/crates/hir-def/src/generics.rs +++ b/crates/hir-def/src/generics.rs @@ -21,7 +21,7 @@ use crate::{ db::DefDatabase, dyn_map::{keys, DynMap}, expander::Expander, - item_tree::{AttrOwner, ItemTree}, + item_tree::ItemTree, lower::LowerCtx, nameres::{DefMap, MacroSubNs}, src::{HasChildSource, HasSource}, @@ -250,7 +250,10 @@ impl GenericParams { &mut self, lower_ctx: &LowerCtx<'_>, node: &dyn HasGenericParams, - add_param_attrs: impl FnMut(AttrOwner, ast::GenericParam), + add_param_attrs: impl FnMut( + Either, Idx>, + ast::GenericParam, + ), ) { if let Some(params) = node.generic_param_list() { self.fill_params(lower_ctx, params, add_param_attrs) @@ -275,7 +278,10 @@ impl GenericParams { &mut self, lower_ctx: &LowerCtx<'_>, params: ast::GenericParamList, - mut add_param_attrs: impl FnMut(AttrOwner, ast::GenericParam), + mut add_param_attrs: impl FnMut( + Either, Idx>, + ast::GenericParam, + ), ) { for type_or_const_param in params.type_or_const_params() { match type_or_const_param { @@ -297,7 +303,7 @@ impl GenericParams { type_param.type_bound_list(), Either::Left(type_ref), ); - add_param_attrs(idx.into(), ast::GenericParam::TypeParam(type_param)); + add_param_attrs(Either::Left(idx), ast::GenericParam::TypeParam(type_param)); } ast::TypeOrConstParam::Const(const_param) => { let name = const_param.name().map_or_else(Name::missing, |it| it.as_name()); @@ -310,7 +316,7 @@ impl GenericParams { default: ConstRef::from_const_param(lower_ctx, &const_param), }; let idx = self.type_or_consts.alloc(param.into()); - add_param_attrs(idx.into(), ast::GenericParam::ConstParam(const_param)); + add_param_attrs(Either::Left(idx), ast::GenericParam::ConstParam(const_param)); } } } @@ -325,7 +331,7 @@ impl GenericParams { lifetime_param.type_bound_list(), Either::Right(lifetime_ref), ); - add_param_attrs(idx.into(), ast::GenericParam::LifetimeParam(lifetime_param)); + add_param_attrs(Either::Right(idx), ast::GenericParam::LifetimeParam(lifetime_param)); } } diff --git a/crates/hir-def/src/item_tree.rs b/crates/hir-def/src/item_tree.rs index 70b96b25739c..3bea91ee61c3 100644 --- a/crates/hir-def/src/item_tree.rs +++ b/crates/hir-def/src/item_tree.rs @@ -43,7 +43,10 @@ use std::{ }; use ast::{AstNode, HasName, StructKind}; -use base_db::CrateId; +use base_db::{ + span::{SpanAnchor, ROOT_ERASED_FILE_AST_ID}, + CrateId, +}; use either::Either; use hir_expand::{ ast_id_map::{AstIdNode, FileAstId}, @@ -119,7 +122,7 @@ impl ItemTree { let mut item_tree = match_ast! { match syntax { ast::SourceFile(file) => { - top_attrs = Some(RawAttrs::new(db.upcast(), &file, ctx.hygiene())); + top_attrs = Some(RawAttrs::new(db.upcast(), SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, &file, ctx.hygiene())); ctx.lower_module_items(&file) }, ast::MacroItems(items) => { diff --git a/crates/hir-def/src/item_tree/lower.rs b/crates/hir-def/src/item_tree/lower.rs index c898eb5f9211..807b2a7bf752 100644 --- a/crates/hir-def/src/item_tree/lower.rs +++ b/crates/hir-def/src/item_tree/lower.rs @@ -2,12 +2,14 @@ use std::collections::hash_map::Entry; +use base_db::span::ErasedFileAstId; use hir_expand::{ast_id_map::AstIdMap, hygiene::Hygiene, HirFileId}; use syntax::ast::{self, HasModuleItem, HasTypeBounds}; use crate::{ generics::{GenericParams, TypeParamData, TypeParamProvenance}, type_ref::{LifetimeRef, TraitBoundModifier, TraitRef}, + LocalLifetimeParamId, LocalTypeOrConstParamId, }; use super::*; @@ -21,6 +23,7 @@ pub(super) struct Ctx<'a> { tree: ItemTree, source_ast_id_map: Arc, body_ctx: crate::lower::LowerCtx<'a>, + file: HirFileId, } impl<'a> Ctx<'a> { @@ -30,6 +33,7 @@ impl<'a> Ctx<'a> { tree: ItemTree::default(), source_ast_id_map: db.ast_id_map(file), body_ctx: crate::lower::LowerCtx::with_file_id(db, file), + file, } } @@ -77,9 +81,18 @@ impl<'a> Ctx<'a> { } pub(super) fn lower_block(mut self, block: &ast::BlockExpr) -> ItemTree { - self.tree - .attrs - .insert(AttrOwner::TopLevel, RawAttrs::new(self.db.upcast(), block, self.hygiene())); + self.tree.attrs.insert( + AttrOwner::TopLevel, + RawAttrs::new( + self.db.upcast(), + SpanAnchor { + file_id: self.file, + ast_id: self.source_ast_id_map.ast_id(block).erase(), + }, + block, + self.hygiene(), + ), + ); self.tree.top_level = block .statements() .filter_map(|stmt| match stmt { @@ -109,8 +122,7 @@ impl<'a> Ctx<'a> { } fn lower_mod_item(&mut self, item: &ast::Item) -> Option { - let attrs = RawAttrs::new(self.db.upcast(), item, self.hygiene()); - let item: ModItem = match item { + let mod_item: ModItem = match item { ast::Item::Struct(ast) => self.lower_struct(ast)?.into(), ast::Item::Union(ast) => self.lower_union(ast)?.into(), ast::Item::Enum(ast) => self.lower_enum(ast)?.into(), @@ -129,10 +141,15 @@ impl<'a> Ctx<'a> { ast::Item::MacroDef(ast) => self.lower_macro_def(ast)?.into(), ast::Item::ExternBlock(ast) => self.lower_extern_block(ast).into(), }; + let attrs = RawAttrs::new( + self.db.upcast(), + SpanAnchor { file_id: self.file, ast_id: mod_item.ast_id(&self.tree).erase() }, + item, + self.hygiene(), + ); + self.add_attrs(mod_item.into(), attrs); - self.add_attrs(item.into(), attrs); - - Some(item) + Some(mod_item) } fn add_attrs(&mut self, item: AttrOwner, attrs: RawAttrs) { @@ -146,21 +163,37 @@ impl<'a> Ctx<'a> { } } - fn lower_assoc_item(&mut self, item: &ast::AssocItem) -> Option { - match item { + fn lower_assoc_item(&mut self, item_node: &ast::AssocItem) -> Option { + let item: AssocItem = match item_node { ast::AssocItem::Fn(ast) => self.lower_function(ast).map(Into::into), ast::AssocItem::TypeAlias(ast) => self.lower_type_alias(ast).map(Into::into), ast::AssocItem::Const(ast) => Some(self.lower_const(ast).into()), ast::AssocItem::MacroCall(ast) => self.lower_macro_call(ast).map(Into::into), - } + }?; + let attrs = RawAttrs::new( + self.db.upcast(), + SpanAnchor { file_id: self.file, ast_id: item.ast_id(&self.tree).erase() }, + item_node, + self.hygiene(), + ); + self.add_attrs( + match item { + AssocItem::Function(it) => AttrOwner::ModItem(ModItem::Function(it)), + AssocItem::TypeAlias(it) => AttrOwner::ModItem(ModItem::TypeAlias(it)), + AssocItem::Const(it) => AttrOwner::ModItem(ModItem::Const(it)), + AssocItem::MacroCall(it) => AttrOwner::ModItem(ModItem::MacroCall(it)), + }, + attrs, + ); + Some(item) } fn lower_struct(&mut self, strukt: &ast::Struct) -> Option> { let visibility = self.lower_visibility(strukt); let name = strukt.name()?.as_name(); - let generic_params = self.lower_generic_params(HasImplicitSelf::No, strukt); - let fields = self.lower_fields(&strukt.kind()); let ast_id = self.source_ast_id_map.ast_id(strukt); + let generic_params = self.lower_generic_params(HasImplicitSelf::No, strukt, ast_id.erase()); + let fields = self.lower_fields(&strukt.kind()); let res = Struct { name, visibility, generic_params, fields, ast_id }; Some(id(self.data().structs.alloc(res))) } @@ -183,8 +216,20 @@ impl<'a> Ctx<'a> { let start = self.next_field_idx(); for field in fields.fields() { if let Some(data) = self.lower_record_field(&field) { + let ast_id = match data.ast_id { + FieldAstId::Record(it) => it.erase(), + FieldAstId::Tuple(it) => it.erase(), + }; let idx = self.data().fields.alloc(data); - self.add_attrs(idx.into(), RawAttrs::new(self.db.upcast(), &field, self.hygiene())); + self.add_attrs( + idx.into(), + RawAttrs::new( + self.db.upcast(), + SpanAnchor { file_id: self.file, ast_id }, + &field, + self.hygiene(), + ), + ); } } let end = self.next_field_idx(); @@ -204,8 +249,20 @@ impl<'a> Ctx<'a> { let start = self.next_field_idx(); for (i, field) in fields.fields().enumerate() { let data = self.lower_tuple_field(i, &field); + let ast_id = match data.ast_id { + FieldAstId::Record(it) => it.erase(), + FieldAstId::Tuple(it) => it.erase(), + }; let idx = self.data().fields.alloc(data); - self.add_attrs(idx.into(), RawAttrs::new(self.db.upcast(), &field, self.hygiene())); + self.add_attrs( + idx.into(), + RawAttrs::new( + self.db.upcast(), + SpanAnchor { file_id: self.file, ast_id }, + &field, + self.hygiene(), + ), + ); } let end = self.next_field_idx(); IdxRange::new(start..end) @@ -222,12 +279,12 @@ impl<'a> Ctx<'a> { fn lower_union(&mut self, union: &ast::Union) -> Option> { let visibility = self.lower_visibility(union); let name = union.name()?.as_name(); - let generic_params = self.lower_generic_params(HasImplicitSelf::No, union); + let ast_id = self.source_ast_id_map.ast_id(union); + let generic_params = self.lower_generic_params(HasImplicitSelf::No, union, ast_id.erase()); let fields = match union.record_field_list() { Some(record_field_list) => self.lower_fields(&StructKind::Record(record_field_list)), None => Fields::Record(IdxRange::new(self.next_field_idx()..self.next_field_idx())), }; - let ast_id = self.source_ast_id_map.ast_id(union); let res = Union { name, visibility, generic_params, fields, ast_id }; Some(id(self.data().unions.alloc(res))) } @@ -235,12 +292,12 @@ impl<'a> Ctx<'a> { fn lower_enum(&mut self, enum_: &ast::Enum) -> Option> { let visibility = self.lower_visibility(enum_); let name = enum_.name()?.as_name(); - let generic_params = self.lower_generic_params(HasImplicitSelf::No, enum_); + let ast_id = self.source_ast_id_map.ast_id(enum_); + let generic_params = self.lower_generic_params(HasImplicitSelf::No, enum_, ast_id.erase()); let variants = match &enum_.variant_list() { Some(variant_list) => self.lower_variants(variant_list), None => IdxRange::new(self.next_variant_idx()..self.next_variant_idx()), }; - let ast_id = self.source_ast_id_map.ast_id(enum_); let res = Enum { name, visibility, generic_params, variants, ast_id }; Some(id(self.data().enums.alloc(res))) } @@ -249,10 +306,16 @@ impl<'a> Ctx<'a> { let start = self.next_variant_idx(); for variant in variants.variants() { if let Some(data) = self.lower_variant(&variant) { + let ast_id = data.ast_id.erase(); let idx = self.data().variants.alloc(data); self.add_attrs( idx.into(), - RawAttrs::new(self.db.upcast(), &variant, self.hygiene()), + RawAttrs::new( + self.db.upcast(), + SpanAnchor { file_id: self.file, ast_id }, + &variant, + self.hygiene(), + ), ); } } @@ -303,28 +366,39 @@ impl<'a> Ctx<'a> { }); self.add_attrs( idx.into(), - RawAttrs::new(self.db.upcast(), &self_param, self.hygiene()), + RawAttrs::new( + self.db.upcast(), + SpanAnchor { file_id: self.file, ast_id: ast_id.erase() }, + &self_param, + self.hygiene(), + ), ); has_self_param = true; } for param in param_list.params() { + let ast_id = self.source_ast_id_map.ast_id(¶m); let idx = match param.dotdotdot_token() { - Some(_) => { - let ast_id = self.source_ast_id_map.ast_id(¶m); - self.data() - .params - .alloc(Param { type_ref: None, ast_id: ParamAstId::Param(ast_id) }) - } + Some(_) => self + .data() + .params + .alloc(Param { type_ref: None, ast_id: ParamAstId::Param(ast_id) }), None => { let type_ref = TypeRef::from_ast_opt(&self.body_ctx, param.ty()); let ty = Interned::new(type_ref); - let ast_id = self.source_ast_id_map.ast_id(¶m); self.data() .params .alloc(Param { type_ref: Some(ty), ast_id: ParamAstId::Param(ast_id) }) } }; - self.add_attrs(idx.into(), RawAttrs::new(self.db.upcast(), ¶m, self.hygiene())); + self.add_attrs( + idx.into(), + RawAttrs::new( + self.db.upcast(), + SpanAnchor { file_id: self.file, ast_id: ast_id.erase() }, + ¶m, + self.hygiene(), + ), + ); } } let end_param = self.next_param_idx(); @@ -381,7 +455,8 @@ impl<'a> Ctx<'a> { ast_id, flags, }; - res.explicit_generic_params = self.lower_generic_params(HasImplicitSelf::No, func); + res.explicit_generic_params = + self.lower_generic_params(HasImplicitSelf::No, func, ast_id.erase()); Some(id(self.data().functions.alloc(res))) } @@ -394,8 +469,9 @@ impl<'a> Ctx<'a> { let type_ref = type_alias.ty().map(|it| self.lower_type_ref(&it)); let visibility = self.lower_visibility(type_alias); let bounds = self.lower_type_bounds(type_alias); - let generic_params = self.lower_generic_params(HasImplicitSelf::No, type_alias); let ast_id = self.source_ast_id_map.ast_id(type_alias); + let generic_params = + self.lower_generic_params(HasImplicitSelf::No, type_alias, ast_id.erase()); let res = TypeAlias { name, visibility, bounds, generic_params, type_ref, ast_id }; Some(id(self.data().type_aliases.alloc(res))) } @@ -443,23 +519,20 @@ impl<'a> Ctx<'a> { fn lower_trait(&mut self, trait_def: &ast::Trait) -> Option> { let name = trait_def.name()?.as_name(); let visibility = self.lower_visibility(trait_def); - let generic_params = - self.lower_generic_params(HasImplicitSelf::Yes(trait_def.type_bound_list()), trait_def); + let ast_id = self.source_ast_id_map.ast_id(trait_def); + let generic_params = self.lower_generic_params( + HasImplicitSelf::Yes(trait_def.type_bound_list()), + trait_def, + ast_id.erase(), + ); let is_auto = trait_def.auto_token().is_some(); let is_unsafe = trait_def.unsafe_token().is_some(); - let ast_id = self.source_ast_id_map.ast_id(trait_def); let items = trait_def .assoc_item_list() .into_iter() .flat_map(|list| list.assoc_items()) - .filter_map(|item| { - let attrs = RawAttrs::new(self.db.upcast(), &item, self.hygiene()); - self.lower_assoc_item(&item).map(|item| { - self.add_attrs(ModItem::from(item).into(), attrs); - item - }) - }) + .filter_map(|item_node| self.lower_assoc_item(&item_node)) .collect(); let def = Trait { name, visibility, generic_params, is_auto, is_unsafe, items, ast_id }; @@ -472,20 +545,23 @@ impl<'a> Ctx<'a> { ) -> Option> { let name = trait_alias_def.name()?.as_name(); let visibility = self.lower_visibility(trait_alias_def); + let ast_id = self.source_ast_id_map.ast_id(trait_alias_def); let generic_params = self.lower_generic_params( HasImplicitSelf::Yes(trait_alias_def.type_bound_list()), trait_alias_def, + ast_id.erase(), ); - let ast_id = self.source_ast_id_map.ast_id(trait_alias_def); let alias = TraitAlias { name, visibility, generic_params, ast_id }; Some(id(self.data().trait_aliases.alloc(alias))) } fn lower_impl(&mut self, impl_def: &ast::Impl) -> Option> { + let ast_id = self.source_ast_id_map.ast_id(impl_def); // Note that trait impls don't get implicit `Self` unlike traits, because here they are a // type alias rather than a type parameter, so this is handled by the resolver. - let generic_params = self.lower_generic_params(HasImplicitSelf::No, impl_def); + let generic_params = + self.lower_generic_params(HasImplicitSelf::No, impl_def, ast_id.erase()); // FIXME: If trait lowering fails, due to a non PathType for example, we treat this impl // as if it was an non-trait impl. Ideally we want to create a unique missing ref that only // equals itself. @@ -499,14 +575,8 @@ impl<'a> Ctx<'a> { .assoc_item_list() .into_iter() .flat_map(|it| it.assoc_items()) - .filter_map(|item| { - let assoc = self.lower_assoc_item(&item)?; - let attrs = RawAttrs::new(self.db.upcast(), &item, self.hygiene()); - self.add_attrs(ModItem::from(assoc).into(), attrs); - Some(assoc) - }) + .filter_map(|item| self.lower_assoc_item(&item)) .collect(); - let ast_id = self.source_ast_id_map.ast_id(impl_def); let res = Impl { generic_params, target_trait, self_ty, is_negative, is_unsafe, items, ast_id }; Some(id(self.data().impls.alloc(res))) @@ -572,15 +642,23 @@ impl<'a> Ctx<'a> { // (in other words, the knowledge that they're in an extern block must not be used). // This is because an extern block can contain macros whose ItemTree's top-level items // should be considered to be in an extern block too. - let attrs = RawAttrs::new(self.db.upcast(), &item, self.hygiene()); - let id: ModItem = match item { - ast::ExternItem::Fn(ast) => self.lower_function(&ast)?.into(), - ast::ExternItem::Static(ast) => self.lower_static(&ast)?.into(), - ast::ExternItem::TypeAlias(ty) => self.lower_type_alias(&ty)?.into(), - ast::ExternItem::MacroCall(call) => self.lower_macro_call(&call)?.into(), + let mod_item: ModItem = match &item { + ast::ExternItem::Fn(ast) => self.lower_function(ast)?.into(), + ast::ExternItem::Static(ast) => self.lower_static(ast)?.into(), + ast::ExternItem::TypeAlias(ty) => self.lower_type_alias(ty)?.into(), + ast::ExternItem::MacroCall(call) => self.lower_macro_call(call)?.into(), }; - self.add_attrs(id.into(), attrs); - Some(id) + let attrs = RawAttrs::new( + self.db.upcast(), + SpanAnchor { + file_id: self.file, + ast_id: mod_item.ast_id(&self.tree).erase(), + }, + &item, + self.hygiene(), + ); + self.add_attrs(mod_item.into(), attrs); + Some(mod_item) }) .collect() }); @@ -593,6 +671,7 @@ impl<'a> Ctx<'a> { &mut self, has_implicit_self: HasImplicitSelf, node: &dyn ast::HasGenericParams, + owner_ast_id: ErasedFileAstId, ) -> Interned { let mut generics = GenericParams::default(); @@ -612,12 +691,21 @@ impl<'a> Ctx<'a> { generics.fill_bounds(&self.body_ctx, bounds, Either::Left(self_param)); } - let add_param_attrs = |item, param| { - let attrs = RawAttrs::new(self.db.upcast(), ¶m, self.body_ctx.hygiene()); + let add_param_attrs = |item: Either, + param| { + let attrs = RawAttrs::new( + self.db.upcast(), + SpanAnchor { file_id: self.file, ast_id: owner_ast_id }, + ¶m, + self.body_ctx.hygiene(), + ); // This is identical to the body of `Ctx::add_attrs()` but we can't call that here // because it requires `&mut self` and the call to `generics.fill()` below also // references `self`. - match self.tree.attrs.entry(item) { + match self.tree.attrs.entry(match item { + Either::Right(id) => id.into(), + Either::Left(id) => id.into(), + }) { Entry::Occupied(mut entry) => { *entry.get_mut() = entry.get().merge(attrs); } diff --git a/crates/hir-def/src/lib.rs b/crates/hir-def/src/lib.rs index fd8f64d6705b..5f09d7c48134 100644 --- a/crates/hir-def/src/lib.rs +++ b/crates/hir-def/src/lib.rs @@ -84,7 +84,7 @@ use nameres::DefMap; use stdx::impl_from; use syntax::ast; -use ::tt::token_id as tt; +pub use hir_expand::tt; use crate::{ builtin_type::BuiltinType, @@ -1341,15 +1341,13 @@ fn attr_macro_as_call_id( def: MacroDefId, ) -> MacroCallId { let arg = match macro_attr.input.as_deref() { - Some(AttrInput::TokenTree(tt)) => ( - { - let mut tt = tt.0.clone(); - tt.delimiter = tt::Delimiter::UNSPECIFIED; - tt - }, - tt.1.clone(), - ), - _ => (tt::Subtree::empty(), Default::default()), + Some(AttrInput::TokenTree(tt)) => { + let mut tt = tt.as_ref().clone(); + tt.delimiter = tt::Delimiter::UNSPECIFIED; + tt + } + + _ => tt::Subtree::empty(), }; def.as_lazy_macro( diff --git a/crates/hir-def/src/macro_expansion_tests/mbe.rs b/crates/hir-def/src/macro_expansion_tests/mbe.rs index d0906213243d..b5d70052a89e 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe.rs @@ -15,7 +15,6 @@ use crate::macro_expansion_tests::check; fn token_mapping_smoke_test() { check( r#" -// +tokenids macro_rules! f { ( struct $ident:ident ) => { struct $ident { @@ -27,23 +26,19 @@ macro_rules! f { // +tokenids f!(struct MyTraitMap2); "#, - expect![[r##" -// call ids will be shifted by Shift(30) -// +tokenids -macro_rules! f {#0 - (#1 struct#2 $#3ident#4:#5ident#6 )#1 =#7>#8 {#9 - struct#10 $#11ident#12 {#13 - map#14:#15 :#16:#17std#18:#19:#20collections#21:#22:#23HashSet#24<#25(#26)#26>#27,#28 - }#13 - }#9;#29 -}#0 - -// // +tokenids -// f!(struct#1 MyTraitMap2#2); -struct#10 MyTraitMap2#32 {#13 - map#14:#15 ::std#18::collections#21::HashSet#24<#25(#26)#26>#27,#28 -}#13 -"##]], + expect![[r#" +macro_rules! f { + ( struct $ident:ident ) => { + struct $ident { + map: ::std::collections::HashSet<()>, + } + }; +} + +struct#SpanAnchor(FileId(0), 1)@58..64 MyTraitMap2#SpanAnchor(FileId(0), 2)@23..34 {#SpanAnchor(FileId(0), 1)@72..73 + map#SpanAnchor(FileId(0), 1)@86..89:#SpanAnchor(FileId(0), 1)@89..90 ::std#SpanAnchor(FileId(0), 1)@93..96::collections#SpanAnchor(FileId(0), 1)@98..109::HashSet#SpanAnchor(FileId(0), 1)@111..118<#SpanAnchor(FileId(0), 1)@118..119(#SpanAnchor(FileId(0), 1)@119..120)#SpanAnchor(FileId(0), 1)@120..121>#SpanAnchor(FileId(0), 1)@121..122,#SpanAnchor(FileId(0), 1)@122..123 +}#SpanAnchor(FileId(0), 1)@132..133 +"#]], ); } @@ -71,31 +66,22 @@ f! { "#, - expect![[r##" -// call ids will be shifted by Shift(18) + expect![[r#" // +tokenids -macro_rules! f {#0 - (#1$#2(#3$#4tt#5:#6tt#7)#3*#8)#1 =#9>#10 {#11 - $#12(#13$#14tt#15)#13*#16 - }#11;#17 -}#0 - -// // +tokenids -// f! { -// fn#1 main#2() { -// 1#5;#6 -// 1.0#7;#8 -// let#9 x#10 =#11 1#12;#13 -// } -// } -fn#19 main#20(#21)#21 {#22 - 1#23;#24 - 1.0#25;#26 - let#27 x#28 =#29 1#30;#31 -}#22 +macro_rules! f { + ($($tt:tt)*) => { + $($tt)* + }; +} +fn#SpanAnchor(FileId(0), 2)@22..24 main#SpanAnchor(FileId(0), 2)@25..29(#SpanAnchor(FileId(0), 2)@29..30)#SpanAnchor(FileId(0), 2)@30..31 {#SpanAnchor(FileId(0), 2)@32..33 + 1#SpanAnchor(FileId(0), 2)@42..43;#SpanAnchor(FileId(0), 2)@43..44 + 1.0#SpanAnchor(FileId(0), 2)@53..56;#SpanAnchor(FileId(0), 2)@56..57 + let#SpanAnchor(FileId(0), 2)@66..69 x#SpanAnchor(FileId(0), 2)@70..71 =#SpanAnchor(FileId(0), 2)@72..73 1#SpanAnchor(FileId(0), 2)@74..75;#SpanAnchor(FileId(0), 2)@75..76 +}#SpanAnchor(FileId(0), 2)@81..82 -"##]], + +"#]], ); } @@ -150,8 +136,7 @@ macro_rules! identity { } fn main(foo: ()) { - // format_args/*+tokenids*/!("{} {} {}"#1,#2 format_args#3!#4("{}"#6,#7 0#8),#9 foo#10,#11 identity#12!#13(10#15),#16 "bar"#17) -builtin#4294967295 ##4294967295format_args#4294967295 (#0"{} {} {}"#1,#2 format_args#3!#4(#5"{}"#6,#7 0#8)#5,#9 foo#10,#11 identity#12!#13(#1410#15)#14,#16 "bar"#17)#0 + builtin#SpanAnchor(FileId(0), 0)@0..0 ##SpanAnchor(FileId(0), 0)@0..0format_args#SpanAnchor(FileId(0), 0)@0..0 (#SpanAnchor(FileId(0), 6)@25..26"{} {} {}"#SpanAnchor(FileId(0), 6)@26..36,#SpanAnchor(FileId(0), 6)@36..37 format_args#SpanAnchor(FileId(0), 6)@38..49!#SpanAnchor(FileId(0), 6)@49..50(#SpanAnchor(FileId(0), 6)@50..51"{}"#SpanAnchor(FileId(0), 6)@51..55,#SpanAnchor(FileId(0), 6)@55..56 0#SpanAnchor(FileId(0), 6)@57..58)#SpanAnchor(FileId(0), 6)@58..59,#SpanAnchor(FileId(0), 6)@59..60 foo#SpanAnchor(FileId(0), 6)@61..64,#SpanAnchor(FileId(0), 6)@64..65 identity#SpanAnchor(FileId(0), 6)@66..74!#SpanAnchor(FileId(0), 6)@74..75(#SpanAnchor(FileId(0), 6)@75..7610#SpanAnchor(FileId(0), 6)@76..78)#SpanAnchor(FileId(0), 6)@78..79,#SpanAnchor(FileId(0), 6)@79..80 "bar"#SpanAnchor(FileId(0), 6)@81..86)#SpanAnchor(FileId(0), 6)@86..87 } "##]], diff --git a/crates/hir-def/src/macro_expansion_tests/mod.rs b/crates/hir-def/src/macro_expansion_tests/mod.rs index 8adced4e0824..1a672b4605c6 100644 --- a/crates/hir-def/src/macro_expansion_tests/mod.rs +++ b/crates/hir-def/src/macro_expansion_tests/mod.rs @@ -16,21 +16,16 @@ mod proc_macros; use std::{iter, ops::Range, sync}; -use ::mbe::TokenMap; use base_db::{fixture::WithFixture, ProcMacro, SourceDatabase}; use expect_test::Expect; -use hir_expand::{ - db::{DeclarativeMacroExpander, ExpandDatabase}, - AstId, InFile, MacroFile, -}; +use hir_expand::{db::ExpandDatabase, HirFileIdExt, InFile, MacroFile, SpanMap}; use stdx::format_to; use syntax::{ ast::{self, edit::IndentLevel}, - AstNode, SyntaxElement, - SyntaxKind::{self, COMMENT, EOF, IDENT, LIFETIME_IDENT}, - SyntaxNode, TextRange, T, + AstNode, + SyntaxKind::{COMMENT, EOF, IDENT, LIFETIME_IDENT}, + SyntaxNode, T, }; -use tt::token_id::{Subtree, TokenId}; use crate::{ db::DefDatabase, @@ -39,6 +34,7 @@ use crate::{ resolver::HasResolver, src::HasSource, test_db::TestDB, + tt::Subtree, AdtId, AsMacroCall, Lookup, ModuleDefId, }; @@ -88,43 +84,6 @@ pub fn identity_when_valid(_attr: TokenStream, item: TokenStream) -> TokenStream let mut text_edits = Vec::new(); let mut expansions = Vec::new(); - for macro_ in source_file.syntax().descendants().filter_map(ast::Macro::cast) { - let mut show_token_ids = false; - for comment in macro_.syntax().children_with_tokens().filter(|it| it.kind() == COMMENT) { - show_token_ids |= comment.to_string().contains("+tokenids"); - } - if !show_token_ids { - continue; - } - - let call_offset = macro_.syntax().text_range().start().into(); - let file_ast_id = db.ast_id_map(source.file_id).ast_id(¯o_); - let ast_id = AstId::new(source.file_id, file_ast_id.upcast()); - - let DeclarativeMacroExpander { mac, def_site_token_map } = - &*db.decl_macro_expander(krate, ast_id); - assert_eq!(mac.err(), None); - let tt = match ¯o_ { - ast::Macro::MacroRules(mac) => mac.token_tree().unwrap(), - ast::Macro::MacroDef(_) => unimplemented!(""), - }; - - let tt_start = tt.syntax().text_range().start(); - tt.syntax().descendants_with_tokens().filter_map(SyntaxElement::into_token).for_each( - |token| { - let range = token.text_range().checked_sub(tt_start).unwrap(); - if let Some(id) = def_site_token_map.token_by_range(range) { - let offset = (range.end() + tt_start).into(); - text_edits.push((offset..offset, format!("#{}", id.0))); - } - }, - ); - text_edits.push(( - call_offset..call_offset, - format!("// call ids will be shifted by {:?}\n", mac.shift()), - )); - } - for macro_call in source_file.syntax().descendants().filter_map(ast::MacroCall::cast) { let macro_call = InFile::new(source.file_id, ¯o_call); let res = macro_call @@ -138,10 +97,10 @@ pub fn identity_when_valid(_attr: TokenStream, item: TokenStream) -> TokenStream let macro_file = MacroFile { macro_call_id }; let mut expansion_result = db.parse_macro_expansion(macro_file); expansion_result.err = expansion_result.err.or(res.err); - expansions.push((macro_call.value.clone(), expansion_result, db.macro_arg(macro_call_id))); + expansions.push((macro_call.value.clone(), expansion_result)); } - for (call, exp, arg) in expansions.into_iter().rev() { + for (call, exp) in expansions.into_iter().rev() { let mut tree = false; let mut expect_errors = false; let mut show_token_ids = false; @@ -185,27 +144,7 @@ pub fn identity_when_valid(_attr: TokenStream, item: TokenStream) -> TokenStream } let range = call.syntax().text_range(); let range: Range = range.into(); - - if show_token_ids { - if let Some((tree, map, _)) = arg.value.as_deref() { - let tt_range = call.token_tree().unwrap().syntax().text_range(); - let mut ranges = Vec::new(); - extract_id_ranges(&mut ranges, map, tree); - for (range, id) in ranges { - let idx = (tt_range.start() + range.end()).into(); - text_edits.push((idx..idx, format!("#{}", id.0))); - } - } - text_edits.push((range.start..range.start, "// ".into())); - call.to_string().match_indices('\n').for_each(|(offset, _)| { - let offset = offset + 1 + range.start; - text_edits.push((offset..offset, "// ".into())); - }); - text_edits.push((range.end..range.end, "\n".into())); - text_edits.push((range.end..range.end, expn_text)); - } else { - text_edits.push((range, expn_text)); - } + text_edits.push((range, expn_text)); } text_edits.sort_by_key(|(range, _)| range.start); @@ -246,20 +185,6 @@ pub fn identity_when_valid(_attr: TokenStream, item: TokenStream) -> TokenStream expect.assert_eq(&expanded_text); } -fn extract_id_ranges(ranges: &mut Vec<(TextRange, TokenId)>, map: &TokenMap, tree: &Subtree) { - tree.token_trees.iter().for_each(|tree| match tree { - tt::TokenTree::Leaf(leaf) => { - let id = match leaf { - tt::Leaf::Literal(it) => it.span, - tt::Leaf::Punct(it) => it.span, - tt::Leaf::Ident(it) => it.span, - }; - ranges.extend(map.ranges_by_token(id, SyntaxKind::ERROR).map(|range| (range, id))); - } - tt::TokenTree::Subtree(tree) => extract_id_ranges(ranges, map, tree), - }); -} - fn reindent(indent: IndentLevel, pp: String) -> String { if !pp.contains('\n') { return pp; @@ -276,7 +201,7 @@ fn reindent(indent: IndentLevel, pp: String) -> String { res } -fn pretty_print_macro_expansion(expn: SyntaxNode, map: Option<&TokenMap>) -> String { +fn pretty_print_macro_expansion(expn: SyntaxNode, map: Option<&SpanMap>) -> String { let mut res = String::new(); let mut prev_kind = EOF; let mut indent_level = 0; @@ -323,8 +248,8 @@ fn pretty_print_macro_expansion(expn: SyntaxNode, map: Option<&TokenMap>) -> Str prev_kind = curr_kind; format_to!(res, "{}", token); if let Some(map) = map { - if let Some(id) = map.token_by_range(token.text_range()) { - format_to!(res, "#{}", id.0); + if let Some(span) = map.span_for_range(token.text_range()) { + format_to!(res, "#{:?}@{:?}", span.anchor, span.range); } } } diff --git a/crates/hir-def/src/nameres/collector.rs b/crates/hir-def/src/nameres/collector.rs index 2d4586146db0..659e7ed5033e 100644 --- a/crates/hir-def/src/nameres/collector.rs +++ b/crates/hir-def/src/nameres/collector.rs @@ -5,6 +5,7 @@ use std::{cmp::Ordering, iter, mem}; +use ::tt::Span; use base_db::{CrateId, Dependency, Edition, FileId}; use cfg::{CfgExpr, CfgOptions}; use either::Either; @@ -85,8 +86,7 @@ pub(super) fn collect_defs(db: &dyn DefDatabase, def_map: DefMap, tree_id: TreeI .enumerate() .map(|(idx, it)| { // FIXME: a hacky way to create a Name from string. - let name = - tt::Ident { text: it.name.clone(), span: tt::TokenId::unspecified() }; + let name = tt::Ident { text: it.name.clone(), span: tt::SpanData::DUMMY }; (name.as_name(), ProcMacroExpander::new(base_db::ProcMacroId(idx as u32))) }) .collect()) @@ -471,7 +471,7 @@ impl DefCollector<'_> { directive.module_id, MacroCallKind::Attr { ast_id: ast_id.ast_id, - attr_args: Arc::new((tt::Subtree::empty(), Default::default())), + attr_args: Arc::new(tt::Subtree::empty()), invoc_attr_index: attr.id, }, attr.path().clone(), @@ -2083,8 +2083,7 @@ impl ModCollector<'_, '_> { let name = match attrs.by_key("rustc_builtin_macro").string_value() { Some(it) => { // FIXME: a hacky way to create a Name from string. - name = - tt::Ident { text: it.clone(), span: tt::TokenId::unspecified() }.as_name(); + name = tt::Ident { text: it.clone(), span: tt::SpanData::DUMMY }.as_name(); &name } None => { diff --git a/crates/hir-def/src/nameres/mod_resolution.rs b/crates/hir-def/src/nameres/mod_resolution.rs index 2dcc2c30fe16..22802433aa58 100644 --- a/crates/hir-def/src/nameres/mod_resolution.rs +++ b/crates/hir-def/src/nameres/mod_resolution.rs @@ -1,7 +1,7 @@ //! This module resolves `mod foo;` declaration to file. use arrayvec::ArrayVec; use base_db::{AnchoredPath, FileId}; -use hir_expand::name::Name; +use hir_expand::{name::Name, HirFileIdExt}; use limit::Limit; use syntax::SmolStr; diff --git a/crates/hir-def/src/nameres/tests/incremental.rs b/crates/hir-def/src/nameres/tests/incremental.rs index 4a86f88e57af..977745c476f1 100644 --- a/crates/hir-def/src/nameres/tests/incremental.rs +++ b/crates/hir-def/src/nameres/tests/incremental.rs @@ -66,7 +66,7 @@ fn typing_inside_a_function_should_not_invalidate_def_map() { #[test] fn typing_inside_a_macro_should_not_invalidate_def_map() { - let (mut db, pos) = TestDB::with_position( + check_def_map_is_not_recomputed( r" //- /lib.rs macro_rules! m { @@ -84,27 +84,15 @@ fn typing_inside_a_macro_should_not_invalidate_def_map() { //- /foo/bar.rs $0 m!(X); + + pub struct S {} ", - ); - let krate = db.test_crate(); - { - let events = db.log_executed(|| { - let crate_def_map = db.crate_def_map(krate); - let (_, module_data) = crate_def_map.modules.iter().last().unwrap(); - assert_eq!(module_data.scope.resolutions().count(), 1); - }); - assert!(format!("{events:?}").contains("crate_def_map"), "{events:#?}") - } - db.set_file_text(pos.file_id, Arc::from("m!(Y);")); + r" + m!(Y); - { - let events = db.log_executed(|| { - let crate_def_map = db.crate_def_map(krate); - let (_, module_data) = crate_def_map.modules.iter().last().unwrap(); - assert_eq!(module_data.scope.resolutions().count(), 1); - }); - assert!(!format!("{events:?}").contains("crate_def_map"), "{events:#?}") - } + pub struct S {} + ", + ); } #[test] diff --git a/crates/hir-expand/src/ast_id_map.rs b/crates/hir-expand/src/ast_id_map.rs index 40726505491b..eb43ae37e08a 100644 --- a/crates/hir-expand/src/ast_id_map.rs +++ b/crates/hir-expand/src/ast_id_map.rs @@ -12,11 +12,13 @@ use std::{ marker::PhantomData, }; -use la_arena::{Arena, Idx}; +use la_arena::{Arena, Idx, RawIdx}; use profile::Count; use rustc_hash::FxHasher; use syntax::{ast, AstNode, AstPtr, SyntaxNode, SyntaxNodePtr}; +pub use base_db::span::ErasedFileAstId; + /// `AstId` points to an AST node in a specific file. pub struct FileAstId { raw: ErasedFileAstId, @@ -62,8 +64,6 @@ impl FileAstId { } } -pub type ErasedFileAstId = Idx; - pub trait AstIdNode: AstNode {} macro_rules! register_ast_id_node { (impl AstIdNode for $($ident:ident),+ ) => { @@ -129,6 +129,11 @@ impl AstIdMap { pub(crate) fn from_source(node: &SyntaxNode) -> AstIdMap { assert!(node.parent().is_none()); let mut res = AstIdMap::default(); + + // make sure to allocate the root node + if !should_alloc_id(node.kind()) { + res.alloc(node); + } // By walking the tree in breadth-first order we make sure that parents // get lower ids then children. That is, adding a new child does not // change parent's id. This means that, say, adding a new function to a @@ -155,6 +160,11 @@ impl AstIdMap { res } + /// The [`AstId`] of the root node + pub fn root(&self) -> SyntaxNodePtr { + self.arena[Idx::from_raw(RawIdx::from_u32(0))].clone() + } + pub fn ast_id(&self, item: &N) -> FileAstId { let raw = self.erased_ast_id(item.syntax()); FileAstId { raw, covariant: PhantomData } @@ -164,7 +174,7 @@ impl AstIdMap { AstPtr::try_from_raw(self.arena[id.raw].clone()).unwrap() } - pub(crate) fn get_raw(&self, id: ErasedFileAstId) -> SyntaxNodePtr { + pub fn get_raw(&self, id: ErasedFileAstId) -> SyntaxNodePtr { self.arena[id].clone() } diff --git a/crates/hir-expand/src/attrs.rs b/crates/hir-expand/src/attrs.rs index 0ec2422b30cf..9652dd345a8d 100644 --- a/crates/hir-expand/src/attrs.rs +++ b/crates/hir-expand/src/attrs.rs @@ -1,7 +1,8 @@ //! A higher level attributes based on TokenTree, with also some shortcuts. use std::{fmt, ops}; -use base_db::CrateId; +use ::tt::Span; +use base_db::{span::SpanAnchor, CrateId}; use cfg::CfgExpr; use either::Either; use intern::Interned; @@ -39,11 +40,16 @@ impl ops::Deref for RawAttrs { impl RawAttrs { pub const EMPTY: Self = Self { entries: None }; - pub fn new(db: &dyn ExpandDatabase, owner: &dyn ast::HasAttrs, hygiene: &Hygiene) -> Self { + pub fn new( + db: &dyn ExpandDatabase, + span_anchor: SpanAnchor, + owner: &dyn ast::HasAttrs, + hygiene: &Hygiene, + ) -> Self { let entries = collect_attrs(owner) .filter_map(|(id, attr)| match attr { Either::Left(attr) => { - attr.meta().and_then(|meta| Attr::from_src(db, meta, hygiene, id)) + attr.meta().and_then(|meta| Attr::from_src(db, span_anchor, meta, hygiene, id)) } Either::Right(comment) => comment.doc_comment().map(|doc| Attr { id, @@ -58,9 +64,13 @@ impl RawAttrs { Self { entries: if entries.is_empty() { None } else { Some(entries) } } } - pub fn from_attrs_owner(db: &dyn ExpandDatabase, owner: InFile<&dyn ast::HasAttrs>) -> Self { + pub fn from_attrs_owner( + db: &dyn ExpandDatabase, + span_anchor: SpanAnchor, + owner: InFile<&dyn ast::HasAttrs>, + ) -> Self { let hygiene = Hygiene::new(db, owner.file_id); - Self::new(db, owner.value, &hygiene) + Self::new(db, span_anchor, owner.value, &hygiene) } pub fn merge(&self, other: Self) -> Self { @@ -190,16 +200,17 @@ pub struct Attr { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum AttrInput { /// `#[attr = "string"]` + // FIXME: This is losing span Literal(SmolStr), /// `#[attr(subtree)]` - TokenTree(Box<(tt::Subtree, mbe::TokenMap)>), + TokenTree(Box), } impl fmt::Display for AttrInput { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { AttrInput::Literal(lit) => write!(f, " = \"{}\"", lit.escape_debug()), - AttrInput::TokenTree(tt) => tt.0.fmt(f), + AttrInput::TokenTree(tt) => tt.fmt(f), } } } @@ -207,6 +218,7 @@ impl fmt::Display for AttrInput { impl Attr { fn from_src( db: &dyn ExpandDatabase, + span_anchor: SpanAnchor, ast: ast::Meta, hygiene: &Hygiene, id: AttrId, @@ -219,8 +231,13 @@ impl Attr { }; Some(Interned::new(AttrInput::Literal(value))) } else if let Some(tt) = ast.token_tree() { - let (tree, map) = syntax_node_to_token_tree(tt.syntax()); - Some(Interned::new(AttrInput::TokenTree(Box::new((tree, map))))) + // FIXME: We could also allocate ids for attributes and use the attribute itself as an anchor + let offset = + db.ast_id_map(span_anchor.file_id).get_raw(span_anchor.ast_id).text_range().start(); + // FIXME: Spanmap + let tree = + syntax_node_to_token_tree(tt.syntax(), span_anchor, offset, &Default::default()); + Some(Interned::new(AttrInput::TokenTree(Box::new(tree)))) } else { None }; @@ -233,10 +250,12 @@ impl Attr { hygiene: &Hygiene, id: AttrId, ) -> Option { - let (parse, _) = mbe::token_tree_to_syntax_node(tt, mbe::TopEntryPoint::MetaItem); + // FIXME: Unecessary roundtrip tt -> ast -> tt + let (parse, _map) = mbe::token_tree_to_syntax_node(tt, mbe::TopEntryPoint::MetaItem); let ast = ast::Meta::cast(parse.syntax_node())?; - Self::from_src(db, ast, hygiene, id) + // FIXME: we discard spans here! + Self::from_src(db, SpanAnchor::DUMMY, ast, hygiene, id) } pub fn path(&self) -> &ModPath { @@ -256,7 +275,7 @@ impl Attr { /// #[path(ident)] pub fn single_ident_value(&self) -> Option<&tt::Ident> { match self.input.as_deref()? { - AttrInput::TokenTree(tt) => match &*tt.0.token_trees { + AttrInput::TokenTree(tt) => match &*tt.token_trees { [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] => Some(ident), _ => None, }, @@ -267,7 +286,7 @@ impl Attr { /// #[path TokenTree] pub fn token_tree_value(&self) -> Option<&Subtree> { match self.input.as_deref()? { - AttrInput::TokenTree(tt) => Some(&tt.0), + AttrInput::TokenTree(tt) => Some(tt), _ => None, } } diff --git a/crates/hir-expand/src/builtin_attr_macro.rs b/crates/hir-expand/src/builtin_attr_macro.rs index 4ee12e2f2129..de8c0ac810bf 100644 --- a/crates/hir-expand/src/builtin_attr_macro.rs +++ b/crates/hir-expand/src/builtin_attr_macro.rs @@ -1,5 +1,7 @@ //! Builtin attributes. +use ::tt::Span; + use crate::{db::ExpandDatabase, name, tt, ExpandResult, MacroCallId, MacroCallKind}; macro_rules! register_builtin { @@ -98,7 +100,7 @@ fn derive_attr_expand( ) -> ExpandResult { let loc = db.lookup_intern_macro_call(id); let derives = match &loc.kind { - MacroCallKind::Attr { attr_args, .. } if loc.def.is_attribute_derive() => &attr_args.0, + MacroCallKind::Attr { attr_args, .. } if loc.def.is_attribute_derive() => attr_args, _ => return ExpandResult::ok(tt::Subtree::empty()), }; pseudo_derive_attr_expansion(tt, derives) @@ -112,7 +114,7 @@ pub fn pseudo_derive_attr_expansion( tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char, spacing: tt::Spacing::Alone, - span: tt::TokenId::unspecified(), + span: tt::SpanData::DUMMY, })) }; diff --git a/crates/hir-expand/src/builtin_derive_macro.rs b/crates/hir-expand/src/builtin_derive_macro.rs index ecc8b407a9c8..16cce35c13d9 100644 --- a/crates/hir-expand/src/builtin_derive_macro.rs +++ b/crates/hir-expand/src/builtin_derive_macro.rs @@ -1,18 +1,20 @@ //! Builtin derives. -use ::tt::Ident; +use ::tt::Span; use base_db::{CrateOrigin, LangCrateOrigin}; use itertools::izip; -use mbe::TokenMap; use rustc_hash::FxHashSet; use stdx::never; use tracing::debug; use crate::{ name::{AsName, Name}, - tt::{self, TokenId}, + tt, SpanMap, +}; +use syntax::{ + ast::{self, AstNode, FieldList, HasAttrs, HasGenericParams, HasName, HasTypeBounds}, + TextSize, }; -use syntax::ast::{self, AstNode, FieldList, HasAttrs, HasGenericParams, HasName, HasTypeBounds}; use crate::{db::ExpandDatabase, name, quote, ExpandError, ExpandResult, MacroCallId}; @@ -29,7 +31,7 @@ macro_rules! register_builtin { db: &dyn ExpandDatabase, id: MacroCallId, tt: &ast::Adt, - token_map: &TokenMap, + token_map: &SpanMap, ) -> ExpandResult { let expander = match *self { $( BuiltinDeriveExpander::$trait => $expand, )* @@ -71,7 +73,7 @@ enum VariantShape { } fn tuple_field_iterator(n: usize) -> impl Iterator { - (0..n).map(|it| Ident::new(format!("f{it}"), tt::TokenId::unspecified())) + (0..n).map(|it| tt::Ident::new(format!("f{it}"), tt::SpanData::DUMMY)) } impl VariantShape { @@ -117,7 +119,7 @@ impl VariantShape { } } - fn from(tm: &TokenMap, value: Option) -> Result { + fn from(tm: &SpanMap, value: Option) -> Result { let r = match value { None => VariantShape::Unit, Some(FieldList::RecordFieldList(it)) => VariantShape::Struct( @@ -189,8 +191,8 @@ struct BasicAdtInfo { associated_types: Vec, } -fn parse_adt(tm: &TokenMap, adt: &ast::Adt) -> Result { - let (name, generic_param_list, shape) = match &adt { +fn parse_adt(tm: &SpanMap, adt: &ast::Adt) -> Result { + let (name, generic_param_list, shape) = match adt { ast::Adt::Struct(it) => ( it.name(), it.generic_param_list(), @@ -234,21 +236,44 @@ fn parse_adt(tm: &TokenMap, adt: &ast::Adt) -> Result match this { Some(it) => { param_type_set.insert(it.as_name()); - mbe::syntax_node_to_token_tree(it.syntax()).0 + mbe::syntax_node_to_token_tree( + it.syntax(), + tm.span_for_range(it.syntax().first_token().unwrap().text_range()) + .unwrap() + .anchor, + TextSize::from(0), + tm, + ) } None => tt::Subtree::empty(), } }; let bounds = match ¶m { - ast::TypeOrConstParam::Type(it) => { - it.type_bound_list().map(|it| mbe::syntax_node_to_token_tree(it.syntax()).0) - } + ast::TypeOrConstParam::Type(it) => it.type_bound_list().map(|it| { + mbe::syntax_node_to_token_tree( + it.syntax(), + tm.span_for_range(it.syntax().first_token().unwrap().text_range()) + .unwrap() + .anchor, + TextSize::from(0), + tm, + ) + }), ast::TypeOrConstParam::Const(_) => None, }; let ty = if let ast::TypeOrConstParam::Const(param) = param { let ty = param .ty() - .map(|ty| mbe::syntax_node_to_token_tree(ty.syntax()).0) + .map(|ty| { + mbe::syntax_node_to_token_tree( + ty.syntax(), + tm.span_for_range(ty.syntax().first_token().unwrap().text_range()) + .unwrap() + .anchor, + TextSize::from(0), + tm, + ) + }) .unwrap_or_else(tt::Subtree::empty); Some(ty) } else { @@ -282,20 +307,26 @@ fn parse_adt(tm: &TokenMap, adt: &ast::Adt) -> Result let name = p.path()?.qualifier()?.as_single_name_ref()?.as_name(); param_type_set.contains(&name).then_some(p) }) - .map(|it| mbe::syntax_node_to_token_tree(it.syntax()).0) + .map(|it| { + mbe::syntax_node_to_token_tree( + it.syntax(), + tm.span_for_range(it.syntax().first_token().unwrap().text_range()).unwrap().anchor, + TextSize::from(0), + tm, + ) + }) .collect(); let name_token = name_to_token(&tm, name)?; Ok(BasicAdtInfo { name: name_token, shape, param_types, associated_types }) } -fn name_to_token(token_map: &TokenMap, name: Option) -> Result { +fn name_to_token(token_map: &SpanMap, name: Option) -> Result { let name = name.ok_or_else(|| { debug!("parsed item has no name"); ExpandError::other("missing name") })?; - let name_token_id = - token_map.token_by_range(name.syntax().text_range()).unwrap_or_else(TokenId::unspecified); - let name_token = tt::Ident { span: name_token_id, text: name.text().into() }; + let span = token_map.span_for_range(name.syntax().text_range()).unwrap(); + let name_token = tt::Ident { span, text: name.text().into() }; Ok(name_token) } @@ -332,7 +363,7 @@ fn name_to_token(token_map: &TokenMap, name: Option) -> Result tt::Subtree, ) -> ExpandResult { @@ -393,7 +424,7 @@ fn copy_expand( db: &dyn ExpandDatabase, id: MacroCallId, tt: &ast::Adt, - tm: &TokenMap, + tm: &SpanMap, ) -> ExpandResult { let krate = find_builtin_crate(db, id); expand_simple_derive(tt, tm, quote! { #krate::marker::Copy }, |_| quote! {}) @@ -403,16 +434,13 @@ fn clone_expand( db: &dyn ExpandDatabase, id: MacroCallId, tt: &ast::Adt, - tm: &TokenMap, + tm: &SpanMap, ) -> ExpandResult { let krate = find_builtin_crate(db, id); expand_simple_derive(tt, tm, quote! { #krate::clone::Clone }, |adt| { if matches!(adt.shape, AdtShape::Union) { - let star = tt::Punct { - char: '*', - spacing: ::tt::Spacing::Alone, - span: tt::TokenId::unspecified(), - }; + let star = + tt::Punct { char: '*', spacing: ::tt::Spacing::Alone, span: tt::SpanData::DUMMY }; return quote! { fn clone(&self) -> Self { #star self @@ -420,11 +448,8 @@ fn clone_expand( }; } if matches!(&adt.shape, AdtShape::Enum { variants, .. } if variants.is_empty()) { - let star = tt::Punct { - char: '*', - spacing: ::tt::Spacing::Alone, - span: tt::TokenId::unspecified(), - }; + let star = + tt::Punct { char: '*', spacing: ::tt::Spacing::Alone, span: tt::SpanData::DUMMY }; return quote! { fn clone(&self) -> Self { match #star self {} @@ -452,16 +477,14 @@ fn clone_expand( } /// This function exists since `quote! { => }` doesn't work. -fn fat_arrow() -> ::tt::Subtree { - let eq = - tt::Punct { char: '=', spacing: ::tt::Spacing::Joint, span: tt::TokenId::unspecified() }; +fn fat_arrow() -> tt::Subtree { + let eq = tt::Punct { char: '=', spacing: ::tt::Spacing::Joint, span: tt::SpanData::DUMMY }; quote! { #eq> } } /// This function exists since `quote! { && }` doesn't work. -fn and_and() -> ::tt::Subtree { - let and = - tt::Punct { char: '&', spacing: ::tt::Spacing::Joint, span: tt::TokenId::unspecified() }; +fn and_and() -> tt::Subtree { + let and = tt::Punct { char: '&', spacing: ::tt::Spacing::Joint, span: tt::SpanData::DUMMY }; quote! { #and& } } @@ -469,7 +492,7 @@ fn default_expand( db: &dyn ExpandDatabase, id: MacroCallId, tt: &ast::Adt, - tm: &TokenMap, + tm: &SpanMap, ) -> ExpandResult { let krate = &find_builtin_crate(db, id); expand_simple_derive(tt, tm, quote! { #krate::default::Default }, |adt| { @@ -509,7 +532,7 @@ fn debug_expand( db: &dyn ExpandDatabase, id: MacroCallId, tt: &ast::Adt, - tm: &TokenMap, + tm: &SpanMap, ) -> ExpandResult { let krate = &find_builtin_crate(db, id); expand_simple_derive(tt, tm, quote! { #krate::fmt::Debug }, |adt| { @@ -540,11 +563,8 @@ fn debug_expand( }, }; if matches!(&adt.shape, AdtShape::Enum { variants, .. } if variants.is_empty()) { - let star = tt::Punct { - char: '*', - spacing: ::tt::Spacing::Alone, - span: tt::TokenId::unspecified(), - }; + let star = + tt::Punct { char: '*', spacing: ::tt::Spacing::Alone, span: tt::SpanData::DUMMY }; return quote! { fn fmt(&self, f: &mut #krate::fmt::Formatter) -> #krate::fmt::Result { match #star self {} @@ -590,7 +610,7 @@ fn hash_expand( db: &dyn ExpandDatabase, id: MacroCallId, tt: &ast::Adt, - tm: &TokenMap, + tm: &SpanMap, ) -> ExpandResult { let krate = &find_builtin_crate(db, id); expand_simple_derive(tt, tm, quote! { #krate::hash::Hash }, |adt| { @@ -599,11 +619,8 @@ fn hash_expand( return quote! {}; } if matches!(&adt.shape, AdtShape::Enum { variants, .. } if variants.is_empty()) { - let star = tt::Punct { - char: '*', - spacing: ::tt::Spacing::Alone, - span: tt::TokenId::unspecified(), - }; + let star = + tt::Punct { char: '*', spacing: ::tt::Spacing::Alone, span: tt::SpanData::DUMMY }; return quote! { fn hash(&self, ra_expand_state: &mut H) { match #star self {} @@ -644,7 +661,7 @@ fn eq_expand( db: &dyn ExpandDatabase, id: MacroCallId, tt: &ast::Adt, - tm: &TokenMap, + tm: &SpanMap, ) -> ExpandResult { let krate = find_builtin_crate(db, id); expand_simple_derive(tt, tm, quote! { #krate::cmp::Eq }, |_| quote! {}) @@ -654,7 +671,7 @@ fn partial_eq_expand( db: &dyn ExpandDatabase, id: MacroCallId, tt: &ast::Adt, - tm: &TokenMap, + tm: &SpanMap, ) -> ExpandResult { let krate = find_builtin_crate(db, id); expand_simple_derive(tt, tm, quote! { #krate::cmp::PartialEq }, |adt| { @@ -674,14 +691,14 @@ fn partial_eq_expand( } [first, rest @ ..] => { let rest = rest.iter().map(|it| { - let t1 = Ident::new(format!("{}_self", it.text), it.span); - let t2 = Ident::new(format!("{}_other", it.text), it.span); + let t1 = tt::Ident::new(format!("{}_self", it.text), it.span); + let t2 = tt::Ident::new(format!("{}_other", it.text), it.span); let and_and = and_and(); quote!(#and_and #t1 .eq( #t2 )) }); let first = { - let t1 = Ident::new(format!("{}_self", first.text), first.span); - let t2 = Ident::new(format!("{}_other", first.text), first.span); + let t1 = tt::Ident::new(format!("{}_self", first.text), first.span); + let t2 = tt::Ident::new(format!("{}_other", first.text), first.span); quote!(#t1 .eq( #t2 )) }; quote!(#first ##rest) @@ -708,11 +725,11 @@ fn self_and_other_patterns( name: &tt::Ident, ) -> (Vec, Vec) { let self_patterns = adt.shape.as_pattern_map(name, |it| { - let t = Ident::new(format!("{}_self", it.text), it.span); + let t = tt::Ident::new(format!("{}_self", it.text), it.span); quote!(#t) }); let other_patterns = adt.shape.as_pattern_map(name, |it| { - let t = Ident::new(format!("{}_other", it.text), it.span); + let t = tt::Ident::new(format!("{}_other", it.text), it.span); quote!(#t) }); (self_patterns, other_patterns) @@ -722,7 +739,7 @@ fn ord_expand( db: &dyn ExpandDatabase, id: MacroCallId, tt: &ast::Adt, - tm: &TokenMap, + tm: &SpanMap, ) -> ExpandResult { let krate = &find_builtin_crate(db, id); expand_simple_derive(tt, tm, quote! { #krate::cmp::Ord }, |adt| { @@ -752,8 +769,8 @@ fn ord_expand( |(pat1, pat2, fields)| { let mut body = quote!(#krate::cmp::Ordering::Equal); for f in fields.into_iter().rev() { - let t1 = Ident::new(format!("{}_self", f.text), f.span); - let t2 = Ident::new(format!("{}_other", f.text), f.span); + let t1 = tt::Ident::new(format!("{}_self", f.text), f.span); + let t2 = tt::Ident::new(format!("{}_other", f.text), f.span); body = compare(krate, quote!(#t1), quote!(#t2), body); } let fat_arrow = fat_arrow(); @@ -784,7 +801,7 @@ fn partial_ord_expand( db: &dyn ExpandDatabase, id: MacroCallId, tt: &ast::Adt, - tm: &TokenMap, + tm: &SpanMap, ) -> ExpandResult { let krate = &find_builtin_crate(db, id); expand_simple_derive(tt, tm, quote! { #krate::cmp::PartialOrd }, |adt| { @@ -817,8 +834,8 @@ fn partial_ord_expand( |(pat1, pat2, fields)| { let mut body = quote!(#krate::option::Option::Some(#krate::cmp::Ordering::Equal)); for f in fields.into_iter().rev() { - let t1 = Ident::new(format!("{}_self", f.text), f.span); - let t2 = Ident::new(format!("{}_other", f.text), f.span); + let t1 = tt::Ident::new(format!("{}_self", f.text), f.span); + let t2 = tt::Ident::new(format!("{}_other", f.text), f.span); body = compare(krate, quote!(#t1), quote!(#t2), body); } let fat_arrow = fat_arrow(); diff --git a/crates/hir-expand/src/builtin_fn_macro.rs b/crates/hir-expand/src/builtin_fn_macro.rs index a04de10b899c..adbe49473ace 100644 --- a/crates/hir-expand/src/builtin_fn_macro.rs +++ b/crates/hir-expand/src/builtin_fn_macro.rs @@ -1,17 +1,22 @@ //! Builtin macro -use base_db::{AnchoredPath, Edition, FileId}; +use base_db::{ + span::{SpanAnchor, ROOT_ERASED_FILE_AST_ID}, + AnchoredPath, Edition, FileId, +}; use cfg::CfgExpr; use either::Either; -use mbe::{parse_exprs_with_sep, parse_to_token_tree, TokenMap}; +use mbe::{parse_exprs_with_sep, parse_to_token_tree}; use syntax::{ ast::{self, AstToken}, SmolStr, }; use crate::{ - db::ExpandDatabase, name, quote, tt, EagerCallInfo, ExpandError, ExpandResult, MacroCallId, - MacroCallLoc, + db::ExpandDatabase, + name, quote, + tt::{self, Span}, + EagerCallInfo, ExpandError, ExpandResult, HirFileIdExt, MacroCallId, MacroCallLoc, }; macro_rules! register_builtin { @@ -110,7 +115,7 @@ register_builtin! { } const DOLLAR_CRATE: tt::Ident = - tt::Ident { text: SmolStr::new_inline("$crate"), span: tt::TokenId::unspecified() }; + tt::Ident { text: SmolStr::new_inline("$crate"), span: tt::SpanData::DUMMY }; fn module_path_expand( _db: &dyn ExpandDatabase, @@ -131,7 +136,7 @@ fn line_expand( delimiter: tt::Delimiter::unspecified(), token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { text: "0u32".into(), - span: tt::Span::UNSPECIFIED, + span: tt::SpanData::DUMMY, }))], }) } @@ -179,7 +184,7 @@ fn assert_expand( token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', spacing: tt::Spacing::Alone, - span: tt::TokenId::unspecified(), + span: tt::SpanData::DUMMY, }))], }; let cond = cond.clone(); @@ -446,7 +451,7 @@ fn concat_bytes_expand( } } } - let ident = tt::Ident { text: bytes.join(", ").into(), span: tt::TokenId::unspecified() }; + let ident = tt::Ident { text: bytes.join(", ").into(), span: tt::SpanData::DUMMY }; ExpandResult { value: quote!([#ident]), err } } @@ -494,7 +499,7 @@ fn concat_idents_expand( } } } - let ident = tt::Ident { text: ident.into(), span: tt::TokenId::unspecified() }; + let ident = tt::Ident { text: ident.into(), span: tt::SpanData::DUMMY }; ExpandResult { value: quote!(#ident), err } } @@ -533,15 +538,16 @@ fn include_expand( _tt: &tt::Subtree, ) -> ExpandResult { match db.include_expand(arg_id) { - Ok((res, _)) => ExpandResult::ok(res.0.clone()), + Ok((res, _)) => ExpandResult::ok(res.as_ref().clone()), Err(e) => ExpandResult::new(tt::Subtree::empty(), e), } } +// FIXME: Check if this is still needed now after the token map rewrite pub(crate) fn include_arg_to_tt( db: &dyn ExpandDatabase, arg_id: MacroCallId, -) -> Result<(triomphe::Arc<(::tt::Subtree<::tt::TokenId>, TokenMap)>, FileId), ExpandError> { +) -> Result<(triomphe::Arc, FileId), ExpandError> { let loc = db.lookup_intern_macro_call(arg_id); let Some(EagerCallInfo { arg, arg_id, .. }) = loc.eager.as_deref() else { panic!("include_arg_to_tt called on non include macro call: {:?}", &loc.eager); @@ -549,9 +555,12 @@ pub(crate) fn include_arg_to_tt( let path = parse_string(&arg.0)?; let file_id = relative_file(db, *arg_id, &path, false)?; - let (subtree, map) = - parse_to_token_tree(&db.file_text(file_id)).ok_or(mbe::ExpandError::ConversionError)?; - Ok((triomphe::Arc::new((subtree, map)), file_id)) + let subtree = parse_to_token_tree( + &db.file_text(file_id), + SpanAnchor { file_id: file_id.into(), ast_id: ROOT_ERASED_FILE_AST_ID }, + ) + .ok_or(mbe::ExpandError::ConversionError)?; + Ok((triomphe::Arc::new(subtree), file_id)) } fn include_bytes_expand( @@ -568,7 +577,7 @@ fn include_bytes_expand( delimiter: tt::Delimiter::unspecified(), token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { text: r#"b"""#.into(), - span: tt::TokenId::unspecified(), + span: tt::SpanData::DUMMY, }))], }; ExpandResult::ok(res) diff --git a/crates/hir-expand/src/db.rs b/crates/hir-expand/src/db.rs index ff0d279d8cce..32ba7b2f9115 100644 --- a/crates/hir-expand/src/db.rs +++ b/crates/hir-expand/src/db.rs @@ -1,22 +1,25 @@ //! Defines database & queries for macro expansion. -use base_db::{salsa, CrateId, Edition, SourceDatabase}; +use base_db::{ + salsa, + span::{SpanAnchor, ROOT_ERASED_FILE_AST_ID}, + CrateId, Edition, SourceDatabase, +}; use either::Either; use limit::Limit; -use mbe::{syntax_node_to_token_tree, ValueResult}; -use rustc_hash::FxHashSet; +use mbe::{map_from_syntax_node, syntax_node_to_token_tree, ValueResult}; use syntax::{ ast::{self, HasAttrs, HasDocComments}, - AstNode, GreenNode, Parse, SyntaxError, SyntaxNode, SyntaxToken, T, + AstNode, Parse, SyntaxError, SyntaxNode, SyntaxToken, TextSize, T, }; use triomphe::Arc; use crate::{ ast_id_map::AstIdMap, builtin_attr_macro::pseudo_derive_attr_expansion, - builtin_fn_macro::EagerExpander, fixup, hygiene::HygieneFrame, tt, AstId, BuiltinAttrExpander, + builtin_fn_macro::EagerExpander, hygiene::HygieneFrame, tt, AstId, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander, EagerCallInfo, ExpandError, ExpandResult, ExpandTo, HirFileId, HirFileIdRepr, MacroCallId, MacroCallKind, MacroCallLoc, MacroDefId, - MacroDefKind, MacroFile, ProcMacroExpander, + MacroDefKind, MacroFile, ProcMacroExpander, SpanMap, SyntaxContext, SyntaxContextId, }; /// Total limit on the number of tokens produced by any macro invocation. @@ -30,8 +33,7 @@ static TOKEN_LIMIT: Limit = Limit::new(1_048_576); #[derive(Debug, Clone, Eq, PartialEq)] /// Old-style `macro_rules` or the new macros 2.0 pub struct DeclarativeMacroExpander { - pub mac: mbe::DeclarativeMacro, - pub def_site_token_map: mbe::TokenMap, + pub mac: mbe::DeclarativeMacro, } impl DeclarativeMacroExpander { @@ -41,21 +43,14 @@ impl DeclarativeMacroExpander { tt::Subtree::empty(), ExpandError::other(format!("invalid macro definition: {e}")), ), - None => self.mac.expand(tt).map_err(Into::into), + None => self.mac.expand(&tt).map_err(Into::into), } } - - pub fn map_id_down(&self, token_id: tt::TokenId) -> tt::TokenId { - self.mac.map_id_down(token_id) - } - - pub fn map_id_up(&self, token_id: tt::TokenId) -> (tt::TokenId, mbe::Origin) { - self.mac.map_id_up(token_id) - } } #[derive(Debug, Clone, Eq, PartialEq)] pub enum TokenExpander { + /// Old-style `macro_rules` or the new macros 2.0 DeclarativeMacro(Arc), /// Stuff like `line!` and `file!`. BuiltIn(BuiltinFnLikeExpander), @@ -69,31 +64,6 @@ pub enum TokenExpander { ProcMacro(ProcMacroExpander), } -// FIXME: Get rid of these methods -impl TokenExpander { - pub(crate) fn map_id_down(&self, id: tt::TokenId) -> tt::TokenId { - match self { - TokenExpander::DeclarativeMacro(expander) => expander.map_id_down(id), - TokenExpander::BuiltIn(..) - | TokenExpander::BuiltInEager(..) - | TokenExpander::BuiltInAttr(..) - | TokenExpander::BuiltInDerive(..) - | TokenExpander::ProcMacro(..) => id, - } - } - - pub(crate) fn map_id_up(&self, id: tt::TokenId) -> (tt::TokenId, mbe::Origin) { - match self { - TokenExpander::DeclarativeMacro(expander) => expander.map_id_up(id), - TokenExpander::BuiltIn(..) - | TokenExpander::BuiltInEager(..) - | TokenExpander::BuiltInAttr(..) - | TokenExpander::BuiltInDerive(..) - | TokenExpander::ProcMacro(..) => (id, mbe::Origin::Call), - } - } -} - #[salsa::query_group(ExpandDatabaseStorage)] pub trait ExpandDatabase: SourceDatabase { fn ast_id_map(&self, file_id: HirFileId) -> Arc; @@ -109,7 +79,7 @@ pub trait ExpandDatabase: SourceDatabase { fn parse_macro_expansion( &self, macro_file: MacroFile, - ) -> ExpandResult<(Parse, Arc)>; + ) -> ExpandResult<(Parse, Arc)>; /// Macro ids. That's probably the tricksiest bit in rust-analyzer, and the /// reason why we use salsa at all. @@ -118,23 +88,16 @@ pub trait ExpandDatabase: SourceDatabase { /// to be incremental. #[salsa::interned] fn intern_macro_call(&self, macro_call: MacroCallLoc) -> MacroCallId; + #[salsa::interned] + fn intern_syntax_context(&self, ctx: SyntaxContext) -> SyntaxContextId; - /// Lowers syntactic macro call to a token tree representation. - #[salsa::transparent] - fn macro_arg( - &self, - id: MacroCallId, - ) -> ValueResult< - Option>, - Arc>, - >; - /// Extracts syntax node, corresponding to a macro call. That's a firewall + /// Lowers syntactic macro call to a token tree representation. That's a firewall /// query, only typing in the macro call itself changes the returned /// subtree. - fn macro_arg_node( + fn macro_arg( &self, id: MacroCallId, - ) -> ValueResult, Arc>>; + ) -> ValueResult>, Arc>>; /// Fetches the expander for this macro. #[salsa::transparent] fn macro_expander(&self, id: MacroDefId) -> TokenExpander; @@ -152,10 +115,7 @@ pub trait ExpandDatabase: SourceDatabase { fn include_expand( &self, arg_id: MacroCallId, - ) -> Result< - (triomphe::Arc<(::tt::Subtree<::tt::TokenId>, mbe::TokenMap)>, base_db::FileId), - ExpandError, - >; + ) -> Result<(triomphe::Arc, base_db::FileId), ExpandError>; /// Special case of the previous query for procedural macros. We can't LRU /// proc macros, since they are not deterministic in general, and /// non-determinism breaks salsa in a very, very, very bad way. @@ -181,21 +141,19 @@ pub fn expand_speculative( token_to_map: SyntaxToken, ) -> Option<(SyntaxNode, SyntaxToken)> { let loc = db.lookup_intern_macro_call(actual_macro_call); - let token_range = token_to_map.text_range(); + let file_id = loc.kind.file_id(); // Build the subtree and token mapping for the speculative args - let censor = censor_for_macro_input(&loc, speculative_args); - let mut fixups = fixup::fixup_syntax(speculative_args); - fixups.replace.extend(censor.into_iter().map(|node| (node.into(), Vec::new()))); - let (mut tt, spec_args_tmap, _) = mbe::syntax_node_to_token_tree_with_modifications( + let _censor = censor_for_macro_input(&loc, speculative_args); + let mut tt = mbe::syntax_node_to_token_tree( speculative_args, - fixups.token_map, - fixups.next_id, - fixups.replace, - fixups.append, + // we don't leak these spans into any query so its fine to make them absolute + SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, + TextSize::new(0), + &Default::default(), ); - let (attr_arg, token_id) = match loc.kind { + let attr_arg = match loc.kind { MacroCallKind::Attr { invoc_attr_index, .. } => { let attr = if loc.def.is_attribute_derive() { // for pseudo-derive expansion we actually pass the attribute itself only @@ -210,48 +168,27 @@ pub fn expand_speculative( }?; match attr.token_tree() { Some(token_tree) => { - let (mut tree, map) = syntax_node_to_token_tree(attr.token_tree()?.syntax()); - tree.delimiter = tt::Delimiter::unspecified(); - - let shift = mbe::Shift::new(&tt); - shift.shift_all(&mut tree); - - let token_id = if token_tree.syntax().text_range().contains_range(token_range) { - let attr_input_start = - token_tree.left_delimiter_token()?.text_range().start(); - let range = token_range.checked_sub(attr_input_start)?; - let token_id = shift.shift(map.token_by_range(range)?); - Some(token_id) - } else { - None - }; - (Some(tree), token_id) - } - _ => (None, None), - } - } - _ => (None, None), - }; - let token_id = match token_id { - Some(token_id) => token_id, - // token wasn't inside an attribute input so it has to be in the general macro input - None => { - let range = token_range.checked_sub(speculative_args.text_range().start())?; - let token_id = spec_args_tmap.token_by_range(range)?; - match loc.def.kind { - MacroDefKind::Declarative(it) => { - db.decl_macro_expander(loc.krate, it).map_id_down(token_id) + let mut tree = syntax_node_to_token_tree( + token_tree.syntax(), + SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, + TextSize::new(0), + &Default::default(), + ); + tree.delimiter = tt::Delimiter::UNSPECIFIED; + + Some(tree) } - _ => token_id, + _ => None, } } + _ => None, }; // Do the actual expansion, we need to directly expand the proc macro due to the attribute args // Otherwise the expand query will fetch the non speculative attribute args and pass those instead. - let mut speculative_expansion = match loc.def.kind { + let speculative_expansion = match loc.def.kind { MacroDefKind::ProcMacro(expander, ..) => { - tt.delimiter = tt::Delimiter::unspecified(); + tt.delimiter = tt::Delimiter::UNSPECIFIED; expander.expand(db, loc.def.krate, loc.krate, &tt, attr_arg.as_ref()) } MacroDefKind::BuiltInAttr(BuiltinAttrExpander::Derive, _) => { @@ -260,7 +197,17 @@ pub fn expand_speculative( MacroDefKind::BuiltInDerive(expander, ..) => { // this cast is a bit sus, can we avoid losing the typedness here? let adt = ast::Adt::cast(speculative_args.clone()).unwrap(); - expander.expand(db, actual_macro_call, &adt, &spec_args_tmap) + expander.expand( + db, + actual_macro_call, + &adt, + &map_from_syntax_node( + speculative_args, + // we don't leak these spans into any query so its fine to make them absolute + SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, + TextSize::new(0), + ), + ) } MacroDefKind::Declarative(it) => db.decl_macro_expander(loc.krate, it).expand(tt), MacroDefKind::BuiltIn(it, _) => it.expand(db, actual_macro_call, &tt).map_err(Into::into), @@ -271,12 +218,14 @@ pub fn expand_speculative( }; let expand_to = macro_expand_to(db, actual_macro_call); - fixup::reverse_fixups(&mut speculative_expansion.value, &spec_args_tmap, &fixups.undo_info); - let (node, rev_tmap) = token_tree_to_syntax_node(&speculative_expansion.value, expand_to); + let (node, rev_tmap) = token_tree_to_syntax_node(db, &speculative_expansion.value, expand_to); let syntax_node = node.syntax_node(); let token = rev_tmap - .ranges_by_token(token_id, token_to_map.kind()) + .ranges_with_span(tt::SpanData { + range: token_to_map.text_range(), + anchor: SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, + }) .filter_map(|range| syntax_node.covering_element(range).into_token()) .min_by_key(|t| { // prefer tokens of the same kind and text @@ -293,7 +242,7 @@ fn ast_id_map(db: &dyn ExpandDatabase, file_id: HirFileId) -> Arc { fn parse_or_expand(db: &dyn ExpandDatabase, file_id: HirFileId) -> SyntaxNode { match file_id.repr() { - HirFileIdRepr::FileId(file_id) => db.parse(file_id).tree().syntax().clone(), + HirFileIdRepr::FileId(file_id) => db.parse(file_id).syntax_node(), HirFileIdRepr::MacroFile(macro_file) => { db.parse_macro_expansion(macro_file).value.0.syntax_node() } @@ -315,7 +264,7 @@ fn parse_or_expand_with_err( fn parse_macro_expansion( db: &dyn ExpandDatabase, macro_file: MacroFile, -) -> ExpandResult<(Parse, Arc)> { +) -> ExpandResult<(Parse, Arc)> { let _p = profile::span("parse_macro_expansion"); let mbe::ValueResult { value: tt, err } = db.macro_expand(macro_file.macro_call_id); @@ -324,7 +273,7 @@ fn parse_macro_expansion( tracing::debug!("expanded = {}", tt.as_debug_string()); tracing::debug!("kind = {:?}", expand_to); - let (parse, rev_token_map) = token_tree_to_syntax_node(&tt, expand_to); + let (parse, rev_token_map) = token_tree_to_syntax_node(db, &tt, expand_to); ExpandResult { value: (parse, Arc::new(rev_token_map)), err } } @@ -340,48 +289,119 @@ fn parse_macro_expansion_error( fn macro_arg( db: &dyn ExpandDatabase, id: MacroCallId, -) -> ValueResult< - Option>, - Arc>, -> { - let loc = db.lookup_intern_macro_call(id); - - if let Some(EagerCallInfo { arg, arg_id: _, error: _ }) = loc.eager.as_deref() { - return ValueResult::ok(Some(Arc::new((arg.0.clone(), arg.1.clone(), Default::default())))); - } - - let ValueResult { value, err } = db.macro_arg_node(id); - let Some(arg) = value else { - return ValueResult { value: None, err }; +) -> ValueResult>, Arc>> { + let mismatched_delimiters = |arg: &SyntaxNode| { + let first = arg.first_child_or_token().map_or(T![.], |it| it.kind()); + let last = arg.last_child_or_token().map_or(T![.], |it| it.kind()); + let well_formed_tt = + matches!((first, last), (T!['('], T![')']) | (T!['['], T![']']) | (T!['{'], T!['}'])); + if !well_formed_tt { + // Don't expand malformed (unbalanced) macro invocations. This is + // less than ideal, but trying to expand unbalanced macro calls + // sometimes produces pathological, deeply nested code which breaks + // all kinds of things. + // + // Some day, we'll have explicit recursion counters for all + // recursive things, at which point this code might be removed. + cov_mark::hit!(issue9358_bad_macro_stack_overflow); + Some(Arc::new(Box::new([SyntaxError::new( + "unbalanced token tree".to_owned(), + arg.text_range(), + )]) as Box<[_]>)) + } else { + None + } }; + let loc = db.lookup_intern_macro_call(id); + if let Some(EagerCallInfo { arg, .. }) = matches!(loc.def.kind, MacroDefKind::BuiltInEager(..)) + .then(|| loc.eager.as_deref()) + .flatten() + { + ValueResult::ok(Some(Arc::new(arg.0.clone()))) + } else { + let (parse, map) = match loc.kind.file_id().repr() { + HirFileIdRepr::FileId(file_id) => { + (db.parse(file_id).to_syntax(), Arc::new(Default::default())) + } + HirFileIdRepr::MacroFile(macro_file) => { + let (parse, map) = db.parse_macro_expansion(macro_file).value; + (parse, map) + } + }; + let root = parse.syntax_node(); + + let (syntax, offset, ast_id) = match loc.kind { + MacroCallKind::FnLike { ast_id, .. } => { + let node = &ast_id.to_ptr(db).to_node(&root); + let offset = node.syntax().text_range().start(); + match node.token_tree().map(|it| it.syntax().clone()) { + Some(tt) => { + if let Some(e) = mismatched_delimiters(&tt) { + return ValueResult::only_err(e); + } + (tt, offset, ast_id.value.erase()) + } + None => { + return ValueResult::only_err(Arc::new(Box::new([ + SyntaxError::new_at_offset("missing token tree".to_owned(), offset), + ]))); + } + } + } + MacroCallKind::Derive { ast_id, .. } => { + let syntax_node = ast_id.to_ptr(db).to_node(&root).syntax().clone(); + let offset = syntax_node.text_range().start(); + (syntax_node, offset, ast_id.value.erase()) + } + MacroCallKind::Attr { ast_id, .. } => { + let syntax_node = ast_id.to_ptr(db).to_node(&root).syntax().clone(); + let offset = syntax_node.text_range().start(); + (syntax_node, offset, ast_id.value.erase()) + } + }; + let censor = censor_for_macro_input(&loc, &syntax); + // let mut fixups = fixup::fixup_syntax(&node); + // fixups.replace.extend(censor.into_iter().map(|node| (node.into(), Vec::new()))); + // let (mut tt, tmap, _) = mbe::syntax_node_to_token_tree_with_modifications( + // &node, + // fixups.token_map, + // fixups.next_id, + // fixups.replace, + // fixups.append, + // ); + let mut tt = mbe::syntax_node_to_token_tree_censored( + &syntax, + SpanAnchor { file_id: loc.kind.file_id(), ast_id }, + offset, + &map, + censor, + ); + + if loc.def.is_proc_macro() { + // proc macros expect their inputs without parentheses, MBEs expect it with them included + tt.delimiter = tt::Delimiter::UNSPECIFIED; + } - let node = SyntaxNode::new_root(arg); - let censor = censor_for_macro_input(&loc, &node); - let mut fixups = fixup::fixup_syntax(&node); - fixups.replace.extend(censor.into_iter().map(|node| (node.into(), Vec::new()))); - let (mut tt, tmap, _) = mbe::syntax_node_to_token_tree_with_modifications( - &node, - fixups.token_map, - fixups.next_id, - fixups.replace, - fixups.append, - ); - - if loc.def.is_proc_macro() { - // proc macros expect their inputs without parentheses, MBEs expect it with them included - tt.delimiter = tt::Delimiter::unspecified(); - } - let val = Some(Arc::new((tt, tmap, fixups.undo_info))); - match err { - Some(err) => ValueResult::new(val, err), - None => ValueResult::ok(val), + if matches!(loc.def.kind, MacroDefKind::BuiltInEager(..)) { + match parse.errors() { + [] => ValueResult::ok(Some(Arc::new(tt))), + errors => ValueResult::new( + Some(Arc::new(tt)), + // Box::<[_]>::from(res.errors()), not stable yet + Arc::new(errors.to_vec().into_boxed_slice()), + ), + } + } else { + ValueResult::ok(Some(Arc::new(tt))) + } } } +// FIXME: Censoring info should be calculated by the caller! Namely by name resolution /// Certain macro calls expect some nodes in the input to be preprocessed away, namely: /// - derives expect all `#[derive(..)]` invocations up to the currently invoked one to be stripped /// - attributes expect the invoking attribute to be stripped -fn censor_for_macro_input(loc: &MacroCallLoc, node: &SyntaxNode) -> FxHashSet { +fn censor_for_macro_input(loc: &MacroCallLoc, node: &SyntaxNode) -> Vec { // FIXME: handle `cfg_attr` (|| { let censor = match loc.kind { @@ -417,103 +437,56 @@ fn censor_for_macro_input(loc: &MacroCallLoc, node: &SyntaxNode) -> FxHashSet ValueResult, Arc>> { - let err = || -> Arc> { - Arc::new(Box::new([SyntaxError::new_at_offset( - "invalid macro call".to_owned(), - syntax::TextSize::from(0), - )])) - }; - let loc = db.lookup_intern_macro_call(id); - let arg = if let MacroDefKind::BuiltInEager(..) = loc.def.kind { - let res = if let Some(EagerCallInfo { arg, .. }) = loc.eager.as_deref() { - Some(mbe::token_tree_to_syntax_node(&arg.0, mbe::TopEntryPoint::MacroEagerInput).0) - } else { - loc.kind - .arg(db) - .and_then(|arg| ast::TokenTree::cast(arg.value)) - .map(|tt| tt.reparse_as_comma_separated_expr().to_syntax()) - }; - match res { - Some(res) if res.errors().is_empty() => res.syntax_node(), - Some(res) => { - return ValueResult::new( - Some(res.syntax_node().green().into()), - // Box::<[_]>::from(res.errors()), not stable yet - Arc::new(res.errors().to_vec().into_boxed_slice()), - ); - } - None => return ValueResult::only_err(err()), - } - } else { - match loc.kind.arg(db) { - Some(res) => res.value, - None => return ValueResult::only_err(err()), - } - }; - if matches!(loc.kind, MacroCallKind::FnLike { .. }) { - let first = arg.first_child_or_token().map_or(T![.], |it| it.kind()); - let last = arg.last_child_or_token().map_or(T![.], |it| it.kind()); - let well_formed_tt = - matches!((first, last), (T!['('], T![')']) | (T!['['], T![']']) | (T!['{'], T!['}'])); - if !well_formed_tt { - // Don't expand malformed (unbalanced) macro invocations. This is - // less than ideal, but trying to expand unbalanced macro calls - // sometimes produces pathological, deeply nested code which breaks - // all kinds of things. - // - // Some day, we'll have explicit recursion counters for all - // recursive things, at which point this code might be removed. - cov_mark::hit!(issue9358_bad_macro_stack_overflow); - return ValueResult::only_err(Arc::new(Box::new([SyntaxError::new( - "unbalanced token tree".to_owned(), - arg.text_range(), - )]))); - } - } - ValueResult::ok(Some(arg.green().into())) -} - fn decl_macro_expander( db: &dyn ExpandDatabase, def_crate: CrateId, id: AstId, ) -> Arc { let is_2021 = db.crate_graph()[def_crate].edition >= Edition::Edition2021; - let (mac, def_site_token_map) = match id.to_node(db) { + let (root, map) = match id.file_id.repr() { + HirFileIdRepr::FileId(file_id) => { + (db.parse(file_id).syntax_node(), Arc::new(Default::default())) + } + HirFileIdRepr::MacroFile(macro_file) => { + let (parse, map) = db.parse_macro_expansion(macro_file).value; + (parse.syntax_node(), map) + } + }; + let mac = match id.to_ptr(db).to_node(&root) { ast::Macro::MacroRules(macro_rules) => match macro_rules.token_tree() { Some(arg) => { - let (tt, def_site_token_map) = mbe::syntax_node_to_token_tree(arg.syntax()); + let tt = mbe::syntax_node_to_token_tree( + arg.syntax(), + SpanAnchor { file_id: id.file_id, ast_id: id.value.erase() }, + macro_rules.syntax().text_range().start(), + &map, + ); let mac = mbe::DeclarativeMacro::parse_macro_rules(&tt, is_2021); - (mac, def_site_token_map) + mac } - None => ( - mbe::DeclarativeMacro::from_err( - mbe::ParseError::Expected("expected a token tree".into()), - is_2021, - ), - Default::default(), + None => mbe::DeclarativeMacro::from_err( + mbe::ParseError::Expected("expected a token tree".into()), + is_2021, ), }, ast::Macro::MacroDef(macro_def) => match macro_def.body() { Some(arg) => { - let (tt, def_site_token_map) = mbe::syntax_node_to_token_tree(arg.syntax()); + let tt = mbe::syntax_node_to_token_tree( + arg.syntax(), + SpanAnchor { file_id: id.file_id, ast_id: id.value.erase() }, + macro_def.syntax().text_range().start(), + &map, + ); let mac = mbe::DeclarativeMacro::parse_macro2(&tt, is_2021); - (mac, def_site_token_map) + mac } - None => ( - mbe::DeclarativeMacro::from_err( - mbe::ParseError::Expected("expected a token tree".into()), - is_2021, - ), - Default::default(), + None => mbe::DeclarativeMacro::from_err( + mbe::ParseError::Expected("expected a token tree".into()), + is_2021, ), }, }; - Arc::new(DeclarativeMacroExpander { mac, def_site_token_map }) + Arc::new(DeclarativeMacroExpander { mac }) } fn macro_expander(db: &dyn ExpandDatabase, id: MacroDefId) -> TokenExpander { @@ -536,25 +509,37 @@ fn macro_expand(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult return db.expand_proc_macro(id), MacroDefKind::BuiltInDerive(expander, ..) => { - let arg = db.macro_arg_node(id).value.unwrap(); - - let node = SyntaxNode::new_root(arg); - let censor = censor_for_macro_input(&loc, &node); - let mut fixups = fixup::fixup_syntax(&node); - fixups.replace.extend(censor.into_iter().map(|node| (node.into(), Vec::new()))); - let (tmap, _) = mbe::syntax_node_to_token_map_with_modifications( + // FIXME: add firewall query for this? + let hir_file_id = loc.kind.file_id(); + let (root, map) = match hir_file_id.repr() { + HirFileIdRepr::FileId(file_id) => (db.parse(file_id).syntax_node(), None), + HirFileIdRepr::MacroFile(macro_file) => { + let (parse, map) = db.parse_macro_expansion(macro_file).value; + (parse.syntax_node(), Some(map)) + } + }; + let MacroCallKind::Derive { ast_id, .. } = loc.kind else { unreachable!() }; + let node = ast_id.to_ptr(db).to_node(&root); + + // FIXME: we might need to remove the spans from the input to the derive macro here + let _censor = censor_for_macro_input(&loc, node.syntax()); + let _t; + expander.expand( + db, + id, &node, - fixups.token_map, - fixups.next_id, - fixups.replace, - fixups.append, - ); - - // this cast is a bit sus, can we avoid losing the typedness here? - let adt = ast::Adt::cast(node).unwrap(); - let mut res = expander.expand(db, id, &adt, &tmap); - fixup::reverse_fixups(&mut res.value, &tmap, &fixups.undo_info); - res + match &map { + Some(map) => map, + None => { + _t = map_from_syntax_node( + node.syntax(), + SpanAnchor { file_id: hir_file_id, ast_id: ast_id.value.erase() }, + node.syntax().text_range().start(), + ); + &_t + } + }, + ) } _ => { let ValueResult { value, err } = db.macro_arg(id); @@ -570,8 +555,8 @@ fn macro_expand(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult { db.decl_macro_expander(loc.def.krate, id).expand(arg.clone()) } @@ -583,11 +568,8 @@ fn macro_expand(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult { - let mut arg = arg.clone(); - fixup::reverse_fixups(&mut arg, arg_tm, undo_info); - return ExpandResult { - value: Arc::new(arg), + value: Arc::new(arg.clone()), err: err.map(|err| { let mut buf = String::new(); for err in &**err { @@ -603,9 +585,7 @@ fn macro_expand(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult it.expand(db, id, &arg).map_err(Into::into), MacroDefKind::BuiltInAttr(it, _) => it.expand(db, id, &arg), _ => unreachable!(), - }; - fixup::reverse_fixups(&mut res.value, arg_tm, undo_info); - res + } } }; @@ -626,6 +606,7 @@ fn macro_expand(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult ExpandResult> { + // FIXME: Syntax fix ups let loc = db.lookup_intern_macro_call(id); let Some(macro_arg) = db.macro_arg(id).value else { return ExpandResult { @@ -639,32 +620,24 @@ fn expand_proc_macro(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult expander, _ => unreachable!(), }; let attr_arg = match &loc.kind { - MacroCallKind::Attr { attr_args, .. } => { - let mut attr_args = attr_args.0.clone(); - mbe::Shift::new(arg_tt).shift_all(&mut attr_args); - Some(attr_args) - } + MacroCallKind::Attr { attr_args, .. } => Some(&**attr_args), _ => None, }; - let ExpandResult { value: mut tt, err } = - expander.expand(db, loc.def.krate, loc.krate, arg_tt, attr_arg.as_ref()); + let ExpandResult { value: tt, err } = + expander.expand(db, loc.def.krate, loc.krate, ¯o_arg, attr_arg); // Set a hard limit for the expanded tt if let Err(value) = check_tt_count(&tt) { return value; } - fixup::reverse_fixups(&mut tt, arg_tm, undo_info); - ExpandResult { value: Arc::new(tt), err } } @@ -677,9 +650,10 @@ fn macro_expand_to(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandTo { } fn token_tree_to_syntax_node( + db: &dyn ExpandDatabase, tt: &tt::Subtree, expand_to: ExpandTo, -) -> (Parse, mbe::TokenMap) { +) -> (Parse, SpanMap) { let entry_point = match expand_to { ExpandTo::Statements => mbe::TopEntryPoint::MacroStmts, ExpandTo::Items => mbe::TopEntryPoint::MacroItems, @@ -687,7 +661,18 @@ fn token_tree_to_syntax_node( ExpandTo::Type => mbe::TopEntryPoint::Type, ExpandTo::Expr => mbe::TopEntryPoint::Expr, }; - mbe::token_tree_to_syntax_node(tt, entry_point) + let mut tm = mbe::token_tree_to_syntax_node(tt, entry_point); + // now what the hell is going on here + tm.1.span_map.sort_by(|(_, a), (_, b)| { + a.anchor.file_id.cmp(&b.anchor.file_id).then_with(|| { + let map = db.ast_id_map(a.anchor.file_id); + map.get_raw(a.anchor.ast_id) + .text_range() + .start() + .cmp(&map.get_raw(b.anchor.ast_id).text_range().start()) + }) + }); + tm } fn check_tt_count(tt: &tt::Subtree) -> Result<(), ExpandResult>> { diff --git a/crates/hir-expand/src/eager.rs b/crates/hir-expand/src/eager.rs index 4110f2847592..ae5f26b5d36a 100644 --- a/crates/hir-expand/src/eager.rs +++ b/crates/hir-expand/src/eager.rs @@ -18,8 +18,11 @@ //! //! //! See the full discussion : -use base_db::CrateId; -use rustc_hash::{FxHashMap, FxHashSet}; +use base_db::{ + span::{SpanAnchor, ROOT_ERASED_FILE_AST_ID}, + CrateId, +}; +use rustc_hash::FxHashMap; use syntax::{ted, Parse, SyntaxNode, TextRange, TextSize, WalkEvent}; use triomphe::Arc; @@ -29,7 +32,7 @@ use crate::{ hygiene::Hygiene, mod_path::ModPath, EagerCallInfo, ExpandError, ExpandResult, ExpandTo, InFile, MacroCallId, MacroCallKind, - MacroCallLoc, MacroDefId, MacroDefKind, + MacroCallLoc, MacroDefId, MacroDefKind, SpanMap, }; pub fn expand_eager_macro_input( @@ -54,15 +57,15 @@ pub fn expand_eager_macro_input( eager: None, kind: MacroCallKind::FnLike { ast_id: call_id, expand_to: ExpandTo::Expr }, }); - let ExpandResult { value: (arg_exp, arg_exp_map), err: parse_err } = + let ExpandResult { value: (arg_exp, _arg_exp_map), err: parse_err } = db.parse_macro_expansion(arg_id.as_macro_file()); // we need this map here as the expansion of the eager input fake file loses whitespace ... - let mut ws_mapping = FxHashMap::default(); - if let Some((_, tm, _)) = db.macro_arg(arg_id).value.as_deref() { - ws_mapping.extend(tm.entries().filter_map(|(id, range)| { - Some((arg_exp_map.first_range_by_token(id, syntax::SyntaxKind::TOMBSTONE)?, range)) - })); - } + // let mut ws_mapping = FxHashMap::default(); + // if let Some((tm)) = db.macro_arg(arg_id).value.as_deref() { + // ws_mapping.extend(tm.entries().filter_map(|(id, range)| { + // Some((arg_exp_map.first_range_by_token(id, syntax::SyntaxKind::TOMBSTONE)?, range)) + // })); + // } let ExpandResult { value: expanded_eager_input, err } = { eager_macro_recur( @@ -75,49 +78,55 @@ pub fn expand_eager_macro_input( }; let err = parse_err.or(err); - let Some((expanded_eager_input, mapping)) = expanded_eager_input else { + let Some((expanded_eager_input, _mapping)) = expanded_eager_input else { return ExpandResult { value: None, err }; }; - let (mut subtree, expanded_eager_input_token_map) = - mbe::syntax_node_to_token_tree(&expanded_eager_input); + let mut subtree = mbe::syntax_node_to_token_tree( + &expanded_eager_input, + // is this right? + SpanAnchor { file_id: arg_id.as_file(), ast_id: ROOT_ERASED_FILE_AST_ID }, + TextSize::new(0), + // FIXME: Spans! `eager_macro_recur` needs to fill out a span map for us + &Default::default(), + ); - let og_tmap = if let Some(tt) = macro_call.value.token_tree() { - let mut ids_used = FxHashSet::default(); - let mut og_tmap = mbe::syntax_node_to_token_map(tt.syntax()); - // The tokenmap and ids of subtree point into the expanded syntax node, but that is inaccessible from the outside - // so we need to remap them to the original input of the eager macro. - subtree.visit_ids(&mut |id| { - // Note: we discard all token ids of braces and the like here, but that's not too bad and only a temporary fix + // let og_tmap = if let Some(tt) = macro_call.value.token_tree() { + // let mut ids_used = FxHashSet::default(); + // let mut og_tmap = mbe::syntax_node_to_token_map(tt.syntax()); + // // The tokenmap and ids of subtree point into the expanded syntax node, but that is inaccessible from the outside + // // so we need to remap them to the original input of the eager macro. + // subtree.visit_ids(&mut |id| { + // // Note: we discard all token ids of braces and the like here, but that's not too bad and only a temporary fix - if let Some(range) = expanded_eager_input_token_map - .first_range_by_token(id, syntax::SyntaxKind::TOMBSTONE) - { - // remap from expanded eager input to eager input expansion - if let Some(og_range) = mapping.get(&range) { - // remap from eager input expansion to original eager input - if let Some(&og_range) = ws_mapping.get(og_range) { - if let Some(og_token) = og_tmap.token_by_range(og_range) { - ids_used.insert(og_token); - return og_token; - } - } - } - } - tt::TokenId::UNSPECIFIED - }); - og_tmap.filter(|id| ids_used.contains(&id)); - og_tmap - } else { - Default::default() - }; - subtree.delimiter = crate::tt::Delimiter::unspecified(); + // if let Some(range) = expanded_eager_input_token_map + // .first_range_by_token(id, syntax::SyntaxKind::TOMBSTONE) + // { + // // remap from expanded eager input to eager input expansion + // if let Some(og_range) = mapping.get(&range) { + // // remap from eager input expansion to original eager input + // if let Some(&og_range) = ws_mapping.get(og_range) { + // if let Some(og_token) = og_tmap.token_by_range(og_range) { + // ids_used.insert(og_token); + // return og_token; + // } + // } + // } + // } + // tt::TokenId::UNSPECIFIED + // }); + // og_tmap.filter(|id| ids_used.contains(&id)); + // og_tmap + // } else { + // Default::default() + // }; + subtree.delimiter = crate::tt::Delimiter::UNSPECIFIED; let loc = MacroCallLoc { def, krate, eager: Some(Box::new(EagerCallInfo { - arg: Arc::new((subtree, og_tmap)), + arg: Arc::new((subtree,)), arg_id, error: err.clone(), })), @@ -132,7 +141,7 @@ fn lazy_expand( def: &MacroDefId, macro_call: InFile, krate: CrateId, -) -> ExpandResult<(InFile>, Arc)> { +) -> ExpandResult<(InFile>, Arc)> { let ast_id = db.ast_id_map(macro_call.file_id).ast_id(¯o_call.value); let expand_to = ExpandTo::from_call_site(¯o_call.value); @@ -214,19 +223,19 @@ fn eager_macro_recur( let ExpandResult { value, err: err2 } = db.parse_macro_expansion(call_id.as_macro_file()); - if let Some(tt) = call.token_tree() { - let call_tt_start = tt.syntax().text_range().start(); - let call_start = - apply_offset(call.syntax().text_range().start(), offset); - if let Some((_, arg_map, _)) = db.macro_arg(call_id).value.as_deref() { - mapping.extend(arg_map.entries().filter_map(|(tid, range)| { - value - .1 - .first_range_by_token(tid, syntax::SyntaxKind::TOMBSTONE) - .map(|r| (r + call_start, range + call_tt_start)) - })); - } - } + // if let Some(tt) = call.token_tree() { + // let call_tt_start = tt.syntax().text_range().start(); + // let call_start = + // apply_offset(call.syntax().text_range().start(), offset); + // if let Some((_, arg_map, _)) = db.macro_arg(call_id).value.as_deref() { + // mapping.extend(arg_map.entries().filter_map(|(tid, range)| { + // value + // .1 + // .first_range_by_token(tid, syntax::SyntaxKind::TOMBSTONE) + // .map(|r| (r + call_start, range + call_tt_start)) + // })); + // } + // } ExpandResult { value: Some(value.0.syntax_node().clone_for_update()), @@ -241,13 +250,8 @@ fn eager_macro_recur( | MacroDefKind::BuiltInAttr(..) | MacroDefKind::BuiltInDerive(..) | MacroDefKind::ProcMacro(..) => { - let ExpandResult { value: (parse, tm), err } = + let ExpandResult { value: (parse, _tm), err } = lazy_expand(db, &def, curr.with_value(call.clone()), krate); - let decl_mac = if let MacroDefKind::Declarative(ast_id) = def.kind { - Some(db.decl_macro_expander(def.krate, ast_id)) - } else { - None - }; // replace macro inside let hygiene = Hygiene::new(db, parse.file_id); @@ -261,24 +265,29 @@ fn eager_macro_recur( ); let err = err.or(error); - if let Some(tt) = call.token_tree() { - let call_tt_start = tt.syntax().text_range().start(); - let call_start = apply_offset(call.syntax().text_range().start(), offset); - if let Some((_tt, arg_map, _)) = parse - .file_id - .macro_file() - .and_then(|id| db.macro_arg(id.macro_call_id).value) - .as_deref() - { - mapping.extend(arg_map.entries().filter_map(|(tid, range)| { - tm.first_range_by_token( - decl_mac.as_ref().map(|it| it.map_id_down(tid)).unwrap_or(tid), - syntax::SyntaxKind::TOMBSTONE, - ) - .map(|r| (r + call_start, range + call_tt_start)) - })); - } - } + // if let Some(tt) = call.token_tree() { + // let decl_mac = if let MacroDefKind::Declarative(ast_id) = def.kind { + // Some(db.decl_macro_expander(def.krate, ast_id)) + // } else { + // None + // }; + // let call_tt_start = tt.syntax().text_range().start(); + // let call_start = apply_offset(call.syntax().text_range().start(), offset); + // if let Some((_tt, arg_map, _)) = parse + // .file_id + // .macro_file() + // .and_then(|id| db.macro_arg(id.macro_call_id).value) + // .as_deref() + // { + // mapping.extend(arg_map.entries().filter_map(|(tid, range)| { + // tm.first_range_by_token( + // decl_mac.as_ref().map(|it| it.map_id_down(tid)).unwrap_or(tid), + // syntax::SyntaxKind::TOMBSTONE, + // ) + // .map(|r| (r + call_start, range + call_tt_start)) + // })); + // } + // } // FIXME: Do we need to re-use _m here? ExpandResult { value: value.map(|(n, _m)| n), err } } diff --git a/crates/hir-expand/src/hygiene.rs b/crates/hir-expand/src/hygiene.rs index ca65db1136ce..ce421d3dcd82 100644 --- a/crates/hir-expand/src/hygiene.rs +++ b/crates/hir-expand/src/hygiene.rs @@ -5,18 +5,16 @@ use base_db::CrateId; use db::TokenExpander; use either::Either; -use mbe::Origin; use syntax::{ ast::{self, HasDocComments}, - AstNode, SyntaxKind, SyntaxNode, TextRange, TextSize, + AstNode, SyntaxNode, TextRange, TextSize, }; use triomphe::Arc; use crate::{ db::{self, ExpandDatabase}, - fixup, name::{AsName, Name}, - HirFileId, InFile, MacroCallKind, MacroCallLoc, MacroDefKind, MacroFile, + HirFileId, InFile, MacroCallKind, MacroCallLoc, MacroDefKind, MacroFile, SpanMap, }; #[derive(Clone, Debug)] @@ -50,23 +48,25 @@ impl Hygiene { Either::Left(name_ref.as_name()) } - pub fn local_inner_macros(&self, db: &dyn ExpandDatabase, path: ast::Path) -> Option { - let mut token = path.syntax().first_token()?.text_range(); + pub fn local_inner_macros(&self, _db: &dyn ExpandDatabase, path: ast::Path) -> Option { + let mut _token = path.syntax().first_token()?.text_range(); let frames = self.frames.as_ref()?; - let mut current = &frames.0; - - loop { - let (mapped, origin) = current.expansion.as_ref()?.map_ident_up(db, token)?; - if origin == Origin::Def { - return if current.local_inner { - frames.root_crate(db, path.syntax()) - } else { - None - }; - } - current = current.call_site.as_ref()?; - token = mapped.value; - } + let mut _current = &frames.0; + + // FIXME: Hygiene ... + return None; + // loop { + // let (mapped, origin) = current.expansion.as_ref()?.map_ident_up(db, token)?; + // if origin == Origin::Def { + // return if current.local_inner { + // frames.root_crate(db, path.syntax()) + // } else { + // None + // }; + // } + // current = current.call_site.as_ref()?; + // token = mapped.value; + // } } } @@ -92,31 +92,33 @@ impl HygieneFrames { HygieneFrames(Arc::new(HygieneFrame::new(db, file_id))) } - fn root_crate(&self, db: &dyn ExpandDatabase, node: &SyntaxNode) -> Option { - let mut token = node.first_token()?.text_range(); - let mut result = self.0.krate; - let mut current = self.0.clone(); + fn root_crate(&self, _db: &dyn ExpandDatabase, node: &SyntaxNode) -> Option { + let mut _token = node.first_token()?.text_range(); + let mut _result = self.0.krate; + let mut _current = self.0.clone(); - while let Some((mapped, origin)) = - current.expansion.as_ref().and_then(|it| it.map_ident_up(db, token)) - { - result = current.krate; + return None; - let site = match origin { - Origin::Def => ¤t.def_site, - Origin::Call => ¤t.call_site, - }; + // while let Some((mapped, origin)) = + // current.expansion.as_ref().and_then(|it| it.map_ident_up(db, token)) + // { + // result = current.krate; - let site = match site { - None => break, - Some(it) => it, - }; + // let site = match origin { + // Origin::Def => ¤t.def_site, + // Origin::Call => ¤t.call_site, + // }; - current = site.clone(); - token = mapped.value; - } + // let site = match site { + // None => break, + // Some(it) => it, + // }; + + // current = site.clone(); + // token = mapped.value; + // } - result + // result } } @@ -127,45 +129,18 @@ struct HygieneInfo { attr_input_or_mac_def_start: Option>, macro_def: TokenExpander, - macro_arg: Arc<(crate::tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>, - macro_arg_shift: mbe::Shift, - exp_map: Arc, + macro_arg: Arc, + exp_map: Arc, } impl HygieneInfo { - fn map_ident_up( + fn _map_ident_up( &self, - db: &dyn ExpandDatabase, - token: TextRange, - ) -> Option<(InFile, Origin)> { - let token_id = self.exp_map.token_by_range(token)?; - let (mut token_id, origin) = self.macro_def.map_id_up(token_id); - - let loc = db.lookup_intern_macro_call(self.file.macro_call_id); - - let (token_map, tt) = match &loc.kind { - MacroCallKind::Attr { attr_args, .. } => match self.macro_arg_shift.unshift(token_id) { - Some(unshifted) => { - token_id = unshifted; - (&attr_args.1, self.attr_input_or_mac_def_start?) - } - None => (&self.macro_arg.1, loc.kind.arg(db)?.map(|it| it.text_range().start())), - }, - _ => match origin { - mbe::Origin::Call => { - (&self.macro_arg.1, loc.kind.arg(db)?.map(|it| it.text_range().start())) - } - mbe::Origin::Def => match (&self.macro_def, &self.attr_input_or_mac_def_start) { - (TokenExpander::DeclarativeMacro(expander), Some(tt)) => { - (&expander.def_site_token_map, *tt) - } - _ => panic!("`Origin::Def` used with non-`macro_rules!` macro"), - }, - }, - }; - - let range = token_map.first_range_by_token(token_id, SyntaxKind::IDENT)?; - Some((tt.with_value(range + tt.value), origin)) + _db: &dyn ExpandDatabase, + _token: TextRange, + ) -> Option> { + // self.exp_map.token_by_range(token).map(|span| InFile::new(span.anchor, span.range)) + None } } @@ -197,18 +172,13 @@ fn make_hygiene_info( let macro_def = db.macro_expander(loc.def); let (_, exp_map) = db.parse_macro_expansion(macro_file).value; let macro_arg = db.macro_arg(macro_file.macro_call_id).value.unwrap_or_else(|| { - Arc::new(( - tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: Vec::new() }, - Default::default(), - Default::default(), - )) + Arc::new(tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: Vec::new() }) }); HygieneInfo { file: macro_file, attr_input_or_mac_def_start: attr_input_or_mac_def .map(|it| it.map(|tt| tt.syntax().text_range().start())), - macro_arg_shift: mbe::Shift::new(¯o_arg.0), macro_arg, macro_def, exp_map, diff --git a/crates/hir-expand/src/lib.rs b/crates/hir-expand/src/lib.rs index 4be55126b862..bd5796e000ac 100644 --- a/crates/hir-expand/src/lib.rs +++ b/crates/hir-expand/src/lib.rs @@ -18,21 +18,13 @@ pub mod quote; pub mod eager; pub mod mod_path; pub mod attrs; -mod fixup; +// mod fixup; -use mbe::TokenMap; -pub use mbe::{Origin, ValueResult}; - -use ::tt::token_id as tt; use triomphe::Arc; use std::{fmt, hash::Hash, iter}; -use base_db::{ - impl_intern_key, - salsa::{self, InternId}, - CrateId, FileId, FileRange, ProcMacroKind, -}; +use base_db::{span::HirFileIdRepr, CrateId, FileId, FileRange, ProcMacroKind}; use either::Either; use syntax::{ algo::{self, skip_trivia_token}, @@ -51,6 +43,25 @@ use crate::{ proc_macro::ProcMacroExpander, }; +pub use base_db::span::{HirFileId, MacroCallId, MacroFile}; +pub use mbe::ValueResult; + +pub type SpanMap = ::mbe::TokenMap; +pub type DeclarativeMacro = ::mbe::DeclarativeMacro; + +pub mod tt { + pub use base_db::span::SpanData; + pub use tt::{DelimiterKind, Spacing, Span}; + + pub type Delimiter = ::tt::Delimiter; + pub type Subtree = ::tt::Subtree; + pub type Leaf = ::tt::Leaf; + pub type Literal = ::tt::Literal; + pub type Punct = ::tt::Punct; + pub type Ident = ::tt::Ident; + pub type TokenTree = ::tt::TokenTree; +} + pub type ExpandResult = ValueResult; #[derive(Debug, PartialEq, Eq, Clone, Hash)] @@ -86,42 +97,43 @@ impl fmt::Display for ExpandError { } } -/// Input to the analyzer is a set of files, where each file is identified by -/// `FileId` and contains source code. However, another source of source code in -/// Rust are macros: each macro can be thought of as producing a "temporary -/// file". To assign an id to such a file, we use the id of the macro call that -/// produced the file. So, a `HirFileId` is either a `FileId` (source code -/// written by user), or a `MacroCallId` (source code produced by macro). -/// -/// What is a `MacroCallId`? Simplifying, it's a `HirFileId` of a file -/// containing the call plus the offset of the macro call in the file. Note that -/// this is a recursive definition! However, the size_of of `HirFileId` is -/// finite (because everything bottoms out at the real `FileId`) and small -/// (`MacroCallId` uses the location interning. You can check details here: -/// ). -/// -/// The two variants are encoded in a single u32 which are differentiated by the MSB. -/// If the MSB is 0, the value represents a `FileId`, otherwise the remaining 31 bits represent a -/// `MacroCallId`. -#[derive(Clone, Copy, PartialEq, Eq, Hash)] -pub struct HirFileId(u32); - -impl fmt::Debug for HirFileId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.repr().fmt(f) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct MacroFile { - pub macro_call_id: MacroCallId, -} - /// `MacroCallId` identifies a particular macro invocation, like /// `println!("Hello, {}", world)`. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct MacroCallId(salsa::InternId); -impl_intern_key!(MacroCallId); +pub struct SyntaxContextId(base_db::salsa::InternId); +base_db::impl_intern_key!(SyntaxContextId); + +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct SyntaxContext { + outer_expn: HirFileId, + outer_transparency: Transparency, + parent: SyntaxContextId, + /// This context, but with all transparent and semi-transparent expansions filtered away. + opaque: SyntaxContextId, + /// This context, but with all transparent expansions filtered away. + opaque_and_semitransparent: SyntaxContextId, + /// Name of the crate to which `$crate` with this context would resolve. + dollar_crate_name: name::Name, +} + +/// A property of a macro expansion that determines how identifiers +/// produced by that expansion are resolved. +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Hash, Debug)] +pub enum Transparency { + /// Identifier produced by a transparent expansion is always resolved at call-site. + /// Call-site spans in procedural macros, hygiene opt-out in `macro` should use this. + Transparent, + /// Identifier produced by a semi-transparent expansion may be resolved + /// either at call-site or at definition-site. + /// If it's a local variable, label or `$crate` then it's resolved at def-site. + /// Otherwise it's resolved at call-site. + /// `macro_rules` macros behave like this, built-in macros currently behave like this too, + /// but that's an implementation detail. + SemiTransparent, + /// Identifier produced by an opaque expansion is always resolved at definition-site. + /// Def-site spans in procedural macros, identifiers from `macro` by default use this. + Opaque, +} #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct MacroCallLoc { @@ -154,7 +166,7 @@ pub enum MacroDefKind { #[derive(Debug, Clone, PartialEq, Eq, Hash)] struct EagerCallInfo { /// The expanded argument of the eager macro. - arg: Arc<(tt::Subtree, TokenMap)>, + arg: Arc<(tt::Subtree,)>, /// Call id of the eager macro's input file (this is the macro file for its fully expanded input). arg_id: MacroCallId, error: Option, @@ -178,7 +190,7 @@ pub enum MacroCallKind { }, Attr { ast_id: AstId, - attr_args: Arc<(tt::Subtree, mbe::TokenMap)>, + attr_args: Arc, /// Syntactical index of the invoking `#[attribute]`. /// /// Outer attributes are counted first, then inner attributes. This does not support @@ -187,34 +199,40 @@ pub enum MacroCallKind { }, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -enum HirFileIdRepr { - FileId(FileId), - MacroFile(MacroFile), -} +pub trait HirFileIdExt { + /// For macro-expansion files, returns the file original source file the + /// expansion originated from. + fn original_file(self, db: &dyn db::ExpandDatabase) -> FileId; + fn expansion_level(self, db: &dyn db::ExpandDatabase) -> u32; -impl From for HirFileId { - fn from(FileId(id): FileId) -> Self { - assert!(id < Self::MAX_FILE_ID); - HirFileId(id) - } -} + /// If this is a macro call, returns the syntax node of the call. + fn call_node(self, db: &dyn db::ExpandDatabase) -> Option>; -impl From for HirFileId { - fn from(MacroFile { macro_call_id: MacroCallId(id) }: MacroFile) -> Self { - let id = id.as_u32(); - assert!(id < Self::MAX_FILE_ID); - HirFileId(id | Self::MACRO_FILE_TAG_MASK) - } -} + /// If this is a macro call, returns the syntax node of the very first macro call this file resides in. + fn original_call_node(self, db: &dyn db::ExpandDatabase) -> Option<(FileId, SyntaxNode)>; -impl HirFileId { - const MAX_FILE_ID: u32 = u32::MAX ^ Self::MACRO_FILE_TAG_MASK; - const MACRO_FILE_TAG_MASK: u32 = 1 << 31; + /// Return expansion information if it is a macro-expansion file + fn expansion_info(self, db: &dyn db::ExpandDatabase) -> Option; - /// For macro-expansion files, returns the file original source file the - /// expansion originated from. - pub fn original_file(self, db: &dyn db::ExpandDatabase) -> FileId { + fn as_builtin_derive_attr_node(&self, db: &dyn db::ExpandDatabase) + -> Option>; + fn is_builtin_derive(&self, db: &dyn db::ExpandDatabase) -> bool; + fn is_custom_derive(&self, db: &dyn db::ExpandDatabase) -> bool; + + /// Return whether this file is an include macro + fn is_include_macro(&self, db: &dyn db::ExpandDatabase) -> bool; + + fn is_eager(&self, db: &dyn db::ExpandDatabase) -> bool; + /// Return whether this file is an attr macro + fn is_attr_macro(&self, db: &dyn db::ExpandDatabase) -> bool; + + /// Return whether this file is the pseudo expansion of the derive attribute. + /// See [`crate::builtin_attr_macro::derive_attr_expand`]. + fn is_derive_attr_pseudo_expansion(&self, db: &dyn db::ExpandDatabase) -> bool; +} + +impl HirFileIdExt for HirFileId { + fn original_file(self, db: &dyn db::ExpandDatabase) -> FileId { let mut file_id = self; loop { match file_id.repr() { @@ -231,7 +249,7 @@ impl HirFileId { } } - pub fn expansion_level(self, db: &dyn db::ExpandDatabase) -> u32 { + fn expansion_level(self, db: &dyn db::ExpandDatabase) -> u32 { let mut level = 0; let mut curr = self; while let Some(macro_file) = curr.macro_file() { @@ -243,15 +261,13 @@ impl HirFileId { level } - /// If this is a macro call, returns the syntax node of the call. - pub fn call_node(self, db: &dyn db::ExpandDatabase) -> Option> { + fn call_node(self, db: &dyn db::ExpandDatabase) -> Option> { let macro_file = self.macro_file()?; let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_file.macro_call_id); Some(loc.to_node(db)) } - /// If this is a macro call, returns the syntax node of the very first macro call this file resides in. - pub fn original_call_node(self, db: &dyn db::ExpandDatabase) -> Option<(FileId, SyntaxNode)> { + fn original_call_node(self, db: &dyn db::ExpandDatabase) -> Option<(FileId, SyntaxNode)> { let mut call = db.lookup_intern_macro_call(self.macro_file()?.macro_call_id).to_node(db); loop { match call.file_id.repr() { @@ -264,12 +280,12 @@ impl HirFileId { } /// Return expansion information if it is a macro-expansion file - pub fn expansion_info(self, db: &dyn db::ExpandDatabase) -> Option { + fn expansion_info(self, db: &dyn db::ExpandDatabase) -> Option { let macro_file = self.macro_file()?; ExpansionInfo::new(db, macro_file) } - pub fn as_builtin_derive_attr_node( + fn as_builtin_derive_attr_node( &self, db: &dyn db::ExpandDatabase, ) -> Option> { @@ -282,7 +298,7 @@ impl HirFileId { Some(attr.with_value(ast::Attr::cast(attr.value.clone())?)) } - pub fn is_custom_derive(&self, db: &dyn db::ExpandDatabase) -> bool { + fn is_custom_derive(&self, db: &dyn db::ExpandDatabase) -> bool { match self.macro_file() { Some(macro_file) => { matches!( @@ -294,7 +310,7 @@ impl HirFileId { } } - pub fn is_builtin_derive(&self, db: &dyn db::ExpandDatabase) -> bool { + fn is_builtin_derive(&self, db: &dyn db::ExpandDatabase) -> bool { match self.macro_file() { Some(macro_file) => { matches!( @@ -306,8 +322,7 @@ impl HirFileId { } } - /// Return whether this file is an include macro - pub fn is_include_macro(&self, db: &dyn db::ExpandDatabase) -> bool { + fn is_include_macro(&self, db: &dyn db::ExpandDatabase) -> bool { match self.macro_file() { Some(macro_file) => { db.lookup_intern_macro_call(macro_file.macro_call_id).def.is_include() @@ -316,7 +331,7 @@ impl HirFileId { } } - pub fn is_eager(&self, db: &dyn db::ExpandDatabase) -> bool { + fn is_eager(&self, db: &dyn db::ExpandDatabase) -> bool { match self.macro_file() { Some(macro_file) => { let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_file.macro_call_id); @@ -326,8 +341,7 @@ impl HirFileId { } } - /// Return whether this file is an attr macro - pub fn is_attr_macro(&self, db: &dyn db::ExpandDatabase) -> bool { + fn is_attr_macro(&self, db: &dyn db::ExpandDatabase) -> bool { match self.macro_file() { Some(macro_file) => { let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_file.macro_call_id); @@ -337,9 +351,7 @@ impl HirFileId { } } - /// Return whether this file is the pseudo expansion of the derive attribute. - /// See [`crate::builtin_attr_macro::derive_attr_expand`]. - pub fn is_derive_attr_pseudo_expansion(&self, db: &dyn db::ExpandDatabase) -> bool { + fn is_derive_attr_pseudo_expansion(&self, db: &dyn db::ExpandDatabase) -> bool { match self.macro_file() { Some(macro_file) => { let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_file.macro_call_id); @@ -348,38 +360,6 @@ impl HirFileId { None => false, } } - - #[inline] - pub fn is_macro(self) -> bool { - self.0 & Self::MACRO_FILE_TAG_MASK != 0 - } - - #[inline] - pub fn macro_file(self) -> Option { - match self.0 & Self::MACRO_FILE_TAG_MASK { - 0 => None, - _ => Some(MacroFile { - macro_call_id: MacroCallId(InternId::from(self.0 ^ Self::MACRO_FILE_TAG_MASK)), - }), - } - } - - #[inline] - pub fn file_id(self) -> Option { - match self.0 & Self::MACRO_FILE_TAG_MASK { - 0 => Some(FileId(self.0)), - _ => None, - } - } - - fn repr(self) -> HirFileIdRepr { - match self.0 & Self::MACRO_FILE_TAG_MASK { - 0 => HirFileIdRepr::FileId(FileId(self.0)), - _ => HirFileIdRepr::MacroFile(MacroFile { - macro_call_id: MacroCallId(InternId::from(self.0 ^ Self::MACRO_FILE_TAG_MASK)), - }), - } - } } impl MacroDefId { @@ -587,16 +567,6 @@ impl MacroCallKind { } } -impl MacroCallId { - pub fn as_file(self) -> HirFileId { - MacroFile { macro_call_id: self }.into() - } - - pub fn as_macro_file(self) -> MacroFile { - MacroFile { macro_call_id: self } - } -} - /// ExpansionInfo mainly describes how to map text range between src and expanded macro #[derive(Debug, Clone, PartialEq, Eq)] pub struct ExpansionInfo { @@ -607,11 +577,8 @@ pub struct ExpansionInfo { attr_input_or_mac_def: Option>, macro_def: TokenExpander, - macro_arg: Arc<(tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>, - /// A shift built from `macro_arg`'s subtree, relevant for attributes as the item is the macro arg - /// and as such we need to shift tokens if they are part of an attributes input instead of their item. - macro_arg_shift: mbe::Shift, - exp_map: Arc, + macro_arg: Arc, + exp_map: Arc, } impl ExpansionInfo { @@ -640,69 +607,33 @@ impl ExpansionInfo { pub fn map_token_down( &self, db: &dyn db::ExpandDatabase, - item: Option, token: InFile<&SyntaxToken>, // FIXME: use this for range mapping, so that we can resolve inline format args _relative_token_offset: Option, ) -> Option> + '_> { assert_eq!(token.file_id, self.arg.file_id); - let token_id_in_attr_input = if let Some(item) = item { - // check if we are mapping down in an attribute input - // this is a special case as attributes can have two inputs - let call_id = self.expanded.file_id.macro_call_id; - let loc = db.lookup_intern_macro_call(call_id); - - let token_range = token.value.text_range(); - match &loc.kind { - MacroCallKind::Attr { attr_args, invoc_attr_index, .. } => { - // FIXME: handle `cfg_attr` - let attr = item - .doc_comments_and_attrs() - .nth(invoc_attr_index.ast_index()) - .and_then(Either::left)?; - match attr.token_tree() { - Some(token_tree) - if token_tree.syntax().text_range().contains_range(token_range) => - { - let attr_input_start = - token_tree.left_delimiter_token()?.text_range().start(); - let relative_range = - token.value.text_range().checked_sub(attr_input_start)?; - // shift by the item's tree's max id - let token_id = attr_args.1.token_by_range(relative_range)?; - - let token_id = if loc.def.is_attribute_derive() { - // we do not shift for `#[derive]`, as we only need to downmap the derive attribute tokens - token_id - } else { - self.macro_arg_shift.shift(token_id) - }; - Some(token_id) - } - _ => None, - } - } - _ => None, - } + let span_map = &self.exp_map.span_map; + let (start, end) = if span_map + .first() + .map_or(false, |(_, span)| span.anchor.file_id == token.file_id) + { + (0, span_map.partition_point(|a| a.1.anchor.file_id == token.file_id)) } else { - None - }; - - let token_id = match token_id_in_attr_input { - Some(token_id) => token_id, - // the token is not inside `an attribute's input so do the lookup in the macro_arg as usual - None => { - let relative_range = - token.value.text_range().checked_sub(self.arg.value.text_range().start())?; - let token_id = self.macro_arg.1.token_by_range(relative_range)?; - // conditionally shift the id by a declarative macro definition - self.macro_def.map_id_down(token_id) - } + let start = span_map.partition_point(|a| a.1.anchor.file_id != token.file_id); + ( + start, + start + span_map[start..].partition_point(|a| a.1.anchor.file_id == token.file_id), + ) }; - - let tokens = self - .exp_map - .ranges_by_token(token_id, token.value.kind()) + let token_text_range = token.value.text_range(); + let ast_id_map = db.ast_id_map(token.file_id); + let tokens = span_map[start..end] + .iter() + .filter_map(move |(range, span)| { + let offset = ast_id_map.get_raw(span.anchor.ast_id).text_range().start(); + let abs_range = span.range + offset; + token_text_range.eq(&abs_range).then_some(*range) + }) .flat_map(move |range| self.expanded.value.covering_element(range).into_token()); Some(tokens.map(move |token| InFile::new(self.expanded.file_id.into(), token))) @@ -713,60 +644,18 @@ impl ExpansionInfo { &self, db: &dyn db::ExpandDatabase, token: InFile<&SyntaxToken>, - ) -> Option<(InFile, Origin)> { - assert_eq!(token.file_id, self.expanded.file_id.into()); - // Fetch the id through its text range, - let token_id = self.exp_map.token_by_range(token.value.text_range())?; - // conditionally unshifting the id to accommodate for macro-rules def site - let (mut token_id, origin) = self.macro_def.map_id_up(token_id); - - let call_id = self.expanded.file_id.macro_call_id; - let loc = db.lookup_intern_macro_call(call_id); - - // Special case: map tokens from `include!` expansions to the included file - if loc.def.is_include() { - if let Ok((tt_and_map, file_id)) = db.include_expand(call_id) { - let range = tt_and_map.1.first_range_by_token(token_id, token.value.kind())?; - let source = db.parse(file_id); - - let token = source.syntax_node().covering_element(range).into_token()?; - - return Some((InFile::new(file_id.into(), token), Origin::Call)); - } - } - - // Attributes are a bit special for us, they have two inputs, the input tokentree and the annotated item. - let (token_map, tt) = match &loc.kind { - MacroCallKind::Attr { attr_args, .. } => { - if loc.def.is_attribute_derive() { - (&attr_args.1, self.attr_input_or_mac_def.clone()?.syntax().cloned()) - } else { - // try unshifting the token id, if unshifting fails, the token resides in the non-item attribute input - // note that the `TokenExpander::map_id_up` earlier only unshifts for declarative macros, so we don't double unshift with this - match self.macro_arg_shift.unshift(token_id) { - Some(unshifted) => { - token_id = unshifted; - (&attr_args.1, self.attr_input_or_mac_def.clone()?.syntax().cloned()) - } - None => (&self.macro_arg.1, self.arg.clone()), - } - } - } - _ => match origin { - mbe::Origin::Call => (&self.macro_arg.1, self.arg.clone()), - mbe::Origin::Def => match (&self.macro_def, &self.attr_input_or_mac_def) { - (TokenExpander::DeclarativeMacro(expander), Some(tt)) => { - (&expander.def_site_token_map, tt.syntax().cloned()) - } - _ => panic!("`Origin::Def` used with non-`macro_rules!` macro"), - }, - }, - }; - - let range = token_map.first_range_by_token(token_id, token.value.kind())?; - let token = - tt.value.covering_element(range + tt.value.text_range().start()).into_token()?; - Some((tt.with_value(token), origin)) + ) -> Option> { + self.exp_map.span_for_range(token.value.text_range()).and_then(|span| { + let anchor = + db.ast_id_map(span.anchor.file_id).get_raw(span.anchor.ast_id).text_range().start(); + InFile::new( + span.anchor.file_id, + db.parse_or_expand(span.anchor.file_id) + .covering_element(span.range + anchor) + .into_token(), + ) + .transpose() + }) } fn new(db: &dyn db::ExpandDatabase, macro_file: MacroFile) -> Option { @@ -779,11 +668,7 @@ impl ExpansionInfo { let expanded = InMacroFile { file_id: macro_file, value: parse.syntax_node() }; let macro_arg = db.macro_arg(macro_file.macro_call_id).value.unwrap_or_else(|| { - Arc::new(( - tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: Vec::new() }, - Default::default(), - Default::default(), - )) + Arc::new(tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: Vec::new() }) }); let def = loc.def.ast_id().left().and_then(|id| { @@ -814,7 +699,6 @@ impl ExpansionInfo { expanded, arg: arg_tt, attr_input_or_mac_def, - macro_arg_shift: mbe::Shift::new(¯o_arg.0), macro_arg, macro_def, exp_map, @@ -1018,7 +902,7 @@ impl InFile<&SyntaxNode> { impl InFile { pub fn upmap(self, db: &dyn db::ExpandDatabase) -> Option> { let expansion = self.file_id.expansion_info(db)?; - expansion.map_token_up(db, self.as_ref()).map(|(it, _)| it) + expansion.map_token_up(db, self.as_ref()) } /// Falls back to the macro call range if the node cannot be mapped up fully. @@ -1067,6 +951,7 @@ impl From> for InFile { } } +// FIXME: Get rid of this fn ascend_node_border_tokens( db: &dyn db::ExpandDatabase, InFile { file_id, value: node }: InFile<&SyntaxNode>, @@ -1090,13 +975,13 @@ fn ascend_call_token( token: InFile, ) -> Option> { let mut mapping = expansion.map_token_up(db, token.as_ref())?; - while let (mapped, Origin::Call) = mapping { - match mapped.file_id.expansion_info(db) { - Some(info) => mapping = info.map_token_up(db, mapped.as_ref())?, - None => return Some(mapped), + + loop { + match mapping.file_id.expansion_info(db) { + Some(info) => mapping = info.map_token_up(db, mapping.as_ref())?, + None => return Some(mapping), } } - None } impl InFile { diff --git a/crates/hir-expand/src/quote.rs b/crates/hir-expand/src/quote.rs index ab3809abc7a2..9dd4965c1506 100644 --- a/crates/hir-expand/src/quote.rs +++ b/crates/hir-expand/src/quote.rs @@ -18,8 +18,8 @@ macro_rules! __quote { crate::tt::Subtree { delimiter: crate::tt::Delimiter { kind: crate::tt::DelimiterKind::$delim, - open: crate::tt::TokenId::unspecified(), - close: crate::tt::TokenId::unspecified(), + open: ::DUMMY, + close: ::DUMMY, }, token_trees: $crate::quote::IntoTt::to_tokens(children), } @@ -32,7 +32,7 @@ macro_rules! __quote { crate::tt::Leaf::Punct(crate::tt::Punct { char: $first, spacing: crate::tt::Spacing::Alone, - span: crate::tt::TokenId::unspecified(), + span: ::DUMMY, }).into() ] } @@ -44,12 +44,12 @@ macro_rules! __quote { crate::tt::Leaf::Punct(crate::tt::Punct { char: $first, spacing: crate::tt::Spacing::Joint, - span: crate::tt::TokenId::unspecified(), + span: ::DUMMY, }).into(), crate::tt::Leaf::Punct(crate::tt::Punct { char: $sec, spacing: crate::tt::Spacing::Alone, - span: crate::tt::TokenId::unspecified(), + span: ::DUMMY, }).into() ] } @@ -89,7 +89,7 @@ macro_rules! __quote { vec![ { crate::tt::Leaf::Ident(crate::tt::Ident { text: stringify!($tt).into(), - span: crate::tt::TokenId::unspecified(), + span: ::DUMMY, }).into() }] }; @@ -195,20 +195,22 @@ macro_rules! impl_to_to_tokentrees { } impl_to_to_tokentrees! { - u32 => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} }; - usize => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} }; - i32 => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} }; - bool => self { crate::tt::Ident{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} }; + u32 => self { crate::tt::Literal{text: self.to_string().into(), span: ::DUMMY} }; + usize => self { crate::tt::Literal{text: self.to_string().into(), span: ::DUMMY} }; + i32 => self { crate::tt::Literal{text: self.to_string().into(), span: ::DUMMY} }; + bool => self { crate::tt::Ident{text: self.to_string().into(), span: ::DUMMY} }; crate::tt::Leaf => self { self }; crate::tt::Literal => self { self }; crate::tt::Ident => self { self }; crate::tt::Punct => self { self }; - &str => self { crate::tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), span: crate::tt::TokenId::unspecified()}}; - String => self { crate::tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), span: crate::tt::TokenId::unspecified()}} + &str => self { crate::tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), span: ::DUMMY}}; + String => self { crate::tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), span: ::DUMMY}} } #[cfg(test)] mod tests { + use expect_test::expect; + #[test] fn test_quote_delimiters() { assert_eq!(quote!({}).to_string(), "{}"); @@ -231,7 +233,10 @@ mod tests { } fn mk_ident(name: &str) -> crate::tt::Ident { - crate::tt::Ident { text: name.into(), span: crate::tt::TokenId::unspecified() } + crate::tt::Ident { + text: name.into(), + span: ::DUMMY, + } } #[test] @@ -241,7 +246,9 @@ mod tests { let quoted = quote!(#a); assert_eq!(quoted.to_string(), "hello"); let t = format!("{quoted:?}"); - assert_eq!(t, "SUBTREE $$ 4294967295 4294967295\n IDENT hello 4294967295"); + expect![[r#" + SUBTREE $$ SpanData { range: 0..0, anchor: SpanAnchor { file_id: FileId(0), ast_id: Idx::>(0) } } SpanData { range: 0..0, anchor: SpanAnchor { file_id: FileId(0), ast_id: Idx::>(0) } } + IDENT hello SpanData { range: 0..0, anchor: SpanAnchor { file_id: FileId(0), ast_id: Idx::>(0) } }"#]].assert_eq(&t); } #[test] @@ -273,8 +280,8 @@ mod tests { let list = crate::tt::Subtree { delimiter: crate::tt::Delimiter { kind: crate::tt::DelimiterKind::Brace, - open: crate::tt::TokenId::unspecified(), - close: crate::tt::TokenId::unspecified(), + open: ::DUMMY, + close: ::DUMMY, }, token_trees: fields.collect(), }; diff --git a/crates/hir-ty/src/diagnostics/decl_check.rs b/crates/hir-ty/src/diagnostics/decl_check.rs index f4c079b48c58..c2ff487ef91d 100644 --- a/crates/hir-ty/src/diagnostics/decl_check.rs +++ b/crates/hir-ty/src/diagnostics/decl_check.rs @@ -24,7 +24,7 @@ use hir_def::{ }; use hir_expand::{ name::{AsName, Name}, - HirFileId, + HirFileId, HirFileIdExt, }; use stdx::{always, never}; use syntax::{ diff --git a/crates/hir-ty/src/mir/eval.rs b/crates/hir-ty/src/mir/eval.rs index 62efb858511b..fbfb6ff8cddd 100644 --- a/crates/hir-ty/src/mir/eval.rs +++ b/crates/hir-ty/src/mir/eval.rs @@ -21,7 +21,7 @@ use hir_def::{ AdtId, ConstId, DefWithBodyId, EnumVariantId, FunctionId, HasModule, ItemContainerId, Lookup, StaticId, VariantId, }; -use hir_expand::{mod_path::ModPath, InFile}; +use hir_expand::{mod_path::ModPath, HirFileIdExt, InFile}; use intern::Interned; use la_arena::ArenaMap; use rustc_hash::{FxHashMap, FxHashSet}; diff --git a/crates/hir/src/db.rs b/crates/hir/src/db.rs index 936581bfe32c..f4129e736ee1 100644 --- a/crates/hir/src/db.rs +++ b/crates/hir/src/db.rs @@ -6,7 +6,7 @@ pub use hir_def::db::*; pub use hir_expand::db::{ AstIdMapQuery, DeclMacroExpanderQuery, ExpandDatabase, ExpandDatabaseStorage, - ExpandProcMacroQuery, HygieneFrameQuery, InternMacroCallQuery, MacroArgNodeQuery, - MacroExpandQuery, ParseMacroExpansionErrorQuery, ParseMacroExpansionQuery, + ExpandProcMacroQuery, HygieneFrameQuery, InternMacroCallQuery, MacroArgQuery, MacroExpandQuery, + ParseMacroExpansionErrorQuery, ParseMacroExpansionQuery, }; pub use hir_ty::db::*; diff --git a/crates/hir/src/lib.rs b/crates/hir/src/lib.rs index 1bfbf7212bf0..50d88b4cf838 100644 --- a/crates/hir/src/lib.rs +++ b/crates/hir/src/lib.rs @@ -124,7 +124,7 @@ pub use { hir_expand::{ attrs::{Attr, AttrId}, name::{known, Name}, - ExpandResult, HirFileId, InFile, MacroFile, Origin, + tt, ExpandResult, HirFileId, HirFileIdExt, InFile, MacroFile, }, hir_ty::{ display::{ClosureStyle, HirDisplay, HirDisplayError, HirWrite}, diff --git a/crates/hir/src/semantics.rs b/crates/hir/src/semantics.rs index a42e0978b25f..758e6118aa12 100644 --- a/crates/hir/src/semantics.rs +++ b/crates/hir/src/semantics.rs @@ -15,7 +15,7 @@ use hir_def::{ type_ref::Mutability, AsMacroCall, DefWithBodyId, FieldId, FunctionId, MacroId, TraitId, VariantId, }; -use hir_expand::{db::ExpandDatabase, name::AsName, ExpansionInfo, MacroCallId}; +use hir_expand::{db::ExpandDatabase, name::AsName, ExpansionInfo, HirFileIdExt, MacroCallId}; use itertools::Itertools; use rustc_hash::{FxHashMap, FxHashSet}; use smallvec::{smallvec, SmallVec}; @@ -549,7 +549,7 @@ impl<'db> SemanticsImpl<'db> { let mut mcache = self.macro_call_cache.borrow_mut(); let mut process_expansion_for_token = - |stack: &mut SmallVec<_>, macro_file, item, token: InFile<&_>| { + |stack: &mut SmallVec<_>, macro_file, token: InFile<&_>| { let expansion_info = cache .entry(macro_file) .or_insert_with(|| macro_file.expansion_info(self.db.upcast())) @@ -562,7 +562,6 @@ impl<'db> SemanticsImpl<'db> { let mapped_tokens = expansion_info.map_token_down( self.db.upcast(), - item, token, relative_token_offset, )?; @@ -587,17 +586,12 @@ impl<'db> SemanticsImpl<'db> { // Don't force populate the dyn cache for items that don't have an attribute anyways return None; } - Some((ctx.item_to_macro_call(token.with_value(item.clone()))?, item)) + Some(ctx.item_to_macro_call(token.with_value(item.clone()))?) }) }); - if let Some((call_id, item)) = containing_attribute_macro_call { + if let Some(call_id) = containing_attribute_macro_call { let file_id = call_id.as_file(); - return process_expansion_for_token( - &mut stack, - file_id, - Some(item), - token.as_ref(), - ); + return process_expansion_for_token(&mut stack, file_id, token.as_ref()); } // Then check for token trees, that means we are either in a function-like macro or @@ -622,7 +616,7 @@ impl<'db> SemanticsImpl<'db> { it } }; - process_expansion_for_token(&mut stack, file_id, None, token.as_ref()) + process_expansion_for_token(&mut stack, file_id, token.as_ref()) } else if let Some(meta) = ast::Meta::cast(parent) { // attribute we failed expansion for earlier, this might be a derive invocation // or derive helper attribute @@ -647,7 +641,6 @@ impl<'db> SemanticsImpl<'db> { return process_expansion_for_token( &mut stack, file_id, - Some(adt.into()), token.as_ref(), ); } @@ -679,13 +672,11 @@ impl<'db> SemanticsImpl<'db> { let id = self.db.ast_id_map(token.file_id).ast_id(&adt); let helpers = def_map.derive_helpers_in_scope(InFile::new(token.file_id, id))?; - let item = Some(adt.into()); let mut res = None; for (.., derive) in helpers.iter().filter(|(helper, ..)| *helper == attr_name) { res = res.or(process_expansion_for_token( &mut stack, derive.as_file(), - item.clone(), token.as_ref(), )); } diff --git a/crates/hir/src/semantics/source_to_def.rs b/crates/hir/src/semantics/source_to_def.rs index aabda3655602..5b20c873157d 100644 --- a/crates/hir/src/semantics/source_to_def.rs +++ b/crates/hir/src/semantics/source_to_def.rs @@ -97,7 +97,7 @@ use hir_def::{ FunctionId, GenericDefId, GenericParamId, ImplId, LifetimeParamId, MacroId, ModuleId, StaticId, StructId, TraitAliasId, TraitId, TypeAliasId, TypeParamId, UnionId, UseId, VariantId, }; -use hir_expand::{attrs::AttrId, name::AsName, HirFileId, MacroCallId}; +use hir_expand::{attrs::AttrId, name::AsName, HirFileId, HirFileIdExt, MacroCallId}; use rustc_hash::FxHashMap; use smallvec::SmallVec; use stdx::{impl_from, never}; diff --git a/crates/hir/src/source_analyzer.rs b/crates/hir/src/source_analyzer.rs index 55c2f8324c6d..8fc7f2c05d74 100644 --- a/crates/hir/src/source_analyzer.rs +++ b/crates/hir/src/source_analyzer.rs @@ -30,7 +30,7 @@ use hir_expand::{ mod_path::path, name, name::{AsName, Name}, - HirFileId, InFile, + HirFileId, HirFileIdExt, InFile, }; use hir_ty::{ diagnostics::{ diff --git a/crates/ide-assists/src/handlers/extract_module.rs b/crates/ide-assists/src/handlers/extract_module.rs index 6839c5820dc9..4b9fedc7e855 100644 --- a/crates/ide-assists/src/handlers/extract_module.rs +++ b/crates/ide-assists/src/handlers/extract_module.rs @@ -3,7 +3,7 @@ use std::{ iter, }; -use hir::{HasSource, ModuleSource}; +use hir::{HasSource, HirFileIdExt, ModuleSource}; use ide_db::{ assists::{AssistId, AssistKind}, base_db::FileId, diff --git a/crates/ide-assists/src/handlers/fix_visibility.rs b/crates/ide-assists/src/handlers/fix_visibility.rs index c9f272474e7e..204e796fa2c0 100644 --- a/crates/ide-assists/src/handlers/fix_visibility.rs +++ b/crates/ide-assists/src/handlers/fix_visibility.rs @@ -1,4 +1,6 @@ -use hir::{db::HirDatabase, HasSource, HasVisibility, ModuleDef, PathResolution, ScopeDef}; +use hir::{ + db::HirDatabase, HasSource, HasVisibility, HirFileIdExt, ModuleDef, PathResolution, ScopeDef, +}; use ide_db::base_db::FileId; use syntax::{ ast::{self, edit_in_place::HasVisibilityEdit, make, HasVisibility as _}, diff --git a/crates/ide-assists/src/handlers/generate_constant.rs b/crates/ide-assists/src/handlers/generate_constant.rs index eccd7675fbaa..a4e8e7388f62 100644 --- a/crates/ide-assists/src/handlers/generate_constant.rs +++ b/crates/ide-assists/src/handlers/generate_constant.rs @@ -1,5 +1,5 @@ use crate::assist_context::{AssistContext, Assists}; -use hir::{HasVisibility, HirDisplay, Module}; +use hir::{HasVisibility, HirDisplay, HirFileIdExt, Module}; use ide_db::{ assists::{AssistId, AssistKind}, base_db::{FileId, Upcast}, diff --git a/crates/ide-assists/src/handlers/generate_enum_variant.rs b/crates/ide-assists/src/handlers/generate_enum_variant.rs index 184f523e01bd..be7a5e6c8bcc 100644 --- a/crates/ide-assists/src/handlers/generate_enum_variant.rs +++ b/crates/ide-assists/src/handlers/generate_enum_variant.rs @@ -1,4 +1,4 @@ -use hir::{HasSource, HirDisplay, InFile}; +use hir::{HasSource, HirDisplay, HirFileIdExt, InFile}; use ide_db::assists::{AssistId, AssistKind}; use syntax::{ ast::{self, make, HasArgList}, diff --git a/crates/ide-assists/src/handlers/generate_function.rs b/crates/ide-assists/src/handlers/generate_function.rs index f74fc5df4bd2..a113c817f7e9 100644 --- a/crates/ide-assists/src/handlers/generate_function.rs +++ b/crates/ide-assists/src/handlers/generate_function.rs @@ -1,5 +1,6 @@ use hir::{ - Adt, AsAssocItem, HasSource, HirDisplay, Module, PathResolution, Semantics, Type, TypeInfo, + Adt, AsAssocItem, HasSource, HirDisplay, HirFileIdExt, Module, PathResolution, Semantics, Type, + TypeInfo, }; use ide_db::{ base_db::FileId, @@ -510,7 +511,7 @@ fn assoc_fn_target_info( } fn get_insert_offset(target: &GeneratedFunctionTarget) -> TextSize { - match &target { + match target { GeneratedFunctionTarget::BehindItem(it) => it.text_range().end(), GeneratedFunctionTarget::InEmptyItemList(it) => it.text_range().start() + TextSize::of('{'), } diff --git a/crates/ide-assists/src/handlers/remove_unused_imports.rs b/crates/ide-assists/src/handlers/remove_unused_imports.rs index 5fcab8c02b06..10076e60c3e8 100644 --- a/crates/ide-assists/src/handlers/remove_unused_imports.rs +++ b/crates/ide-assists/src/handlers/remove_unused_imports.rs @@ -1,6 +1,6 @@ use std::collections::{hash_map::Entry, HashMap}; -use hir::{InFile, Module, ModuleSource}; +use hir::{HirFileIdExt, InFile, Module, ModuleSource}; use ide_db::{ base_db::FileRange, defs::Definition, diff --git a/crates/ide-assists/src/handlers/replace_derive_with_manual_impl.rs b/crates/ide-assists/src/handlers/replace_derive_with_manual_impl.rs index 69a4e748b7c5..1b373bcb8ce9 100644 --- a/crates/ide-assists/src/handlers/replace_derive_with_manual_impl.rs +++ b/crates/ide-assists/src/handlers/replace_derive_with_manual_impl.rs @@ -1,4 +1,4 @@ -use hir::{InFile, ModuleDef}; +use hir::{HirFileIdExt, InFile, ModuleDef}; use ide_db::{helpers::mod_path_to_ast, imports::import_assets::NameToImport, items_locator}; use itertools::Itertools; use syntax::{ diff --git a/crates/ide-completion/src/completions.rs b/crates/ide-completion/src/completions.rs index 7d38c638a8ed..f49abcbae9bc 100644 --- a/crates/ide-completion/src/completions.rs +++ b/crates/ide-completion/src/completions.rs @@ -683,7 +683,7 @@ pub(super) fn complete_name_ref( ctx: &CompletionContext<'_>, NameRefContext { nameref, kind }: &NameRefContext, ) { - match kind { + match dbg!(kind) { NameRefKind::Path(path_ctx) => { flyimport::import_on_the_fly_path(acc, ctx, path_ctx); diff --git a/crates/ide-completion/src/completions/mod_.rs b/crates/ide-completion/src/completions/mod_.rs index 1e09894059d5..5d138eea46f4 100644 --- a/crates/ide-completion/src/completions/mod_.rs +++ b/crates/ide-completion/src/completions/mod_.rs @@ -2,7 +2,7 @@ use std::iter; -use hir::{Module, ModuleSource}; +use hir::{HirFileIdExt, Module, ModuleSource}; use ide_db::{ base_db::{SourceDatabaseExt, VfsPath}, FxHashSet, RootDatabase, SymbolKind, diff --git a/crates/ide-db/src/apply_change.rs b/crates/ide-db/src/apply_change.rs index a0b05c87ae73..65eaa6510f0e 100644 --- a/crates/ide-db/src/apply_change.rs +++ b/crates/ide-db/src/apply_change.rs @@ -99,7 +99,7 @@ impl RootDatabase { hir::db::AstIdMapQuery hir::db::ParseMacroExpansionQuery hir::db::InternMacroCallQuery - hir::db::MacroArgNodeQuery + hir::db::MacroArgQuery hir::db::DeclMacroExpanderQuery hir::db::MacroExpandQuery hir::db::ExpandProcMacroQuery diff --git a/crates/ide-db/src/lib.rs b/crates/ide-db/src/lib.rs index 226def4d5268..258d893b47da 100644 --- a/crates/ide-db/src/lib.rs +++ b/crates/ide-db/src/lib.rs @@ -204,7 +204,7 @@ impl RootDatabase { hir_db::AstIdMapQuery // hir_db::ParseMacroExpansionQuery // hir_db::InternMacroCallQuery - hir_db::MacroArgNodeQuery + hir_db::MacroArgQuery hir_db::DeclMacroExpanderQuery // hir_db::MacroExpandQuery hir_db::ExpandProcMacroQuery diff --git a/crates/ide-db/src/rename.rs b/crates/ide-db/src/rename.rs index 353a9749a37d..cc9038fdfa2f 100644 --- a/crates/ide-db/src/rename.rs +++ b/crates/ide-db/src/rename.rs @@ -24,7 +24,7 @@ use std::fmt; use base_db::{AnchoredPathBuf, FileId, FileRange}; use either::Either; -use hir::{FieldSource, HasSource, InFile, ModuleSource, Semantics}; +use hir::{FieldSource, HasSource, HirFileIdExt, InFile, ModuleSource, Semantics}; use stdx::never; use syntax::{ ast::{self, HasName}, diff --git a/crates/ide-db/src/search.rs b/crates/ide-db/src/search.rs index 22438a203bd7..68f2ad494570 100644 --- a/crates/ide-db/src/search.rs +++ b/crates/ide-db/src/search.rs @@ -8,7 +8,8 @@ use std::mem; use base_db::{salsa::Database, FileId, FileRange, SourceDatabase, SourceDatabaseExt}; use hir::{ - AsAssocItem, DefWithBody, HasAttrs, HasSource, InFile, ModuleSource, Semantics, Visibility, + AsAssocItem, DefWithBody, HasAttrs, HasSource, HirFileIdExt, InFile, ModuleSource, Semantics, + Visibility, }; use memchr::memmem::Finder; use nohash_hasher::IntMap; diff --git a/crates/ide-db/src/test_data/test_doc_alias.txt b/crates/ide-db/src/test_data/test_doc_alias.txt index 7834c66033c0..72a6eb5eabd1 100644 --- a/crates/ide-db/src/test_data/test_doc_alias.txt +++ b/crates/ide-db/src/test_data/test_doc_alias.txt @@ -21,9 +21,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: STRUCT, @@ -50,9 +48,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: STRUCT, @@ -79,9 +75,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: STRUCT, @@ -108,9 +102,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: STRUCT, @@ -137,9 +129,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: STRUCT, @@ -166,9 +156,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: STRUCT, @@ -195,9 +183,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: STRUCT, diff --git a/crates/ide-db/src/test_data/test_symbol_index_collection.txt b/crates/ide-db/src/test_data/test_symbol_index_collection.txt index 87ad5844c64c..375ac5598152 100644 --- a/crates/ide-db/src/test_data/test_symbol_index_collection.txt +++ b/crates/ide-db/src/test_data/test_symbol_index_collection.txt @@ -19,9 +19,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: TYPE_ALIAS, @@ -46,9 +44,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: CONST, @@ -73,9 +69,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: CONST, @@ -102,9 +96,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: ENUM, @@ -131,9 +123,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: USE_TREE, @@ -160,9 +150,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: MACRO_DEF, @@ -187,9 +175,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: STATIC, @@ -216,9 +202,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: STRUCT, @@ -245,11 +229,7 @@ ), loc: DeclarationLocation { hir_file_id: MacroFile( - MacroFile { - macro_call_id: MacroCallId( - 0, - ), - }, + 0, ), ptr: SyntaxNodePtr { kind: STRUCT, @@ -276,9 +256,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: STRUCT, @@ -307,9 +285,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: STRUCT, @@ -338,9 +314,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: STRUCT, @@ -365,9 +339,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: TRAIT, @@ -394,9 +366,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: USE_TREE, @@ -423,9 +393,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: UNION, @@ -452,9 +420,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: MODULE, @@ -481,9 +447,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: MODULE, @@ -510,9 +474,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: MACRO_RULES, @@ -537,9 +499,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: FN, @@ -566,9 +526,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: MACRO_RULES, @@ -593,9 +551,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: FN, @@ -622,9 +578,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: USE_TREE, @@ -649,9 +603,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: FN, @@ -691,9 +643,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 0, - ), + 0, ), ptr: SyntaxNodePtr { kind: STRUCT, @@ -731,9 +681,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 1, - ), + 1, ), ptr: SyntaxNodePtr { kind: USE_TREE, @@ -760,9 +708,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 1, - ), + 1, ), ptr: SyntaxNodePtr { kind: STRUCT, @@ -789,9 +735,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 1, - ), + 1, ), ptr: SyntaxNodePtr { kind: USE_TREE, @@ -818,9 +762,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 1, - ), + 1, ), ptr: SyntaxNodePtr { kind: USE_TREE, @@ -847,9 +789,7 @@ ), loc: DeclarationLocation { hir_file_id: FileId( - FileId( - 1, - ), + 1, ), ptr: SyntaxNodePtr { kind: USE_TREE, diff --git a/crates/ide-diagnostics/src/handlers/missing_fields.rs b/crates/ide-diagnostics/src/handlers/missing_fields.rs index d7dca1083a07..cb38bc54d7d6 100644 --- a/crates/ide-diagnostics/src/handlers/missing_fields.rs +++ b/crates/ide-diagnostics/src/handlers/missing_fields.rs @@ -1,7 +1,7 @@ use either::Either; use hir::{ db::{ExpandDatabase, HirDatabase}, - known, AssocItem, HirDisplay, InFile, Type, + known, AssocItem, HirDisplay, HirFileIdExt, InFile, Type, }; use ide_db::{ assists::Assist, famous_defs::FamousDefs, imports::import_assets::item_for_path_search, diff --git a/crates/ide-diagnostics/src/handlers/missing_unsafe.rs b/crates/ide-diagnostics/src/handlers/missing_unsafe.rs index 0f695b2745a8..f93a35cf181c 100644 --- a/crates/ide-diagnostics/src/handlers/missing_unsafe.rs +++ b/crates/ide-diagnostics/src/handlers/missing_unsafe.rs @@ -1,4 +1,5 @@ use hir::db::ExpandDatabase; +use hir::HirFileIdExt; use ide_db::{assists::Assist, source_change::SourceChange}; use syntax::{ast, SyntaxNode}; use syntax::{match_ast, AstNode}; diff --git a/crates/ide-diagnostics/src/handlers/no_such_field.rs b/crates/ide-diagnostics/src/handlers/no_such_field.rs index ee8a9c95793c..0abcbffe72b6 100644 --- a/crates/ide-diagnostics/src/handlers/no_such_field.rs +++ b/crates/ide-diagnostics/src/handlers/no_such_field.rs @@ -1,5 +1,5 @@ use either::Either; -use hir::{db::ExpandDatabase, HasSource, HirDisplay, Semantics}; +use hir::{db::ExpandDatabase, HasSource, HirDisplay, HirFileIdExt, Semantics}; use ide_db::{base_db::FileId, source_change::SourceChange, RootDatabase}; use syntax::{ ast::{self, edit::IndentLevel, make}, diff --git a/crates/ide-diagnostics/src/handlers/replace_filter_map_next_with_find_map.rs b/crates/ide-diagnostics/src/handlers/replace_filter_map_next_with_find_map.rs index d15233d15c2c..258ac6cd8233 100644 --- a/crates/ide-diagnostics/src/handlers/replace_filter_map_next_with_find_map.rs +++ b/crates/ide-diagnostics/src/handlers/replace_filter_map_next_with_find_map.rs @@ -1,4 +1,4 @@ -use hir::{db::ExpandDatabase, InFile}; +use hir::{db::ExpandDatabase, HirFileIdExt, InFile}; use ide_db::source_change::SourceChange; use syntax::{ ast::{self, HasArgList}, diff --git a/crates/ide-diagnostics/src/handlers/type_mismatch.rs b/crates/ide-diagnostics/src/handlers/type_mismatch.rs index c92d92ceae8c..fd00535d0c34 100644 --- a/crates/ide-diagnostics/src/handlers/type_mismatch.rs +++ b/crates/ide-diagnostics/src/handlers/type_mismatch.rs @@ -1,4 +1,4 @@ -use hir::{db::ExpandDatabase, ClosureStyle, HirDisplay, InFile, Type}; +use hir::{db::ExpandDatabase, ClosureStyle, HirDisplay, HirFileIdExt, InFile, Type}; use ide_db::{famous_defs::FamousDefs, source_change::SourceChange}; use syntax::{ ast::{self, BlockExpr, ExprStmt}, diff --git a/crates/ide-diagnostics/src/handlers/unresolved_module.rs b/crates/ide-diagnostics/src/handlers/unresolved_module.rs index be24e50c9871..bbbd21741e5d 100644 --- a/crates/ide-diagnostics/src/handlers/unresolved_module.rs +++ b/crates/ide-diagnostics/src/handlers/unresolved_module.rs @@ -1,4 +1,4 @@ -use hir::db::ExpandDatabase; +use hir::{db::ExpandDatabase, HirFileIdExt}; use ide_db::{assists::Assist, base_db::AnchoredPathBuf, source_change::FileSystemEdit}; use itertools::Itertools; use syntax::AstNode; diff --git a/crates/ide/src/call_hierarchy.rs b/crates/ide/src/call_hierarchy.rs index f834f2ce5927..6f41f51f80ec 100644 --- a/crates/ide/src/call_hierarchy.rs +++ b/crates/ide/src/call_hierarchy.rs @@ -149,7 +149,7 @@ mod tests { fn check_hierarchy( ra_fixture: &str, - expected: Expect, + expected_nav: Expect, expected_incoming: Expect, expected_outgoing: Expect, ) { @@ -158,7 +158,7 @@ mod tests { let mut navs = analysis.call_hierarchy(pos).unwrap().unwrap().info; assert_eq!(navs.len(), 1); let nav = navs.pop().unwrap(); - expected.assert_eq(&nav.debug_render()); + expected_nav.assert_eq(&nav.debug_render()); let item_pos = FilePosition { file_id: nav.file_id, offset: nav.focus_or_full_range().start() }; diff --git a/crates/ide/src/expand_macro.rs b/crates/ide/src/expand_macro.rs index 322077456775..f7c6a0139e0c 100644 --- a/crates/ide/src/expand_macro.rs +++ b/crates/ide/src/expand_macro.rs @@ -1,4 +1,4 @@ -use hir::Semantics; +use hir::{HirFileIdExt, Semantics}; use ide_db::{ base_db::FileId, helpers::pick_best_token, syntax_helpers::insert_whitespace_into_node::insert_ws_into, RootDatabase, diff --git a/crates/ide/src/goto_definition.rs b/crates/ide/src/goto_definition.rs index e09b9f391482..59e8578cf154 100644 --- a/crates/ide/src/goto_definition.rs +++ b/crates/ide/src/goto_definition.rs @@ -60,13 +60,13 @@ pub(crate) fn goto_definition( .into_iter() .filter_map(|token| { let parent = token.parent()?; - if let Some(tt) = ast::TokenTree::cast(parent) { + if let Some(tt) = ast::TokenTree::cast(parent.clone()) { if let Some(x) = try_lookup_include_path(sema, tt, token.clone(), file_id) { return Some(vec![x]); } } Some( - IdentClass::classify_token(sema, &token)? + IdentClass::classify_node(sema, &parent)? .definitions() .into_iter() .flat_map(|def| { @@ -392,6 +392,8 @@ fn bar() { ); } + // FIXME: We should emit two targets here, one for the identifier in the declaration, one for + // the macro call #[test] fn goto_def_for_macro_defined_fn_no_arg() { check( @@ -399,10 +401,10 @@ fn bar() { //- /lib.rs macro_rules! define_fn { () => (fn foo() {}) + //^^^ } define_fn!(); -//^^^^^^^^^^^^^ fn bar() { $0foo(); diff --git a/crates/ide/src/rename.rs b/crates/ide/src/rename.rs index ac9df5ed6d1f..ad1eb2499718 100644 --- a/crates/ide/src/rename.rs +++ b/crates/ide/src/rename.rs @@ -4,7 +4,7 @@ //! tests. This module also implements a couple of magic tricks, like renaming //! `self` and to `self` (to switch between associated function and method). -use hir::{AsAssocItem, InFile, Semantics}; +use hir::{AsAssocItem, HirFileIdExt, InFile, Semantics}; use ide_db::{ base_db::FileId, defs::{Definition, NameClass, NameRefClass}, diff --git a/crates/ide/src/runnables.rs b/crates/ide/src/runnables.rs index 07cdddd15f82..954a364c7870 100644 --- a/crates/ide/src/runnables.rs +++ b/crates/ide/src/runnables.rs @@ -2,7 +2,7 @@ use std::fmt; use ast::HasName; use cfg::CfgExpr; -use hir::{db::HirDatabase, AsAssocItem, HasAttrs, HasSource, Semantics}; +use hir::{db::HirDatabase, AsAssocItem, HasAttrs, HasSource, HirFileIdExt, Semantics}; use ide_assists::utils::test_related_attribute; use ide_db::{ base_db::{FilePosition, FileRange}, diff --git a/crates/ide/src/static_index.rs b/crates/ide/src/static_index.rs index aabd26da289c..7ee50f7a67f8 100644 --- a/crates/ide/src/static_index.rs +++ b/crates/ide/src/static_index.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; -use hir::{db::HirDatabase, Crate, Module}; +use hir::{db::HirDatabase, Crate, HirFileIdExt, Module}; use ide_db::helpers::get_definition; use ide_db::{ base_db::{FileId, FileRange, SourceDatabaseExt}, diff --git a/crates/ide/src/syntax_highlighting/highlight.rs b/crates/ide/src/syntax_highlighting/highlight.rs index 7d00282fc14b..1bffab29cf93 100644 --- a/crates/ide/src/syntax_highlighting/highlight.rs +++ b/crates/ide/src/syntax_highlighting/highlight.rs @@ -1,6 +1,6 @@ //! Computes color for a single element. -use hir::{AsAssocItem, HasVisibility, Semantics}; +use hir::{AsAssocItem, HasVisibility, HirFileIdExt, Semantics}; use ide_db::{ defs::{Definition, IdentClass, NameClass, NameRefClass}, FxHashMap, RootDatabase, SymbolKind, diff --git a/crates/load-cargo/src/lib.rs b/crates/load-cargo/src/lib.rs index 68b592ffaa4d..4d1319094933 100644 --- a/crates/load-cargo/src/lib.rs +++ b/crates/load-cargo/src/lib.rs @@ -4,13 +4,12 @@ // to run rust-analyzer as a library. use std::{collections::hash_map::Entry, mem, path::Path, sync}; -use ::tt::token_id as tt; use crossbeam_channel::{unbounded, Receiver}; use ide::{AnalysisHost, Change, SourceRoot}; use ide_db::{ base_db::{ - CrateGraph, Env, ProcMacro, ProcMacroExpander, ProcMacroExpansionError, ProcMacroKind, - ProcMacroLoadResult, ProcMacros, + span::SpanData, CrateGraph, Env, ProcMacro, ProcMacroExpander, ProcMacroExpansionError, + ProcMacroKind, ProcMacroLoadResult, ProcMacros, }, FxHashMap, }; @@ -374,16 +373,19 @@ struct Expander(proc_macro_api::ProcMacro); impl ProcMacroExpander for Expander { fn expand( &self, - subtree: &tt::Subtree, - attrs: Option<&tt::Subtree>, + subtree: &tt::Subtree, + attrs: Option<&tt::Subtree>, env: &Env, - ) -> Result { - let env = env.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(); - match self.0.expand(subtree, attrs, env) { - Ok(Ok(subtree)) => Ok(subtree), - Ok(Err(err)) => Err(ProcMacroExpansionError::Panic(err.0)), - Err(err) => Err(ProcMacroExpansionError::System(err.to_string())), - } + ) -> Result, ProcMacroExpansionError> { + let _ = (subtree, attrs, env); + + // let env = env.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(); + // match self.0.expand(subtree, attrs, env) { + // Ok(Ok(subtree)) => Ok(subtree), + // Ok(Err(err)) => Err(ProcMacroExpansionError::Panic(err.0)), + // Err(err) => Err(ProcMacroExpansionError::System(err.to_string())), + // } + todo!() } } @@ -394,10 +396,10 @@ struct IdentityExpander; impl ProcMacroExpander for IdentityExpander { fn expand( &self, - subtree: &tt::Subtree, - _: Option<&tt::Subtree>, + subtree: &tt::Subtree, + _: Option<&tt::Subtree>, _: &Env, - ) -> Result { + ) -> Result, ProcMacroExpansionError> { Ok(subtree.clone()) } } @@ -409,10 +411,10 @@ struct EmptyExpander; impl ProcMacroExpander for EmptyExpander { fn expand( &self, - _: &tt::Subtree, - _: Option<&tt::Subtree>, + _: &tt::Subtree, + _: Option<&tt::Subtree>, _: &Env, - ) -> Result { + ) -> Result, ProcMacroExpansionError> { Ok(tt::Subtree::empty()) } } diff --git a/crates/mbe/src/benchmark.rs b/crates/mbe/src/benchmark.rs index 19cb20354b74..4f60e90773ee 100644 --- a/crates/mbe/src/benchmark.rs +++ b/crates/mbe/src/benchmark.rs @@ -6,13 +6,19 @@ use syntax::{ AstNode, SmolStr, }; use test_utils::{bench, bench_fixture, skip_slow_tests}; -use tt::{Span, TokenId}; +use tt::{Span, SpanData}; use crate::{ parser::{MetaVarKind, Op, RepeatKind, Separator}, syntax_node_to_token_tree, DeclarativeMacro, }; +#[derive(PartialEq, Eq, Clone, Copy, Debug)] +struct DummyFile; +impl Span for DummyFile { + const DUMMY: Self = DummyFile; +} + #[test] fn benchmark_parse_macro_rules() { if skip_slow_tests() { @@ -39,7 +45,7 @@ fn benchmark_expand_macro_rules() { invocations .into_iter() .map(|(id, tt)| { - let res = rules[&id].expand(tt); + let res = rules[&id].expand(&tt); assert!(res.err.is_none()); res.value.token_trees.len() }) @@ -48,14 +54,14 @@ fn benchmark_expand_macro_rules() { assert_eq!(hash, 69413); } -fn macro_rules_fixtures() -> FxHashMap { +fn macro_rules_fixtures() -> FxHashMap>> { macro_rules_fixtures_tt() .into_iter() .map(|(id, tt)| (id, DeclarativeMacro::parse_macro_rules(&tt, true))) .collect() } -fn macro_rules_fixtures_tt() -> FxHashMap> { +fn macro_rules_fixtures_tt() -> FxHashMap>> { let fixture = bench_fixture::numerous_macro_rules(); let source_file = ast::SourceFile::parse(&fixture).ok().unwrap(); @@ -65,7 +71,12 @@ fn macro_rules_fixtures_tt() -> FxHashMap> { .filter_map(ast::MacroRules::cast) .map(|rule| { let id = rule.name().unwrap().to_string(); - let (def_tt, _) = syntax_node_to_token_tree(rule.token_tree().unwrap().syntax()); + let def_tt = syntax_node_to_token_tree( + rule.token_tree().unwrap().syntax(), + DummyFile, + 0.into(), + &Default::default(), + ); (id, def_tt) }) .collect() @@ -73,8 +84,8 @@ fn macro_rules_fixtures_tt() -> FxHashMap> { /// Generate random invocation fixtures from rules fn invocation_fixtures( - rules: &FxHashMap, -) -> Vec<(String, tt::Subtree)> { + rules: &FxHashMap>>, +) -> Vec<(String, tt::Subtree>)> { let mut seed = 123456789; let mut res = Vec::new(); @@ -96,8 +107,8 @@ fn invocation_fixtures( loop { let mut subtree = tt::Subtree { delimiter: tt::Delimiter { - open: tt::TokenId::DUMMY, - close: tt::TokenId::DUMMY, + open: SpanData::DUMMY, + close: SpanData::DUMMY, kind: tt::DelimiterKind::Invisible, }, token_trees: vec![], @@ -105,7 +116,7 @@ fn invocation_fixtures( for op in rule.lhs.iter() { collect_from_op(op, &mut subtree, &mut seed); } - if it.expand(subtree.clone()).err.is_none() { + if it.expand(&subtree).err.is_none() { res.push((name.clone(), subtree)); break; } @@ -119,7 +130,11 @@ fn invocation_fixtures( } return res; - fn collect_from_op(op: &Op, parent: &mut tt::Subtree, seed: &mut usize) { + fn collect_from_op( + op: &Op>, + parent: &mut tt::Subtree>, + seed: &mut usize, + ) { return match op { Op::Var { kind, .. } => match kind.as_ref() { Some(MetaVarKind::Ident) => parent.token_trees.push(make_ident("foo")), @@ -205,32 +220,22 @@ fn invocation_fixtures( *seed = usize::wrapping_add(usize::wrapping_mul(*seed, a), c); *seed } - fn make_ident(ident: &str) -> tt::TokenTree { - tt::Leaf::Ident(tt::Ident { span: tt::TokenId::DUMMY, text: SmolStr::new(ident) }) - .into() + fn make_ident(ident: &str) -> tt::TokenTree> { + tt::Leaf::Ident(tt::Ident { span: SpanData::DUMMY, text: SmolStr::new(ident) }).into() } - fn make_punct(char: char) -> tt::TokenTree { - tt::Leaf::Punct(tt::Punct { - span: tt::TokenId::DUMMY, - char, - spacing: tt::Spacing::Alone, - }) - .into() - } - fn make_literal(lit: &str) -> tt::TokenTree { - tt::Leaf::Literal(tt::Literal { span: tt::TokenId::DUMMY, text: SmolStr::new(lit) }) + fn make_punct(char: char) -> tt::TokenTree> { + tt::Leaf::Punct(tt::Punct { span: SpanData::DUMMY, char, spacing: tt::Spacing::Alone }) .into() } + fn make_literal(lit: &str) -> tt::TokenTree> { + tt::Leaf::Literal(tt::Literal { span: SpanData::DUMMY, text: SmolStr::new(lit) }).into() + } fn make_subtree( kind: tt::DelimiterKind, - token_trees: Option>>, - ) -> tt::TokenTree { + token_trees: Option>>>, + ) -> tt::TokenTree> { tt::Subtree { - delimiter: tt::Delimiter { - open: tt::TokenId::DUMMY, - close: tt::TokenId::DUMMY, - kind, - }, + delimiter: tt::Delimiter { open: SpanData::DUMMY, close: SpanData::DUMMY, kind }, token_trees: token_trees.unwrap_or_default(), } .into() diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index 46599802935e..43543479eb94 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -19,7 +19,7 @@ mod benchmark; mod token_map; use stdx::impl_from; -use tt::{Span, TokenId}; +use tt::Span; use std::fmt; @@ -34,10 +34,8 @@ pub use tt::{Delimiter, DelimiterKind, Punct}; pub use crate::{ syntax_bridge::{ - parse_exprs_with_sep, parse_to_token_tree, syntax_node_to_token_map, - syntax_node_to_token_map_with_modifications, syntax_node_to_token_tree, - syntax_node_to_token_tree_with_modifications, token_tree_to_syntax_node, SyntheticToken, - SyntheticTokenId, + map_from_syntax_node, parse_exprs_with_sep, parse_to_token_tree, syntax_node_to_token_tree, + syntax_node_to_token_tree_censored, token_tree_to_syntax_node, }, token_map::TokenMap, }; @@ -125,10 +123,8 @@ impl fmt::Display for CountError { /// `tt::TokenTree`, but there's a crucial difference: in macro rules, `$ident` /// and `$()*` have special meaning (see `Var` and `Repeat` data structures) #[derive(Clone, Debug, PartialEq, Eq)] -pub struct DeclarativeMacro { - rules: Box<[Rule]>, - /// Highest id of the token we have in TokenMap - shift: Shift, +pub struct DeclarativeMacro { + rules: Box<[Rule]>, // This is used for correctly determining the behavior of the pat fragment // FIXME: This should be tracked by hygiene of the fragment identifier! is_2021: bool, @@ -141,91 +137,13 @@ struct Rule { rhs: MetaTemplate, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct Shift(u32); - -impl Shift { - pub fn new(tt: &tt::Subtree) -> Shift { - // Note that TokenId is started from zero, - // We have to add 1 to prevent duplication. - let value = max_id(tt).map_or(0, |it| it + 1); - return Shift(value); - - // Find the max token id inside a subtree - fn max_id(subtree: &tt::Subtree) -> Option { - let filter = - |tt: &_| match tt { - tt::TokenTree::Subtree(subtree) => { - let tree_id = max_id(subtree); - if subtree.delimiter.open != tt::TokenId::unspecified() { - Some(tree_id.map_or(subtree.delimiter.open.0, |t| { - t.max(subtree.delimiter.open.0) - })) - } else { - tree_id - } - } - tt::TokenTree::Leaf(leaf) => { - let &(tt::Leaf::Ident(tt::Ident { span, .. }) - | tt::Leaf::Punct(tt::Punct { span, .. }) - | tt::Leaf::Literal(tt::Literal { span, .. })) = leaf; - - (span != tt::TokenId::unspecified()).then_some(span.0) - } - }; - subtree.token_trees.iter().filter_map(filter).max() - } - } - - /// Shift given TokenTree token id - pub fn shift_all(self, tt: &mut tt::Subtree) { - for t in &mut tt.token_trees { - match t { - tt::TokenTree::Leaf( - tt::Leaf::Ident(tt::Ident { span, .. }) - | tt::Leaf::Punct(tt::Punct { span, .. }) - | tt::Leaf::Literal(tt::Literal { span, .. }), - ) => *span = self.shift(*span), - tt::TokenTree::Subtree(tt) => { - tt.delimiter.open = self.shift(tt.delimiter.open); - tt.delimiter.close = self.shift(tt.delimiter.close); - self.shift_all(tt) - } - } - } - } - - pub fn shift(self, id: tt::TokenId) -> tt::TokenId { - if id == tt::TokenId::unspecified() { - id - } else { - tt::TokenId(id.0 + self.0) - } - } - - pub fn unshift(self, id: tt::TokenId) -> Option { - id.0.checked_sub(self.0).map(tt::TokenId) - } -} - -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -pub enum Origin { - Def, - Call, -} - -impl DeclarativeMacro { - pub fn from_err(err: ParseError, is_2021: bool) -> DeclarativeMacro { - DeclarativeMacro { - rules: Box::default(), - shift: Shift(0), - is_2021, - err: Some(Box::new(err)), - } +impl DeclarativeMacro { + pub fn from_err(err: ParseError, is_2021: bool) -> DeclarativeMacro { + DeclarativeMacro { rules: Box::default(), is_2021, err: Some(Box::new(err)) } } /// The old, `macro_rules! m {}` flavor. - pub fn parse_macro_rules(tt: &tt::Subtree, is_2021: bool) -> DeclarativeMacro { + pub fn parse_macro_rules(tt: &tt::Subtree, is_2021: bool) -> DeclarativeMacro { // Note: this parsing can be implemented using mbe machinery itself, by // matching against `$($lhs:tt => $rhs:tt);*` pattern, but implementing // manually seems easier. @@ -257,11 +175,11 @@ impl DeclarativeMacro { } } - DeclarativeMacro { rules: rules.into_boxed_slice(), shift: Shift::new(tt), is_2021, err } + DeclarativeMacro { rules: rules.into_boxed_slice(), is_2021, err } } /// The new, unstable `macro m {}` flavor. - pub fn parse_macro2(tt: &tt::Subtree, is_2021: bool) -> DeclarativeMacro { + pub fn parse_macro2(tt: &tt::Subtree, is_2021: bool) -> DeclarativeMacro { let mut src = TtIter::new(tt); let mut rules = Vec::new(); let mut err = None; @@ -308,31 +226,15 @@ impl DeclarativeMacro { } } - DeclarativeMacro { rules: rules.into_boxed_slice(), shift: Shift::new(tt), is_2021, err } - } - - pub fn expand(&self, mut tt: tt::Subtree) -> ExpandResult> { - self.shift.shift_all(&mut tt); - expander::expand_rules(&self.rules, &tt, self.is_2021) + DeclarativeMacro { rules: rules.into_boxed_slice(), is_2021, err } } pub fn err(&self) -> Option<&ParseError> { self.err.as_deref() } - pub fn map_id_down(&self, id: tt::TokenId) -> tt::TokenId { - self.shift.shift(id) - } - - pub fn map_id_up(&self, id: tt::TokenId) -> (tt::TokenId, Origin) { - match self.shift.unshift(id) { - Some(id) => (id, Origin::Call), - None => (id, Origin::Def), - } - } - - pub fn shift(&self) -> Shift { - self.shift + pub fn expand(&self, tt: &tt::Subtree) -> ExpandResult> { + expander::expand_rules(&self.rules, &tt, self.is_2021) } } diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 01aab6b659cb..c8c2e5dcd55a 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -1,16 +1,15 @@ //! Conversions between [`SyntaxNode`] and [`tt::TokenTree`]. -use rustc_hash::FxHashMap; -use stdx::{always, non_empty_vec::NonEmptyVec}; +use stdx::non_empty_vec::NonEmptyVec; use syntax::{ ast::{self, make::tokens::doc_comment}, - AstToken, Parse, PreorderWithTokens, SmolStr, SyntaxElement, SyntaxKind, + AstToken, NodeOrToken, Parse, PreorderWithTokens, SmolStr, SyntaxElement, SyntaxKind, SyntaxKind::*, SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, WalkEvent, T, }; use tt::{ buffer::{Cursor, TokenBuffer}, - TokenId, + Span, SpanData, }; use crate::{to_parser_input::to_parser_input, tt_iter::TtIter, TokenMap}; @@ -20,75 +19,37 @@ mod tests; /// Convert the syntax node to a `TokenTree` (what macro /// will consume). -pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> (tt::Subtree, TokenMap) { - let (subtree, token_map, _) = syntax_node_to_token_tree_with_modifications( - node, - Default::default(), - 0, - Default::default(), - Default::default(), - ); - (subtree, token_map) -} - -/// Convert the syntax node to a `TokenTree` (what macro will consume) -/// with the censored range excluded. -pub fn syntax_node_to_token_tree_with_modifications( +/// `anchor` and `anchor_offset` are used to convert the node's spans +/// to relative spans, relative to the passed anchor. +/// `map` is used to resolve the converted spans accordingly. +/// TODO: Flesh out the doc comment more thoroughly +pub fn syntax_node_to_token_tree( node: &SyntaxNode, - existing_token_map: TokenMap, - next_id: u32, - replace: FxHashMap>, - append: FxHashMap>, -) -> (tt::Subtree, TokenMap, u32) { - let global_offset = node.text_range().start(); - let mut c = Converter::new(node, global_offset, existing_token_map, next_id, replace, append); - let subtree = convert_tokens(&mut c); - c.id_alloc.map.shrink_to_fit(); - always!(c.replace.is_empty(), "replace: {:?}", c.replace); - always!(c.append.is_empty(), "append: {:?}", c.append); - (subtree, c.id_alloc.map, c.id_alloc.next_id) -} - -/// Convert the syntax node to a `TokenTree` (what macro -/// will consume). -pub fn syntax_node_to_token_map(node: &SyntaxNode) -> TokenMap { - syntax_node_to_token_map_with_modifications( - node, - Default::default(), - 0, - Default::default(), - Default::default(), - ) - .0 + anchor: SpanAnchor, + anchor_offset: TextSize, + map: &TokenMap>, +) -> tt::Subtree> +where + SpanData: Span, +{ + assert!(anchor_offset <= node.text_range().start()); + let mut c = Converter::new(node, anchor_offset, anchor, vec![], map); + convert_tokens(&mut c) } -/// Convert the syntax node to a `TokenTree` (what macro will consume) -/// with the censored range excluded. -pub fn syntax_node_to_token_map_with_modifications( +pub fn syntax_node_to_token_tree_censored( node: &SyntaxNode, - existing_token_map: TokenMap, - next_id: u32, - replace: FxHashMap>, - append: FxHashMap>, -) -> (TokenMap, u32) { - let global_offset = node.text_range().start(); - let mut c = Converter::new(node, global_offset, existing_token_map, next_id, replace, append); - collect_tokens(&mut c); - c.id_alloc.map.shrink_to_fit(); - always!(c.replace.is_empty(), "replace: {:?}", c.replace); - always!(c.append.is_empty(), "append: {:?}", c.append); - (c.id_alloc.map, c.id_alloc.next_id) -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct SyntheticTokenId(pub u32); - -#[derive(Debug, Clone)] -pub struct SyntheticToken { - pub kind: SyntaxKind, - pub text: SmolStr, - pub range: TextRange, - pub id: SyntheticTokenId, + anchor: SpanAnchor, + anchor_offset: TextSize, + map: &TokenMap>, + censored: Vec, +) -> tt::Subtree> +where + SpanData: Span, +{ + assert!(anchor_offset <= node.text_range().start()); + let mut c = Converter::new(node, anchor_offset, anchor, censored, map); + convert_tokens(&mut c) } // The following items are what `rustc` macro can be parsed into : @@ -103,10 +64,13 @@ pub struct SyntheticToken { // * AssocItems(SmallVec<[ast::AssocItem; 1]>) // * ForeignItems(SmallVec<[ast::ForeignItem; 1]> -pub fn token_tree_to_syntax_node( - tt: &tt::Subtree, +pub fn token_tree_to_syntax_node( + tt: &tt::Subtree>, entry_point: parser::TopEntryPoint, -) -> (Parse, TokenMap) { +) -> (Parse, TokenMap>) +where + SpanData: Span, +{ let buffer = match tt { tt::Subtree { delimiter: tt::Delimiter { kind: tt::DelimiterKind::Invisible, .. }, @@ -133,29 +97,40 @@ pub fn token_tree_to_syntax_node( tree_sink.finish() } +pub fn map_from_syntax_node( + node: &SyntaxNode, + anchor: SpanAnchor, + anchor_offset: TextSize, +) -> TokenMap> +where + SpanAnchor: Copy, + SpanData: Span, +{ + let mut map = TokenMap::default(); + node.descendants_with_tokens().filter_map(NodeOrToken::into_token).for_each(|t| { + map.insert(t.text_range(), SpanData { range: t.text_range() - anchor_offset, anchor }); + }); + map +} + /// Convert a string to a `TokenTree` -pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> { +pub fn parse_to_token_tree( + text: &str, + file_id: SpanAnchor, +) -> Option>> +where + SpanData: Span, +{ let lexed = parser::LexedStr::new(text); if lexed.errors().next().is_some() { return None; } - - let mut conv = RawConverter { - lexed, - pos: 0, - id_alloc: TokenIdAlloc { - map: Default::default(), - global_offset: TextSize::default(), - next_id: 0, - }, - }; - - let subtree = convert_tokens(&mut conv); - Some((subtree, conv.id_alloc.map)) + let mut conv = RawConverter { lexed, pos: 0, _offset: TextSize::default(), file_id }; + Some(convert_tokens(&mut conv)) } /// Split token tree with separate expr: $($e:expr)SEP* -pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec> { +pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec> { if tt.token_trees.is_empty() { return Vec::new(); } @@ -191,47 +166,33 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec(conv: &mut C) -> tt::Subtree { - struct StackEntry { - subtree: tt::Subtree, - idx: usize, - open_range: TextRange, - } - - let entry = StackEntry { - subtree: tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] }, - // never used (delimiter is `None`) - idx: !0, - open_range: TextRange::empty(TextSize::of('.')), - }; +fn convert_tokens>( + conv: &mut C, +) -> tt::Subtree> +where + SpanData: Span, + SpanAnchor: Copy, +{ + let entry = tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] }; let mut stack = NonEmptyVec::new(entry); + let anchor = conv.anchor(); loop { - let StackEntry { subtree, .. } = stack.last_mut(); + let subtree = stack.last_mut(); let result = &mut subtree.token_trees; - let (token, range) = match conv.bump() { - Some(it) => it, - None => break, - }; - let synth_id = token.synthetic_id(conv); + let Some((token, rel_range, abs_range)) = conv.bump() else { break }; let kind = token.kind(conv); if kind == COMMENT { - // Since `convert_doc_comment` can fail, we need to peek the next id, so that we can - // figure out which token id to use for the doc comment, if it is converted successfully. - let next_id = conv.id_alloc().peek_next_id(); - if let Some(tokens) = conv.convert_doc_comment(&token, next_id) { - let id = conv.id_alloc().alloc(range, synth_id); - debug_assert_eq!(id, next_id); + if let Some(tokens) = conv.convert_doc_comment( + &token, + conv.span_for(abs_range).unwrap_or(SpanData { range: rel_range, anchor }), + ) { result.extend(tokens); } continue; } let tt = if kind.is_punct() && kind != UNDERSCORE { - if synth_id.is_none() { - assert_eq!(range.len(), TextSize::of('.')); - } - let expected = match subtree.delimiter.kind { tt::DelimiterKind::Parenthesis => Some(T![')']), tt::DelimiterKind::Brace => Some(T!['}']), @@ -241,9 +202,11 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { if let Some(expected) = expected { if kind == expected { - if let Some(entry) = stack.pop() { - conv.id_alloc().close_delim(entry.idx, Some(range)); - stack.last_mut().subtree.token_trees.push(entry.subtree.into()); + if let Some(mut subtree) = stack.pop() { + subtree.delimiter.close = conv + .span_for(abs_range) + .unwrap_or(SpanData { range: rel_range, anchor }); + stack.last_mut().token_trees.push(subtree.into()); } continue; } @@ -257,12 +220,18 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { }; if let Some(kind) = delim { - let (id, idx) = conv.id_alloc().open_delim(range, synth_id); let subtree = tt::Subtree { - delimiter: tt::Delimiter { open: id, close: tt::TokenId::UNSPECIFIED, kind }, + delimiter: tt::Delimiter { + // FIXME: Open and close spans + open: conv + .span_for(abs_range) + .unwrap_or(SpanData { range: rel_range, anchor }), + close: Span::DUMMY, + kind, + }, token_trees: vec![], }; - stack.push(StackEntry { subtree, idx, open_range: range }); + stack.push(subtree); continue; } @@ -279,39 +248,43 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { tt::Leaf::from(tt::Punct { char, spacing, - span: conv.id_alloc().alloc(range, synth_id), + span: conv.span_for(abs_range).unwrap_or(SpanData { range: rel_range, anchor }), }) .into() } else { macro_rules! make_leaf { ($i:ident) => { tt::$i { - span: conv.id_alloc().alloc(range, synth_id), + span: conv + .span_for(abs_range) + .unwrap_or(SpanData { range: rel_range, anchor }), text: token.to_text(conv), } .into() }; } - let leaf: tt::Leaf = match kind { + let leaf: tt::Leaf<_> = match kind { T![true] | T![false] => make_leaf!(Ident), IDENT => make_leaf!(Ident), UNDERSCORE => make_leaf!(Ident), k if k.is_keyword() => make_leaf!(Ident), k if k.is_literal() => make_leaf!(Literal), + // FIXME: Check whether span splitting works as intended LIFETIME_IDENT => { let char_unit = TextSize::of('\''); - let r = TextRange::at(range.start(), char_unit); + let r = TextRange::at(rel_range.start(), char_unit); let apostrophe = tt::Leaf::from(tt::Punct { char: '\'', spacing: tt::Spacing::Joint, - span: conv.id_alloc().alloc(r, synth_id), + span: conv.span_for(abs_range).unwrap_or(SpanData { range: r, anchor }), }); result.push(apostrophe.into()); - let r = TextRange::at(range.start() + char_unit, range.len() - char_unit); + let r = + TextRange::at(rel_range.start() + char_unit, rel_range.len() - char_unit); let ident = tt::Leaf::from(tt::Ident { text: SmolStr::new(&token.to_text(conv)[1..]), - span: conv.id_alloc().alloc(r, synth_id), + span: conv.span_for(abs_range).unwrap_or(SpanData { range: r, anchor }), }); result.push(ident.into()); continue; @@ -330,10 +303,9 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { while let Some(entry) = stack.pop() { let parent = stack.last_mut(); - conv.id_alloc().close_delim(entry.idx, None); - let leaf: tt::Leaf = tt::Punct { - span: conv.id_alloc().alloc(entry.open_range, None), - char: match entry.subtree.delimiter.kind { + let leaf: tt::Leaf<_> = tt::Punct { + span: entry.delimiter.open, + char: match entry.delimiter.kind { tt::DelimiterKind::Parenthesis => '(', tt::DelimiterKind::Brace => '{', tt::DelimiterKind::Bracket => '[', @@ -342,11 +314,11 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { spacing: tt::Spacing::Alone, } .into(); - parent.subtree.token_trees.push(leaf.into()); - parent.subtree.token_trees.extend(entry.subtree.token_trees); + parent.token_trees.push(leaf.into()); + parent.token_trees.extend(entry.token_trees); } - let subtree = stack.into_last().subtree; + let subtree = stack.into_last(); if let [tt::TokenTree::Subtree(first)] = &*subtree.token_trees { first.clone() } else { @@ -354,111 +326,6 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { } } -fn collect_tokens(conv: &mut C) { - struct StackEntry { - idx: usize, - open_range: TextRange, - delimiter: tt::DelimiterKind, - } - - let entry = StackEntry { - delimiter: tt::DelimiterKind::Invisible, - // never used (delimiter is `None`) - idx: !0, - open_range: TextRange::empty(TextSize::of('.')), - }; - let mut stack = NonEmptyVec::new(entry); - - loop { - let StackEntry { delimiter, .. } = stack.last_mut(); - let (token, range) = match conv.bump() { - Some(it) => it, - None => break, - }; - let synth_id = token.synthetic_id(conv); - - let kind = token.kind(conv); - if kind == COMMENT { - // Since `convert_doc_comment` can fail, we need to peek the next id, so that we can - // figure out which token id to use for the doc comment, if it is converted successfully. - let next_id = conv.id_alloc().peek_next_id(); - if let Some(_tokens) = conv.convert_doc_comment(&token, next_id) { - let id = conv.id_alloc().alloc(range, synth_id); - debug_assert_eq!(id, next_id); - } - continue; - } - if kind.is_punct() && kind != UNDERSCORE { - if synth_id.is_none() { - assert_eq!(range.len(), TextSize::of('.')); - } - - let expected = match delimiter { - tt::DelimiterKind::Parenthesis => Some(T![')']), - tt::DelimiterKind::Brace => Some(T!['}']), - tt::DelimiterKind::Bracket => Some(T![']']), - tt::DelimiterKind::Invisible => None, - }; - - if let Some(expected) = expected { - if kind == expected { - if let Some(entry) = stack.pop() { - conv.id_alloc().close_delim(entry.idx, Some(range)); - } - continue; - } - } - - let delim = match kind { - T!['('] => Some(tt::DelimiterKind::Parenthesis), - T!['{'] => Some(tt::DelimiterKind::Brace), - T!['['] => Some(tt::DelimiterKind::Bracket), - _ => None, - }; - - if let Some(kind) = delim { - let (_id, idx) = conv.id_alloc().open_delim(range, synth_id); - - stack.push(StackEntry { idx, open_range: range, delimiter: kind }); - continue; - } - - conv.id_alloc().alloc(range, synth_id); - } else { - macro_rules! make_leaf { - ($i:ident) => {{ - conv.id_alloc().alloc(range, synth_id); - }}; - } - match kind { - T![true] | T![false] => make_leaf!(Ident), - IDENT => make_leaf!(Ident), - UNDERSCORE => make_leaf!(Ident), - k if k.is_keyword() => make_leaf!(Ident), - k if k.is_literal() => make_leaf!(Literal), - LIFETIME_IDENT => { - let char_unit = TextSize::of('\''); - let r = TextRange::at(range.start(), char_unit); - conv.id_alloc().alloc(r, synth_id); - - let r = TextRange::at(range.start() + char_unit, range.len() - char_unit); - conv.id_alloc().alloc(r, synth_id); - continue; - } - _ => continue, - }; - }; - - // If we get here, we've consumed all input tokens. - // We might have more than one subtree in the stack, if the delimiters are improperly balanced. - // Merge them so we're left with one. - while let Some(entry) = stack.pop() { - conv.id_alloc().close_delim(entry.idx, None); - conv.id_alloc().alloc(entry.open_range, None); - } - } -} - fn is_single_token_op(kind: SyntaxKind) -> bool { matches!( kind, @@ -507,112 +374,54 @@ fn doc_comment_text(comment: &ast::Comment) -> SmolStr { text.into() } -fn convert_doc_comment( +fn convert_doc_comment( token: &syntax::SyntaxToken, - span: tt::TokenId, -) -> Option>> { + span: S, +) -> Option>> { cov_mark::hit!(test_meta_doc_comments); let comment = ast::Comment::cast(token.clone())?; let doc = comment.kind().doc?; - // Make `doc="\" Comments\"" - let meta_tkns = - vec![mk_ident("doc", span), mk_punct('=', span), mk_doc_literal(&comment, span)]; - - // Make `#![]` - let mut token_trees = Vec::with_capacity(3); - token_trees.push(mk_punct('#', span)); - if let ast::CommentPlacement::Inner = doc { - token_trees.push(mk_punct('!', span)); - } - token_trees.push(tt::TokenTree::from(tt::Subtree { - delimiter: tt::Delimiter { open: span, close: span, kind: tt::DelimiterKind::Bracket }, - token_trees: meta_tkns, - })); - - return Some(token_trees); + let mk_ident = + |s: &str| tt::TokenTree::from(tt::Leaf::from(tt::Ident { text: s.into(), span })); - // Helper functions - fn mk_ident(s: &str, span: tt::TokenId) -> tt::TokenTree { - tt::TokenTree::from(tt::Leaf::from(tt::Ident { text: s.into(), span })) - } - - fn mk_punct(c: char, span: tt::TokenId) -> tt::TokenTree { + let mk_punct = |c: char| { tt::TokenTree::from(tt::Leaf::from(tt::Punct { char: c, spacing: tt::Spacing::Alone, span, })) - } + }; - fn mk_doc_literal(comment: &ast::Comment, span: tt::TokenId) -> tt::TokenTree { + let mk_doc_literal = |comment: &ast::Comment| { let lit = tt::Literal { text: doc_comment_text(comment), span }; tt::TokenTree::from(tt::Leaf::from(lit)) - } -} - -struct TokenIdAlloc { - map: TokenMap, - global_offset: TextSize, - next_id: u32, -} - -impl TokenIdAlloc { - fn alloc( - &mut self, - absolute_range: TextRange, - synthetic_id: Option, - ) -> tt::TokenId { - let relative_range = absolute_range - self.global_offset; - let token_id = tt::TokenId(self.next_id); - self.next_id += 1; - self.map.insert(token_id, relative_range); - if let Some(id) = synthetic_id { - self.map.insert_synthetic(token_id, id); - } - token_id - } + }; - fn open_delim( - &mut self, - open_abs_range: TextRange, - synthetic_id: Option, - ) -> (tt::TokenId, usize) { - let token_id = tt::TokenId(self.next_id); - self.next_id += 1; - let idx = self.map.insert_delim( - token_id, - open_abs_range - self.global_offset, - open_abs_range - self.global_offset, - ); - if let Some(id) = synthetic_id { - self.map.insert_synthetic(token_id, id); - } - (token_id, idx) - } + // Make `doc="\" Comments\"" + let meta_tkns = vec![mk_ident("doc"), mk_punct('='), mk_doc_literal(&comment)]; - fn close_delim(&mut self, idx: usize, close_abs_range: Option) { - match close_abs_range { - None => { - self.map.remove_delim(idx); - } - Some(close) => { - self.map.update_close_delim(idx, close - self.global_offset); - } - } + // Make `#![]` + let mut token_trees = Vec::with_capacity(3); + token_trees.push(mk_punct('#')); + if let ast::CommentPlacement::Inner = doc { + token_trees.push(mk_punct('!')); } + token_trees.push(tt::TokenTree::from(tt::Subtree { + delimiter: tt::Delimiter { open: span, close: span, kind: tt::DelimiterKind::Bracket }, + token_trees: meta_tkns, + })); - fn peek_next_id(&self) -> tt::TokenId { - tt::TokenId(self.next_id) - } + Some(token_trees) } /// A raw token (straight from lexer) converter -struct RawConverter<'a> { +struct RawConverter<'a, SpanAnchor> { lexed: parser::LexedStr<'a>, pos: usize, - id_alloc: TokenIdAlloc, + _offset: TextSize, + file_id: SpanAnchor, } trait SrcToken: std::fmt::Debug { @@ -621,66 +430,64 @@ trait SrcToken: std::fmt::Debug { fn to_char(&self, ctx: &Ctx) -> Option; fn to_text(&self, ctx: &Ctx) -> SmolStr; - - fn synthetic_id(&self, ctx: &Ctx) -> Option; } -trait TokenConverter: Sized { +trait TokenConverter: Sized { type Token: SrcToken; fn convert_doc_comment( &self, token: &Self::Token, - span: tt::TokenId, - ) -> Option>>; + span: SpanData, + ) -> Option>>>; - fn bump(&mut self) -> Option<(Self::Token, TextRange)>; + fn bump(&mut self) -> Option<(Self::Token, TextRange, TextRange)>; fn peek(&self) -> Option; - fn id_alloc(&mut self) -> &mut TokenIdAlloc; + fn anchor(&self) -> SpanAnchor; + fn span_for(&self, range: TextRange) -> Option>; } -impl SrcToken> for usize { - fn kind(&self, ctx: &RawConverter<'_>) -> SyntaxKind { +impl SrcToken> for usize { + fn kind(&self, ctx: &RawConverter<'_, SpanAnchor>) -> SyntaxKind { ctx.lexed.kind(*self) } - fn to_char(&self, ctx: &RawConverter<'_>) -> Option { + fn to_char(&self, ctx: &RawConverter<'_, SpanAnchor>) -> Option { ctx.lexed.text(*self).chars().next() } - fn to_text(&self, ctx: &RawConverter<'_>) -> SmolStr { + fn to_text(&self, ctx: &RawConverter<'_, SpanAnchor>) -> SmolStr { ctx.lexed.text(*self).into() } - - fn synthetic_id(&self, _ctx: &RawConverter<'_>) -> Option { - None - } } -impl TokenConverter for RawConverter<'_> { +impl TokenConverter for RawConverter<'_, SpanAnchor> +where + SpanData: Span, +{ type Token = usize; fn convert_doc_comment( &self, &token: &usize, - span: tt::TokenId, - ) -> Option>> { + span: SpanData, + ) -> Option>>> { let text = self.lexed.text(token); convert_doc_comment(&doc_comment(text), span) } - fn bump(&mut self) -> Option<(Self::Token, TextRange)> { + fn bump(&mut self) -> Option<(Self::Token, TextRange, TextRange)> { if self.pos == self.lexed.len() { return None; } let token = self.pos; self.pos += 1; let range = self.lexed.text_range(token); - let range = TextRange::new(range.start.try_into().unwrap(), range.end.try_into().unwrap()); + let range = TextRange::new(range.start.try_into().ok()?, range.end.try_into().ok()?); - Some((token, range)) + Some((token, range, range)) } fn peek(&self) -> Option { @@ -690,77 +497,60 @@ impl TokenConverter for RawConverter<'_> { Some(self.pos) } - fn id_alloc(&mut self) -> &mut TokenIdAlloc { - &mut self.id_alloc + fn anchor(&self) -> SpanAnchor { + self.file_id + } + fn span_for(&self, _: TextRange) -> Option> { + None } } -struct Converter { - id_alloc: TokenIdAlloc, +struct Converter<'a, SpanAnchor> { current: Option, - current_synthetic: Vec, preorder: PreorderWithTokens, - replace: FxHashMap>, - append: FxHashMap>, range: TextRange, punct_offset: Option<(SyntaxToken, TextSize)>, + /// Used to make the emitted text ranges in the spans relative to the span anchor. + offset: TextSize, + file_id: SpanAnchor, + map: &'a TokenMap>, + censored: Vec, } -impl Converter { +impl<'a, SpanAnchor> Converter<'a, SpanAnchor> { fn new( node: &SyntaxNode, - global_offset: TextSize, - existing_token_map: TokenMap, - next_id: u32, - mut replace: FxHashMap>, - mut append: FxHashMap>, - ) -> Converter { + anchor_offset: TextSize, + file_id: SpanAnchor, + censored: Vec, + map: &'a TokenMap>, + ) -> Converter<'a, SpanAnchor> { let range = node.text_range(); let mut preorder = node.preorder_with_tokens(); - let (first, synthetic) = Self::next_token(&mut preorder, &mut replace, &mut append); + let first = Self::next_token(&mut preorder, &censored); Converter { - id_alloc: { TokenIdAlloc { map: existing_token_map, global_offset, next_id } }, current: first, - current_synthetic: synthetic, preorder, range, - replace, - append, punct_offset: None, + offset: anchor_offset, + file_id, + censored, + map, } } - fn next_token( - preorder: &mut PreorderWithTokens, - replace: &mut FxHashMap>, - append: &mut FxHashMap>, - ) -> (Option, Vec) { + fn next_token(preorder: &mut PreorderWithTokens, censor: &[SyntaxNode]) -> Option { while let Some(ev) = preorder.next() { - let ele = match ev { - WalkEvent::Enter(ele) => ele, - WalkEvent::Leave(ele) => { - if let Some(mut v) = append.remove(&ele) { - if !v.is_empty() { - v.reverse(); - return (None, v); - } - } - continue; + match ev { + WalkEvent::Enter(SyntaxElement::Token(t)) => return Some(t), + WalkEvent::Enter(SyntaxElement::Node(n)) if censor.contains(&n) => { + preorder.skip_subtree() } - }; - if let Some(mut v) = replace.remove(&ele) { - preorder.skip_subtree(); - if !v.is_empty() { - v.reverse(); - return (None, v); - } - } - match ele { - SyntaxElement::Token(t) => return (Some(t), Vec::new()), - _ => {} + _ => (), } } - (None, Vec::new()) + None } } @@ -768,100 +558,79 @@ impl Converter { enum SynToken { Ordinary(SyntaxToken), // FIXME is this supposed to be `Punct`? - Punch(SyntaxToken, TextSize), - Synthetic(SyntheticToken), + Punct(SyntaxToken, usize), } impl SynToken { - fn token(&self) -> Option<&SyntaxToken> { + fn token(&self) -> &SyntaxToken { match self { - SynToken::Ordinary(it) | SynToken::Punch(it, _) => Some(it), - SynToken::Synthetic(_) => None, + SynToken::Ordinary(it) | SynToken::Punct(it, _) => it, } } } -impl SrcToken for SynToken { - fn kind(&self, ctx: &Converter) -> SyntaxKind { +impl SrcToken> for SynToken { + fn kind(&self, ctx: &Converter<'_, SpanAnchor>) -> SyntaxKind { match self { SynToken::Ordinary(token) => token.kind(), - SynToken::Punch(..) => SyntaxKind::from_char(self.to_char(ctx).unwrap()).unwrap(), - SynToken::Synthetic(token) => token.kind, + SynToken::Punct(..) => SyntaxKind::from_char(self.to_char(ctx).unwrap()).unwrap(), } } - fn to_char(&self, _ctx: &Converter) -> Option { + fn to_char(&self, _ctx: &Converter<'_, SpanAnchor>) -> Option { match self { SynToken::Ordinary(_) => None, - SynToken::Punch(it, i) => it.text().chars().nth((*i).into()), - SynToken::Synthetic(token) if token.text.len() == 1 => token.text.chars().next(), - SynToken::Synthetic(_) => None, - } - } - fn to_text(&self, _ctx: &Converter) -> SmolStr { - match self { - SynToken::Ordinary(token) => token.text().into(), - SynToken::Punch(token, _) => token.text().into(), - SynToken::Synthetic(token) => token.text.clone(), + SynToken::Punct(it, i) => it.text().chars().nth(*i), } } - - fn synthetic_id(&self, _ctx: &Converter) -> Option { + fn to_text(&self, _ctx: &Converter<'_, SpanAnchor>) -> SmolStr { match self { - SynToken::Synthetic(token) => Some(token.id), - _ => None, + SynToken::Ordinary(token) | SynToken::Punct(token, _) => token.text().into(), } } } -impl TokenConverter for Converter { +impl TokenConverter for Converter<'_, SpanAnchor> +where + SpanData: Span, +{ type Token = SynToken; fn convert_doc_comment( &self, token: &Self::Token, - span: tt::TokenId, - ) -> Option>> { - convert_doc_comment(token.token()?, span) + span: SpanData, + ) -> Option>>> { + convert_doc_comment(token.token(), span) } - fn bump(&mut self) -> Option<(Self::Token, TextRange)> { + fn bump(&mut self) -> Option<(Self::Token, TextRange, TextRange)> { if let Some((punct, offset)) = self.punct_offset.clone() { if usize::from(offset) + 1 < punct.text().len() { let offset = offset + TextSize::of('.'); let range = punct.text_range(); self.punct_offset = Some((punct.clone(), offset)); let range = TextRange::at(range.start() + offset, TextSize::of('.')); - return Some((SynToken::Punch(punct, offset), range)); + return Some(( + SynToken::Punct(punct, u32::from(offset) as usize), + range - self.offset, + range, + )); } } - if let Some(synth_token) = self.current_synthetic.pop() { - if self.current_synthetic.is_empty() { - let (new_current, new_synth) = - Self::next_token(&mut self.preorder, &mut self.replace, &mut self.append); - self.current = new_current; - self.current_synthetic = new_synth; - } - let range = synth_token.range; - return Some((SynToken::Synthetic(synth_token), range)); - } - let curr = self.current.clone()?; if !self.range.contains_range(curr.text_range()) { return None; } - let (new_current, new_synth) = - Self::next_token(&mut self.preorder, &mut self.replace, &mut self.append); - self.current = new_current; - self.current_synthetic = new_synth; + self.current = Self::next_token(&mut self.preorder, &self.censored); let token = if curr.kind().is_punct() { self.punct_offset = Some((curr.clone(), 0.into())); let range = curr.text_range(); let range = TextRange::at(range.start(), TextSize::of('.')); - (SynToken::Punch(curr, 0.into()), range) + (SynToken::Punct(curr, 0 as usize), range - self.offset, range) } else { self.punct_offset = None; let range = curr.text_range(); - (SynToken::Ordinary(curr), range) + (SynToken::Ordinary(curr), range - self.offset, range) }; Some(token) @@ -871,54 +640,54 @@ impl TokenConverter for Converter { if let Some((punct, mut offset)) = self.punct_offset.clone() { offset += TextSize::of('.'); if usize::from(offset) < punct.text().len() { - return Some(SynToken::Punch(punct, offset)); + return Some(SynToken::Punct(punct, usize::from(offset))); } } - if let Some(synth_token) = self.current_synthetic.last() { - return Some(SynToken::Synthetic(synth_token.clone())); - } - let curr = self.current.clone()?; if !self.range.contains_range(curr.text_range()) { return None; } let token = if curr.kind().is_punct() { - SynToken::Punch(curr, 0.into()) + SynToken::Punct(curr, 0 as usize) } else { SynToken::Ordinary(curr) }; Some(token) } - fn id_alloc(&mut self) -> &mut TokenIdAlloc { - &mut self.id_alloc + fn anchor(&self) -> SpanAnchor { + self.file_id + } + fn span_for(&self, range: TextRange) -> Option> { + self.map.span_for_range(range) } } -struct TtTreeSink<'a> { +struct TtTreeSink<'a, SpanAnchor> { buf: String, - cursor: Cursor<'a, TokenId>, - open_delims: FxHashMap, + cursor: Cursor<'a, SpanData>, text_pos: TextSize, inner: SyntaxTreeBuilder, - token_map: TokenMap, + token_map: TokenMap>, } -impl<'a> TtTreeSink<'a> { - fn new(cursor: Cursor<'a, TokenId>) -> Self { +impl<'a, SpanAnchor> TtTreeSink<'a, SpanAnchor> +where + SpanData: Span, +{ + fn new(cursor: Cursor<'a, SpanData>) -> Self { TtTreeSink { buf: String::new(), cursor, - open_delims: FxHashMap::default(), text_pos: 0.into(), inner: SyntaxTreeBuilder::default(), token_map: TokenMap::default(), } } - fn finish(mut self) -> (Parse, TokenMap) { + fn finish(mut self) -> (Parse, TokenMap>) { self.token_map.shrink_to_fit(); (self.inner.finish(), self.token_map) } @@ -936,7 +705,10 @@ fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> Option<&'static str> { Some(&texts[idx..texts.len() - (1 - idx)]) } -impl TtTreeSink<'_> { +impl TtTreeSink<'_, SpanAnchor> +where + SpanData: Span, +{ /// Parses a float literal as if it was a one to two name ref nodes with a dot inbetween. /// This occurs when a float literal is used as a field access. fn float_split(&mut self, has_pseudo_dot: bool) { @@ -991,7 +763,7 @@ impl TtTreeSink<'_> { break match self.cursor.token_tree() { Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => { // Mark the range if needed - let (text, id) = match leaf { + let (text, span) = match leaf { tt::Leaf::Ident(ident) => (ident.text.as_str(), ident.span), tt::Leaf::Punct(punct) => { assert!(punct.char.is_ascii()); @@ -1004,7 +776,7 @@ impl TtTreeSink<'_> { tt::Leaf::Literal(lit) => (lit.text.as_str(), lit.span), }; let range = TextRange::at(self.text_pos, TextSize::of(text)); - self.token_map.insert(id, range); + self.token_map.insert(range, span); self.cursor = self.cursor.bump(); text } @@ -1012,7 +784,8 @@ impl TtTreeSink<'_> { self.cursor = self.cursor.subtree().unwrap(); match delim_to_str(subtree.delimiter.kind, false) { Some(it) => { - self.open_delims.insert(subtree.delimiter.open, self.text_pos); + let range = TextRange::at(self.text_pos, TextSize::of(it)); + self.token_map.insert(range, subtree.delimiter.open); it } None => continue, @@ -1023,18 +796,8 @@ impl TtTreeSink<'_> { self.cursor = self.cursor.bump(); match delim_to_str(parent.delimiter.kind, true) { Some(it) => { - if let Some(open_delim) = - self.open_delims.get(&parent.delimiter.open) - { - let open_range = TextRange::at(*open_delim, TextSize::of('(')); - let close_range = - TextRange::at(self.text_pos, TextSize::of('(')); - self.token_map.insert_delim( - parent.delimiter.open, - open_range, - close_range, - ); - } + let range = TextRange::at(self.text_pos, TextSize::of(it)); + self.token_map.insert(range, parent.delimiter.close); it } None => continue, diff --git a/crates/mbe/src/syntax_bridge/tests.rs b/crates/mbe/src/syntax_bridge/tests.rs index fa0125f3e9e0..32dfb4d7e0b3 100644 --- a/crates/mbe/src/syntax_bridge/tests.rs +++ b/crates/mbe/src/syntax_bridge/tests.rs @@ -4,24 +4,32 @@ use syntax::{ast, AstNode}; use test_utils::extract_annotations; use tt::{ buffer::{TokenBuffer, TokenTreeRef}, - Leaf, Punct, Spacing, + Leaf, Punct, Spacing, Span, }; +use crate::syntax_bridge::SpanData; + use super::syntax_node_to_token_tree; fn check_punct_spacing(fixture: &str) { + #[derive(PartialEq, Eq, Clone, Copy, Debug)] + struct DummyFile; + impl Span for DummyFile { + const DUMMY: Self = DummyFile; + } + let source_file = ast::SourceFile::parse(fixture).ok().unwrap(); - let (subtree, token_map) = syntax_node_to_token_tree(source_file.syntax()); + let subtree = + syntax_node_to_token_tree(source_file.syntax(), DummyFile, 0.into(), &Default::default()); let mut annotations: HashMap<_, _> = extract_annotations(fixture) .into_iter() .map(|(range, annotation)| { - let token = token_map.token_by_range(range).expect("no token found"); let spacing = match annotation.as_str() { "Alone" => Spacing::Alone, "Joint" => Spacing::Joint, a => panic!("unknown annotation: {a}"), }; - (token, spacing) + (range, spacing) }) .collect(); @@ -29,8 +37,12 @@ fn check_punct_spacing(fixture: &str) { let mut cursor = buf.begin(); while !cursor.eof() { while let Some(token_tree) = cursor.token_tree() { - if let TokenTreeRef::Leaf(Leaf::Punct(Punct { spacing, span, .. }), _) = token_tree { - if let Some(expected) = annotations.remove(span) { + if let TokenTreeRef::Leaf( + Leaf::Punct(Punct { spacing, span: SpanData { range, .. }, .. }), + _, + ) = token_tree + { + if let Some(expected) = annotations.remove(range) { assert_eq!(expected, *spacing); } } diff --git a/crates/mbe/src/token_map.rs b/crates/mbe/src/token_map.rs index 73a27df5dbca..1af50c8b3b95 100644 --- a/crates/mbe/src/token_map.rs +++ b/crates/mbe/src/token_map.rs @@ -2,123 +2,121 @@ use std::hash::Hash; -use parser::{SyntaxKind, T}; -use syntax::{TextRange, TextSize}; - -use crate::syntax_bridge::SyntheticTokenId; - -#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] -enum TokenTextRange { - Token(TextRange), - Delimiter(TextRange), +use syntax::TextRange; +use tt::Span; + +// pub type HirFile = u32; +// pub type FileRange = (HirFile, TextRange); +// Option, LocalSyntaxContet +// pub type SyntaxContext = (); +// pub type LocalSyntaxContext = u32; + +/// Maps absolute text ranges for the corresponding file to the relevant span data. +#[derive(Debug, PartialEq, Eq, Clone, Hash)] +// FIXME: Rename to SpanMap +pub struct TokenMap { + // FIXME: This needs to be sorted by (FileId, AstId) + // Then we can do a binary search on the file id, + // then a bin search on the ast id + pub span_map: Vec<(TextRange, S)>, + // span_map2: rustc_hash::FxHashMap, } -impl TokenTextRange { - fn by_kind(self, kind: SyntaxKind) -> Option { - match self { - TokenTextRange::Token(it) => Some(it), - TokenTextRange::Delimiter(it) => match kind { - T!['{'] | T!['('] | T!['['] => Some(TextRange::at(it.start(), 1.into())), - T!['}'] | T![')'] | T![']'] => { - Some(TextRange::at(it.end() - TextSize::of('}'), 1.into())) - } - _ => None, - }, - } +impl Default for TokenMap { + fn default() -> Self { + Self { span_map: Vec::new() } } } -/// Maps `tt::TokenId` to the relative range of the original token. -#[derive(Debug, PartialEq, Eq, Clone, Default, Hash)] -pub struct TokenMap { - /// Maps `tt::TokenId` to the *relative* source range. - entries: Vec<(tt::TokenId, TokenTextRange)>, - pub synthetic_entries: Vec<(tt::TokenId, SyntheticTokenId)>, -} - -impl TokenMap { - pub fn token_by_range(&self, relative_range: TextRange) -> Option { - let &(token_id, _) = self.entries.iter().find(|(_, range)| match range { - TokenTextRange::Token(it) => *it == relative_range, - TokenTextRange::Delimiter(it) => { - let open = TextRange::at(it.start(), 1.into()); - let close = TextRange::at(it.end() - TextSize::of('}'), 1.into()); - open == relative_range || close == relative_range - } - })?; - Some(token_id) - } - - pub fn ranges_by_token( - &self, - token_id: tt::TokenId, - kind: SyntaxKind, - ) -> impl Iterator + '_ { - self.entries - .iter() - .filter(move |&&(tid, _)| tid == token_id) - .filter_map(move |(_, range)| range.by_kind(kind)) - } - - pub fn synthetic_token_id(&self, token_id: tt::TokenId) -> Option { - self.synthetic_entries.iter().find(|(tid, _)| *tid == token_id).map(|(_, id)| *id) - } - - pub fn first_range_by_token( - &self, - token_id: tt::TokenId, - kind: SyntaxKind, - ) -> Option { - self.ranges_by_token(token_id, kind).next() - } - +impl TokenMap { pub(crate) fn shrink_to_fit(&mut self) { - self.entries.shrink_to_fit(); - self.synthetic_entries.shrink_to_fit(); - } - - pub(crate) fn insert(&mut self, token_id: tt::TokenId, relative_range: TextRange) { - self.entries.push((token_id, TokenTextRange::Token(relative_range))); - } - - pub(crate) fn insert_synthetic(&mut self, token_id: tt::TokenId, id: SyntheticTokenId) { - self.synthetic_entries.push((token_id, id)); + self.span_map.shrink_to_fit(); } - pub(crate) fn insert_delim( - &mut self, - token_id: tt::TokenId, - open_relative_range: TextRange, - close_relative_range: TextRange, - ) -> usize { - let res = self.entries.len(); - let cover = open_relative_range.cover(close_relative_range); - - self.entries.push((token_id, TokenTextRange::Delimiter(cover))); - res + pub(crate) fn insert(&mut self, range: TextRange, span: S) { + self.span_map.push((range, span)); } - pub(crate) fn update_close_delim(&mut self, idx: usize, close_relative_range: TextRange) { - let (_, token_text_range) = &mut self.entries[idx]; - if let TokenTextRange::Delimiter(dim) = token_text_range { - let cover = dim.cover(close_relative_range); - *token_text_range = TokenTextRange::Delimiter(cover); - } - } - - pub(crate) fn remove_delim(&mut self, idx: usize) { - // FIXME: This could be accidentally quadratic - self.entries.remove(idx); + pub fn ranges_with_span(&self, span: S) -> impl Iterator + '_ { + self.span_map.iter().filter_map( + move |(range, s)| { + if s == &span { + Some(*range) + } else { + None + } + }, + ) } - pub fn entries(&self) -> impl Iterator + '_ { - self.entries.iter().filter_map(|&(tid, tr)| match tr { - TokenTextRange::Token(range) => Some((tid, range)), - TokenTextRange::Delimiter(_) => None, - }) + pub fn span_for_range(&self, range: TextRange) -> Option { + self.span_map.iter().find_map(|(r, s)| if r == &range { Some(s.clone()) } else { None }) } - pub fn filter(&mut self, id: impl Fn(tt::TokenId) -> bool) { - self.entries.retain(|&(tid, _)| id(tid)); - } + // pub fn ranges_by_token( + // &self, + // token_id: tt::TokenId, + // kind: SyntaxKind, + // ) -> impl Iterator + '_ { + // self.entries + // .iter() + // .filter(move |&&(tid, _)| tid == token_id) + // .filter_map(move |(_, range)| range.by_kind(kind)) + // } + + // pub(crate) fn remove_delim(&mut self, idx: usize) { + // // FIXME: This could be accidentally quadratic + // self.entries.remove(idx); + // } + + // pub fn entries(&self) -> impl Iterator + '_ { + // self.entries.iter().filter_map(|&(tid, tr)| match tr { + // TokenTextRange::Token(range) => Some((tid, range)), + // TokenTextRange::Delimiter(_) => None, + // }) + // } + + // pub fn filter(&mut self, id: impl Fn(tt::TokenId) -> bool) { + // self.entries.retain(|&(tid, _)| id(tid)); + // } + // pub fn synthetic_token_id(&self, token_id: tt::TokenId) -> Option { + // self.synthetic_entries.iter().find(|(tid, _)| *tid == token_id).map(|(_, id)| *id) + // } + + // pub fn first_range_by_token( + // &self, + // token_id: tt::TokenId, + // kind: SyntaxKind, + // ) -> Option { + // self.ranges_by_token(token_id, kind).next() + // } + + // pub(crate) fn insert(&mut self, token_id: tt::TokenId, relative_range: TextRange) { + // self.entries.push((token_id, TokenTextRange::Token(relative_range))); + // } + + // pub(crate) fn insert_synthetic(&mut self, token_id: tt::TokenId, id: SyntheticTokenId) { + // self.synthetic_entries.push((token_id, id)); + // } + + // pub(crate) fn insert_delim( + // &mut self, + // token_id: tt::TokenId, + // open_relative_range: TextRange, + // close_relative_range: TextRange, + // ) -> usize { + // let res = self.entries.len(); + // let cover = open_relative_range.cover(close_relative_range); + + // self.entries.push((token_id, TokenTextRange::Delimiter(cover))); + // res + // } + + // pub(crate) fn update_close_delim(&mut self, idx: usize, close_relative_range: TextRange) { + // let (_, token_text_range) = &mut self.entries[idx]; + // if let TokenTextRange::Delimiter(dim) = token_text_range { + // let cover = dim.cover(close_relative_range); + // *token_text_range = TokenTextRange::Delimiter(cover); + // } + // } } diff --git a/crates/proc-macro-api/Cargo.toml b/crates/proc-macro-api/Cargo.toml index 2c2d2e8a9452..4c87c89adde9 100644 --- a/crates/proc-macro-api/Cargo.toml +++ b/crates/proc-macro-api/Cargo.toml @@ -31,5 +31,6 @@ paths.workspace = true tt.workspace = true stdx.workspace = true profile.workspace = true +text-size.workspace = true # Intentionally *not* depend on anything salsa-related # base-db.workspace = true diff --git a/crates/proc-macro-api/src/msg/flat.rs b/crates/proc-macro-api/src/msg/flat.rs index fe82b8d04542..bfb3213a25d9 100644 --- a/crates/proc-macro-api/src/msg/flat.rs +++ b/crates/proc-macro-api/src/msg/flat.rs @@ -38,6 +38,7 @@ use std::collections::{HashMap, VecDeque}; use serde::{Deserialize, Serialize}; +use text_size::TextRange; use tt::Span; use crate::msg::{ENCODE_CLOSE_SPAN_VERSION, VARIABLE_SIZED_SPANS}; @@ -55,6 +56,19 @@ impl SerializableSpan<1> for tt::TokenId { } } +impl SerializableSpan<3> for tt::SpanData +where + FileId: From + Into, + Self: Span, +{ + fn into_u32(self) -> [u32; 3] { + [self.anchor.into(), self.range.start().into(), self.range.end().into()] + } + fn from_u32([file_id, start, end]: [u32; 3]) -> Self { + tt::SpanData { anchor: file_id.into(), range: TextRange::new(start.into(), end.into()) } + } +} + #[derive(Serialize, Deserialize, Debug)] pub struct FlatTree { subtree: Vec, diff --git a/crates/rust-analyzer/src/cargo_target_spec.rs b/crates/rust-analyzer/src/cargo_target_spec.rs index c7b84c41b33e..b6dcc26de621 100644 --- a/crates/rust-analyzer/src/cargo_target_spec.rs +++ b/crates/rust-analyzer/src/cargo_target_spec.rs @@ -209,17 +209,24 @@ mod tests { use super::*; use cfg::CfgExpr; + use hir::HirFileId; + use ide_db::base_db::span::{SpanAnchor, ROOT_ERASED_FILE_AST_ID}; use mbe::syntax_node_to_token_tree; use syntax::{ ast::{self, AstNode}, - SmolStr, + SmolStr, TextSize, }; fn check(cfg: &str, expected_features: &[&str]) { let cfg_expr = { let source_file = ast::SourceFile::parse(cfg).ok().unwrap(); let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - let (tt, _) = syntax_node_to_token_tree(tt.syntax()); + let tt = syntax_node_to_token_tree( + tt.syntax(), + SpanAnchor { file_id: HirFileId::from(0), ast_id: ROOT_ERASED_FILE_AST_ID }, + TextSize::new(0), + &Default::default(), + ); CfgExpr::parse(&tt) }; diff --git a/crates/rust-analyzer/src/cli/analysis_stats.rs b/crates/rust-analyzer/src/cli/analysis_stats.rs index 0f6539f224d7..7e795cf46323 100644 --- a/crates/rust-analyzer/src/cli/analysis_stats.rs +++ b/crates/rust-analyzer/src/cli/analysis_stats.rs @@ -8,7 +8,7 @@ use std::{ use hir::{ db::{DefDatabase, ExpandDatabase, HirDatabase}, - Adt, AssocItem, Crate, DefWithBody, HasSource, HirDisplay, ModuleDef, Name, + Adt, AssocItem, Crate, DefWithBody, HasSource, HirDisplay, HirFileIdExt, ModuleDef, Name, }; use hir_def::{ body::{BodySourceMap, SyntheticSyntax}, diff --git a/crates/rust-analyzer/src/cli/diagnostics.rs b/crates/rust-analyzer/src/cli/diagnostics.rs index 8541be715a93..abec2679464e 100644 --- a/crates/rust-analyzer/src/cli/diagnostics.rs +++ b/crates/rust-analyzer/src/cli/diagnostics.rs @@ -4,7 +4,7 @@ use project_model::{CargoConfig, RustLibSource}; use rustc_hash::FxHashSet; -use hir::{db::HirDatabase, Crate, Module}; +use hir::{db::HirDatabase, Crate, HirFileIdExt, Module}; use ide::{AssistResolveStrategy, DiagnosticsConfig, Severity}; use ide_db::base_db::SourceDatabaseExt; use load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice}; diff --git a/crates/syntax/src/lib.rs b/crates/syntax/src/lib.rs index 4939ab39049c..1d7b7de390e4 100644 --- a/crates/syntax/src/lib.rs +++ b/crates/syntax/src/lib.rs @@ -75,7 +75,7 @@ pub use smol_str::SmolStr; #[derive(Debug, PartialEq, Eq)] pub struct Parse { green: GreenNode, - errors: Arc<[SyntaxError]>, + errors: Option>, _ty: PhantomData T>, } @@ -87,14 +87,18 @@ impl Clone for Parse { impl Parse { fn new(green: GreenNode, errors: Vec) -> Parse { - Parse { green, errors: errors.into(), _ty: PhantomData } + Parse { + green, + errors: if errors.is_empty() { None } else { Some(errors.into()) }, + _ty: PhantomData, + } } pub fn syntax_node(&self) -> SyntaxNode { SyntaxNode::new_root(self.green.clone()) } pub fn errors(&self) -> &[SyntaxError] { - &self.errors + self.errors.as_deref().unwrap_or_default() } } @@ -108,10 +112,9 @@ impl Parse { } pub fn ok(self) -> Result> { - if self.errors.is_empty() { - Ok(self.tree()) - } else { - Err(self.errors) + match self.errors { + Some(e) => Err(e), + None => Ok(self.tree()), } } } @@ -129,7 +132,7 @@ impl Parse { impl Parse { pub fn debug_dump(&self) -> String { let mut buf = format!("{:#?}", self.tree().syntax()); - for err in self.errors.iter() { + for err in self.errors.as_deref().into_iter().flat_map(<[_]>::iter) { format_to!(buf, "error {:?}: {}\n", err.range(), err); } buf @@ -141,13 +144,16 @@ impl Parse { fn incremental_reparse(&self, indel: &Indel) -> Option> { // FIXME: validation errors are not handled here - parsing::incremental_reparse(self.tree().syntax(), indel, self.errors.to_vec()).map( - |(green_node, errors, _reparsed_range)| Parse { - green: green_node, - errors: errors.into(), - _ty: PhantomData, - }, + parsing::incremental_reparse( + self.tree().syntax(), + indel, + self.errors.as_deref().unwrap_or_default().iter().cloned(), ) + .map(|(green_node, errors, _reparsed_range)| Parse { + green: green_node, + errors: if errors.is_empty() { None } else { Some(errors.into()) }, + _ty: PhantomData, + }) } fn full_reparse(&self, indel: &Indel) -> Parse { @@ -168,7 +174,11 @@ impl SourceFile { errors.extend(validation::validate(&root)); assert_eq!(root.kind(), SyntaxKind::SOURCE_FILE); - Parse { green, errors: errors.into(), _ty: PhantomData } + Parse { + green, + errors: if errors.is_empty() { None } else { Some(errors.into()) }, + _ty: PhantomData, + } } } @@ -275,7 +285,11 @@ impl ast::TokenTree { let (green, errors) = builder.finish_raw(); - Parse { green, errors: errors.into(), _ty: PhantomData } + Parse { + green, + errors: if errors.is_empty() { None } else { Some(errors.into()) }, + _ty: PhantomData, + } } } diff --git a/crates/syntax/src/parsing/reparsing.rs b/crates/syntax/src/parsing/reparsing.rs index 45e591609828..0ddc641711fb 100644 --- a/crates/syntax/src/parsing/reparsing.rs +++ b/crates/syntax/src/parsing/reparsing.rs @@ -20,7 +20,7 @@ use crate::{ pub(crate) fn incremental_reparse( node: &SyntaxNode, edit: &Indel, - errors: Vec, + errors: impl IntoIterator, ) -> Option<(GreenNode, Vec, TextRange)> { if let Some((green, new_errors, old_range)) = reparse_token(node, edit) { return Some((green, merge_errors(errors, new_errors, old_range, edit), old_range)); @@ -147,7 +147,7 @@ fn is_balanced(lexed: &parser::LexedStr<'_>) -> bool { } fn merge_errors( - old_errors: Vec, + old_errors: impl IntoIterator, new_errors: Vec, range_before_reparse: TextRange, edit: &Indel, @@ -191,8 +191,12 @@ mod tests { let fully_reparsed = SourceFile::parse(&after); let incrementally_reparsed: Parse = { let before = SourceFile::parse(&before); - let (green, new_errors, range) = - incremental_reparse(before.tree().syntax(), &edit, before.errors.to_vec()).unwrap(); + let (green, new_errors, range) = incremental_reparse( + before.tree().syntax(), + &edit, + before.errors.as_deref().unwrap_or_default().iter().cloned(), + ) + .unwrap(); assert_eq!(range.len(), reparsed_len.into(), "reparsed fragment has wrong length"); Parse::new(green, new_errors) }; diff --git a/crates/syntax/src/tests.rs b/crates/syntax/src/tests.rs index 3010d77d827e..8ae1242cf7fd 100644 --- a/crates/syntax/src/tests.rs +++ b/crates/syntax/src/tests.rs @@ -38,7 +38,7 @@ fn benchmark_parser() { let tree = { let _b = bench("parsing"); let p = SourceFile::parse(&data); - assert!(p.errors.is_empty()); + assert!(p.errors.is_none()); assert_eq!(p.tree().syntax.text_range().len(), 352474.into()); p.tree() }; diff --git a/crates/tt/Cargo.toml b/crates/tt/Cargo.toml index a28ee5f1ca2b..57222449790e 100644 --- a/crates/tt/Cargo.toml +++ b/crates/tt/Cargo.toml @@ -13,5 +13,6 @@ doctest = false [dependencies] smol_str.workspace = true +text-size.workspace = true stdx.workspace = true diff --git a/crates/tt/src/lib.rs b/crates/tt/src/lib.rs index a4ffc328f217..89cb12d2c267 100644 --- a/crates/tt/src/lib.rs +++ b/crates/tt/src/lib.rs @@ -7,6 +7,7 @@ use std::fmt; use stdx::impl_from; +use text_size::{TextRange, TextSize}; pub use smol_str::SmolStr; @@ -31,36 +32,25 @@ impl TokenId { Self::UNSPECIFIED } } - -pub mod token_id { - pub use crate::{DelimiterKind, Spacing, TokenId}; - pub type Span = crate::TokenId; - pub type Subtree = crate::Subtree; - pub type Punct = crate::Punct; - pub type Delimiter = crate::Delimiter; - pub type Leaf = crate::Leaf; - pub type Ident = crate::Ident; - pub type Literal = crate::Literal; - pub type TokenTree = crate::TokenTree; - pub mod buffer { - pub type TokenBuffer<'a> = crate::buffer::TokenBuffer<'a, super::Span>; - pub type Cursor<'a> = crate::buffer::Cursor<'a, super::Span>; - pub type TokenTreeRef<'a> = crate::buffer::TokenTreeRef<'a, super::Span>; - } +impl Span for TokenId { + const DUMMY: Self = TokenId(!0); } -pub trait Span: std::fmt::Debug + Copy + Sized { - const DUMMY: Self; - fn is_dummy(&self) -> bool; +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +pub struct SpanData { + /// The text range of this span, relative to the anchor. + pub range: TextRange, + pub anchor: Anchor, } -impl Span for TokenId { - const DUMMY: Self = TokenId(!0); - fn is_dummy(&self) -> bool { - *self == Self::DUMMY - } +impl Span for SpanData { + const DUMMY: Self = + SpanData { range: TextRange::empty(TextSize::new(0)), anchor: Anchor::DUMMY }; } +pub trait Span: std::fmt::Debug + Copy + Sized + Eq { + const DUMMY: Self; +} #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct SyntaxContext(pub u32); @@ -134,7 +124,6 @@ impl Delimiter { } } - #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum DelimiterKind { Parenthesis, From e36b3f7b8cabcf34b89a2be14eff474815476590 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Fri, 6 Oct 2023 14:47:11 +0200 Subject: [PATCH 04/24] Proper span representation with syntax context --- crates/base-db/src/span.rs | 16 +- crates/cfg/src/tests.rs | 37 +- crates/hir-def/src/attr/tests.rs | 16 +- crates/hir-expand/src/attrs.rs | 2 +- crates/hir-expand/src/builtin_fn_macro.rs | 3 +- crates/hir-expand/src/db.rs | 26 +- crates/hir-expand/src/hygiene.rs | 242 +++-------- crates/hir-expand/src/lib.rs | 40 +- crates/hir-expand/src/quote.rs | 4 +- crates/mbe/src/benchmark.rs | 36 +- crates/mbe/src/syntax_bridge.rs | 376 +++++++++--------- crates/mbe/src/syntax_bridge/tests.rs | 14 +- crates/proc-macro-api/src/msg.rs | 7 +- crates/proc-macro-api/src/msg/flat.rs | 31 +- crates/rust-analyzer/src/cargo_target_spec.rs | 4 +- crates/tt/src/lib.rs | 28 +- 16 files changed, 413 insertions(+), 469 deletions(-) diff --git a/crates/base-db/src/span.rs b/crates/base-db/src/span.rs index 1072d937a3bd..acc1e5243f73 100644 --- a/crates/base-db/src/span.rs +++ b/crates/base-db/src/span.rs @@ -1,6 +1,7 @@ use std::fmt; use salsa::InternId; +use tt::SyntaxContext; use vfs::FileId; pub type ErasedFileAstId = la_arena::Idx; @@ -9,10 +10,17 @@ pub type ErasedFileAstId = la_arena::Idx; pub const ROOT_ERASED_FILE_AST_ID: ErasedFileAstId = la_arena::Idx::from_raw(la_arena::RawIdx::from_u32(0)); -#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] -pub struct SyntaxContext; +pub type SpanData = tt::SpanData; -pub type SpanData = tt::SpanData; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct SyntaxContextId(InternId); +crate::impl_intern_key!(SyntaxContextId); + +impl SyntaxContext for SyntaxContextId { + // FIXME: This is very much UB, salsa exposes no way to create an InternId in a const context + // currently (which kind of makes sense but we need it here!) + const DUMMY: Self = SyntaxContextId(unsafe { core::mem::transmute(1) }); +} #[derive(Copy, Clone, PartialEq, Eq, Hash)] pub struct SpanAnchor { @@ -26,7 +34,7 @@ impl fmt::Debug for SpanAnchor { } } -impl tt::Span for SpanAnchor { +impl tt::SpanAnchor for SpanAnchor { const DUMMY: Self = SpanAnchor { file_id: HirFileId(0), ast_id: ROOT_ERASED_FILE_AST_ID }; } diff --git a/crates/cfg/src/tests.rs b/crates/cfg/src/tests.rs index 242929c006a5..0ea176858c94 100644 --- a/crates/cfg/src/tests.rs +++ b/crates/cfg/src/tests.rs @@ -2,20 +2,30 @@ use arbitrary::{Arbitrary, Unstructured}; use expect_test::{expect, Expect}; use mbe::syntax_node_to_token_tree; use syntax::{ast, AstNode}; -use tt::Span; +use tt::{SpanAnchor, SyntaxContext}; use crate::{CfgAtom, CfgExpr, CfgOptions, DnfExpr}; #[derive(Debug, Copy, Clone, PartialEq, Eq)] struct DummyFile; -impl Span for DummyFile { +impl SpanAnchor for DummyFile { const DUMMY: Self = DummyFile; } +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +struct DummyCtx; +impl SyntaxContext for DummyCtx { + const DUMMY: Self = DummyCtx; +} fn assert_parse_result(input: &str, expected: CfgExpr) { let source_file = ast::SourceFile::parse(input).ok().unwrap(); let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - let tt = syntax_node_to_token_tree(tt.syntax(), DummyFile, 0.into(), &Default::default()); + let tt = syntax_node_to_token_tree::<_, DummyCtx>( + tt.syntax(), + DummyFile, + 0.into(), + &Default::default(), + ); let cfg = CfgExpr::parse(&tt); assert_eq!(cfg, expected); } @@ -23,7 +33,12 @@ fn assert_parse_result(input: &str, expected: CfgExpr) { fn check_dnf(input: &str, expect: Expect) { let source_file = ast::SourceFile::parse(input).ok().unwrap(); let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - let tt = syntax_node_to_token_tree(tt.syntax(), DummyFile, 0.into(), &Default::default()); + let tt = syntax_node_to_token_tree::<_, DummyCtx>( + tt.syntax(), + DummyFile, + 0.into(), + &Default::default(), + ); let cfg = CfgExpr::parse(&tt); let actual = format!("#![cfg({})]", DnfExpr::new(cfg)); expect.assert_eq(&actual); @@ -32,7 +47,12 @@ fn check_dnf(input: &str, expect: Expect) { fn check_why_inactive(input: &str, opts: &CfgOptions, expect: Expect) { let source_file = ast::SourceFile::parse(input).ok().unwrap(); let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - let tt = syntax_node_to_token_tree(tt.syntax(), DummyFile, 0.into(), &Default::default()); + let tt = syntax_node_to_token_tree::<_, DummyCtx>( + tt.syntax(), + DummyFile, + 0.into(), + &Default::default(), + ); let cfg = CfgExpr::parse(&tt); let dnf = DnfExpr::new(cfg); let why_inactive = dnf.why_inactive(opts).unwrap().to_string(); @@ -43,7 +63,12 @@ fn check_why_inactive(input: &str, opts: &CfgOptions, expect: Expect) { fn check_enable_hints(input: &str, opts: &CfgOptions, expected_hints: &[&str]) { let source_file = ast::SourceFile::parse(input).ok().unwrap(); let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - let tt = syntax_node_to_token_tree(tt.syntax(), DummyFile, 0.into(), &Default::default()); + let tt = syntax_node_to_token_tree::<_, DummyCtx>( + tt.syntax(), + DummyFile, + 0.into(), + &Default::default(), + ); let cfg = CfgExpr::parse(&tt); let dnf = DnfExpr::new(cfg); let hints = dnf.compute_enable_hints(opts).map(|diff| diff.to_string()).collect::>(); diff --git a/crates/hir-def/src/attr/tests.rs b/crates/hir-def/src/attr/tests.rs index ad101e9bdf77..60e5cebd3cb0 100644 --- a/crates/hir-def/src/attr/tests.rs +++ b/crates/hir-def/src/attr/tests.rs @@ -4,15 +4,25 @@ use base_db::span::SpanAnchor; use mbe::syntax_node_to_token_tree; use syntax::{ast, AstNode}; -use tt::Span; +use tt::{SpanAnchor as _, SyntaxContext}; use crate::attr::{DocAtom, DocExpr}; +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +struct DummyCtx; +impl SyntaxContext for DummyCtx { + const DUMMY: Self = DummyCtx; +} + fn assert_parse_result(input: &str, expected: DocExpr) { let source_file = ast::SourceFile::parse(input).ok().unwrap(); let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - let tt = - syntax_node_to_token_tree(tt.syntax(), SpanAnchor::DUMMY, 0.into(), &Default::default()); + let tt = syntax_node_to_token_tree::<_, DummyCtx>( + tt.syntax(), + SpanAnchor::DUMMY, + 0.into(), + &Default::default(), + ); let cfg = DocExpr::parse(&tt); assert_eq!(cfg, expected); } diff --git a/crates/hir-expand/src/attrs.rs b/crates/hir-expand/src/attrs.rs index 9652dd345a8d..01a66cd03ab1 100644 --- a/crates/hir-expand/src/attrs.rs +++ b/crates/hir-expand/src/attrs.rs @@ -1,7 +1,7 @@ //! A higher level attributes based on TokenTree, with also some shortcuts. use std::{fmt, ops}; -use ::tt::Span; +use ::tt::SpanAnchor as _; use base_db::{span::SpanAnchor, CrateId}; use cfg::CfgExpr; use either::Either; diff --git a/crates/hir-expand/src/builtin_fn_macro.rs b/crates/hir-expand/src/builtin_fn_macro.rs index adbe49473ace..2a541a36735b 100644 --- a/crates/hir-expand/src/builtin_fn_macro.rs +++ b/crates/hir-expand/src/builtin_fn_macro.rs @@ -1,5 +1,6 @@ //! Builtin macro +use ::tt::Span; use base_db::{ span::{SpanAnchor, ROOT_ERASED_FILE_AST_ID}, AnchoredPath, Edition, FileId, @@ -15,7 +16,7 @@ use syntax::{ use crate::{ db::ExpandDatabase, name, quote, - tt::{self, Span}, + tt::{self}, EagerCallInfo, ExpandError, ExpandResult, HirFileIdExt, MacroCallId, MacroCallLoc, }; diff --git a/crates/hir-expand/src/db.rs b/crates/hir-expand/src/db.rs index 32ba7b2f9115..1a68653a6fcd 100644 --- a/crates/hir-expand/src/db.rs +++ b/crates/hir-expand/src/db.rs @@ -1,8 +1,9 @@ //! Defines database & queries for macro expansion. +use ::tt::SyntaxContext; use base_db::{ salsa, - span::{SpanAnchor, ROOT_ERASED_FILE_AST_ID}, + span::{SpanAnchor, SyntaxContextId, ROOT_ERASED_FILE_AST_ID}, CrateId, Edition, SourceDatabase, }; use either::Either; @@ -15,11 +16,13 @@ use syntax::{ use triomphe::Arc; use crate::{ - ast_id_map::AstIdMap, builtin_attr_macro::pseudo_derive_attr_expansion, - builtin_fn_macro::EagerExpander, hygiene::HygieneFrame, tt, AstId, BuiltinAttrExpander, - BuiltinDeriveExpander, BuiltinFnLikeExpander, EagerCallInfo, ExpandError, ExpandResult, - ExpandTo, HirFileId, HirFileIdRepr, MacroCallId, MacroCallKind, MacroCallLoc, MacroDefId, - MacroDefKind, MacroFile, ProcMacroExpander, SpanMap, SyntaxContext, SyntaxContextId, + ast_id_map::AstIdMap, + builtin_attr_macro::pseudo_derive_attr_expansion, + builtin_fn_macro::EagerExpander, + hygiene::{self, HygieneFrame, SyntaxContextData}, + tt, AstId, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander, EagerCallInfo, + ExpandError, ExpandResult, ExpandTo, HirFileId, HirFileIdRepr, MacroCallId, MacroCallKind, + MacroCallLoc, MacroDefId, MacroDefKind, MacroFile, ProcMacroExpander, SpanMap, }; /// Total limit on the number of tokens produced by any macro invocation. @@ -89,7 +92,15 @@ pub trait ExpandDatabase: SourceDatabase { #[salsa::interned] fn intern_macro_call(&self, macro_call: MacroCallLoc) -> MacroCallId; #[salsa::interned] - fn intern_syntax_context(&self, ctx: SyntaxContext) -> SyntaxContextId; + fn intern_syntax_context(&self, ctx: SyntaxContextData) -> SyntaxContextId; + #[salsa::transparent] + #[salsa::invoke(hygiene::apply_mark)] + fn apply_mark( + &self, + ctxt: SyntaxContextData, + file_id: HirFileId, + transparency: hygiene::Transparency, + ) -> SyntaxContextId; /// Lowers syntactic macro call to a token tree representation. That's a firewall /// query, only typing in the macro call itself changes the returned @@ -225,6 +236,7 @@ pub fn expand_speculative( .ranges_with_span(tt::SpanData { range: token_to_map.text_range(), anchor: SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, + ctx: SyntaxContextId::DUMMY, }) .filter_map(|range| syntax_node.covering_element(range).into_token()) .min_by_key(|t| { diff --git a/crates/hir-expand/src/hygiene.rs b/crates/hir-expand/src/hygiene.rs index ce421d3dcd82..e0688178ffb9 100644 --- a/crates/hir-expand/src/hygiene.rs +++ b/crates/hir-expand/src/hygiene.rs @@ -2,71 +2,92 @@ //! //! Specifically, `ast` + `Hygiene` allows you to create a `Name`. Note that, at //! this moment, this is horribly incomplete and handles only `$crate`. -use base_db::CrateId; -use db::TokenExpander; +use base_db::{span::SyntaxContextId, CrateId}; use either::Either; use syntax::{ - ast::{self, HasDocComments}, - AstNode, SyntaxNode, TextRange, TextSize, + ast::{self}, + TextRange, }; use triomphe::Arc; use crate::{ - db::{self, ExpandDatabase}, + db::ExpandDatabase, name::{AsName, Name}, - HirFileId, InFile, MacroCallKind, MacroCallLoc, MacroDefKind, MacroFile, SpanMap, + HirFileId, InFile, }; -#[derive(Clone, Debug)] -pub struct Hygiene { - frames: Option, +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct SyntaxContextData { + // FIXME: This might only need to be Option? + outer_expn: HirFileId, + outer_transparency: Transparency, + parent: SyntaxContextId, + /// This context, but with all transparent and semi-transparent expansions filtered away. + opaque: SyntaxContextId, + /// This context, but with all transparent expansions filtered away. + opaque_and_semitransparent: SyntaxContextId, + /// Name of the crate to which `$crate` with this context would resolve. + dollar_crate_name: Name, +} + +/// A property of a macro expansion that determines how identifiers +/// produced by that expansion are resolved. +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Hash, Debug)] +pub enum Transparency { + /// Identifier produced by a transparent expansion is always resolved at call-site. + /// Call-site spans in procedural macros, hygiene opt-out in `macro` should use this. + Transparent, + /// Identifier produced by a semi-transparent expansion may be resolved + /// either at call-site or at definition-site. + /// If it's a local variable, label or `$crate` then it's resolved at def-site. + /// Otherwise it's resolved at call-site. + /// `macro_rules` macros behave like this, built-in macros currently behave like this too, + /// but that's an implementation detail. + SemiTransparent, + /// Identifier produced by an opaque expansion is always resolved at definition-site. + /// Def-site spans in procedural macros, identifiers from `macro` by default use this. + Opaque, +} + +pub(super) fn apply_mark( + _db: &dyn ExpandDatabase, + _ctxt: SyntaxContextData, + _file_id: HirFileId, + _transparency: Transparency, +) -> SyntaxContextId { + _db.intern_syntax_context(_ctxt) } +// pub(super) fn with_ctxt_from_mark(db: &ExpandDatabase, file_id: HirFileId) { +// self.with_ctxt_from_mark(expn_id, Transparency::Transparent) +// } +// pub(super) fn with_call_site_ctxt(db: &ExpandDatabase, file_id: HirFileId) { +// self.with_ctxt_from_mark(expn_id, Transparency::Transparent) +// } + +#[derive(Clone, Debug)] +pub struct Hygiene {} + impl Hygiene { - pub fn new(db: &dyn ExpandDatabase, file_id: HirFileId) -> Hygiene { - Hygiene { frames: Some(HygieneFrames::new(db, file_id)) } + pub fn new(_: &dyn ExpandDatabase, _: HirFileId) -> Hygiene { + Hygiene {} } pub fn new_unhygienic() -> Hygiene { - Hygiene { frames: None } + Hygiene {} } // FIXME: this should just return name pub fn name_ref_to_name( &self, - db: &dyn ExpandDatabase, + _: &dyn ExpandDatabase, name_ref: ast::NameRef, ) -> Either { - if let Some(frames) = &self.frames { - if name_ref.text() == "$crate" { - if let Some(krate) = frames.root_crate(db, name_ref.syntax()) { - return Either::Right(krate); - } - } - } - Either::Left(name_ref.as_name()) } - pub fn local_inner_macros(&self, _db: &dyn ExpandDatabase, path: ast::Path) -> Option { - let mut _token = path.syntax().first_token()?.text_range(); - let frames = self.frames.as_ref()?; - let mut _current = &frames.0; - - // FIXME: Hygiene ... - return None; - // loop { - // let (mapped, origin) = current.expansion.as_ref()?.map_ident_up(db, token)?; - // if origin == Origin::Def { - // return if current.local_inner { - // frames.root_crate(db, path.syntax()) - // } else { - // None - // }; - // } - // current = current.call_site.as_ref()?; - // token = mapped.value; - // } + pub fn local_inner_macros(&self, _: &dyn ExpandDatabase, _: ast::Path) -> Option { + None } } @@ -74,150 +95,19 @@ impl Hygiene { struct HygieneFrames(Arc); #[derive(Clone, Debug, Eq, PartialEq)] -pub struct HygieneFrame { - expansion: Option, - - // Indicate this is a local inner macro - local_inner: bool, - krate: Option, - - call_site: Option>, - def_site: Option>, -} - -impl HygieneFrames { - fn new(db: &dyn ExpandDatabase, file_id: HirFileId) -> Self { - // Note that this intentionally avoids the `hygiene_frame` query to avoid blowing up memory - // usage. The query is only helpful for nested `HygieneFrame`s as it avoids redundant work. - HygieneFrames(Arc::new(HygieneFrame::new(db, file_id))) - } - - fn root_crate(&self, _db: &dyn ExpandDatabase, node: &SyntaxNode) -> Option { - let mut _token = node.first_token()?.text_range(); - let mut _result = self.0.krate; - let mut _current = self.0.clone(); - - return None; - - // while let Some((mapped, origin)) = - // current.expansion.as_ref().and_then(|it| it.map_ident_up(db, token)) - // { - // result = current.krate; - - // let site = match origin { - // Origin::Def => ¤t.def_site, - // Origin::Call => ¤t.call_site, - // }; - - // let site = match site { - // None => break, - // Some(it) => it, - // }; - - // current = site.clone(); - // token = mapped.value; - // } - - // result - } -} +pub struct HygieneFrame {} #[derive(Debug, Clone, PartialEq, Eq)] -struct HygieneInfo { - file: MacroFile, - /// The start offset of the `macro_rules!` arguments or attribute input. - attr_input_or_mac_def_start: Option>, - - macro_def: TokenExpander, - macro_arg: Arc, - exp_map: Arc, -} +struct HygieneInfo {} impl HygieneInfo { - fn _map_ident_up( - &self, - _db: &dyn ExpandDatabase, - _token: TextRange, - ) -> Option> { - // self.exp_map.token_by_range(token).map(|span| InFile::new(span.anchor, span.range)) + fn _map_ident_up(&self, _: &dyn ExpandDatabase, _: TextRange) -> Option> { None } } -fn make_hygiene_info( - db: &dyn ExpandDatabase, - macro_file: MacroFile, - loc: &MacroCallLoc, -) -> HygieneInfo { - let def = loc.def.ast_id().left().and_then(|id| { - let def_tt = match id.to_node(db) { - ast::Macro::MacroRules(mac) => mac.token_tree()?, - ast::Macro::MacroDef(mac) => mac.body()?, - }; - Some(InFile::new(id.file_id, def_tt)) - }); - let attr_input_or_mac_def = def.or_else(|| match loc.kind { - MacroCallKind::Attr { ast_id, invoc_attr_index, .. } => { - let tt = ast_id - .to_node(db) - .doc_comments_and_attrs() - .nth(invoc_attr_index.ast_index()) - .and_then(Either::left)? - .token_tree()?; - Some(InFile::new(ast_id.file_id, tt)) - } - _ => None, - }); - - let macro_def = db.macro_expander(loc.def); - let (_, exp_map) = db.parse_macro_expansion(macro_file).value; - let macro_arg = db.macro_arg(macro_file.macro_call_id).value.unwrap_or_else(|| { - Arc::new(tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: Vec::new() }) - }); - - HygieneInfo { - file: macro_file, - attr_input_or_mac_def_start: attr_input_or_mac_def - .map(|it| it.map(|tt| tt.syntax().text_range().start())), - macro_arg, - macro_def, - exp_map, - } -} - impl HygieneFrame { - pub(crate) fn new(db: &dyn ExpandDatabase, file_id: HirFileId) -> HygieneFrame { - let (info, krate, local_inner) = match file_id.macro_file() { - None => (None, None, false), - Some(macro_file) => { - let loc = db.lookup_intern_macro_call(macro_file.macro_call_id); - let info = Some((make_hygiene_info(db, macro_file, &loc), loc.kind.file_id())); - match loc.def.kind { - MacroDefKind::Declarative(_) => { - (info, Some(loc.def.krate), loc.def.local_inner) - } - MacroDefKind::BuiltIn(..) => (info, Some(loc.def.krate), false), - MacroDefKind::BuiltInAttr(..) => (info, None, false), - MacroDefKind::BuiltInDerive(..) => (info, None, false), - MacroDefKind::BuiltInEager(..) => (info, None, false), - MacroDefKind::ProcMacro(..) => (info, None, false), - } - } - }; - - let Some((info, calling_file)) = info else { - return HygieneFrame { - expansion: None, - local_inner, - krate, - call_site: None, - def_site: None, - }; - }; - - let def_site = info.attr_input_or_mac_def_start.map(|it| db.hygiene_frame(it.file_id)); - let call_site = Some(db.hygiene_frame(calling_file)); - - HygieneFrame { expansion: Some(info), local_inner, krate, call_site, def_site } + pub(crate) fn new(_: &dyn ExpandDatabase, _: HirFileId) -> HygieneFrame { + HygieneFrame {} } } diff --git a/crates/hir-expand/src/lib.rs b/crates/hir-expand/src/lib.rs index bd5796e000ac..ae07cf4b151b 100644 --- a/crates/hir-expand/src/lib.rs +++ b/crates/hir-expand/src/lib.rs @@ -51,7 +51,7 @@ pub type DeclarativeMacro = ::mbe::DeclarativeMacro; pub mod tt { pub use base_db::span::SpanData; - pub use tt::{DelimiterKind, Spacing, Span}; + pub use tt::{DelimiterKind, Spacing, Span, SpanAnchor}; pub type Delimiter = ::tt::Delimiter; pub type Subtree = ::tt::Subtree; @@ -97,44 +97,6 @@ impl fmt::Display for ExpandError { } } -/// `MacroCallId` identifies a particular macro invocation, like -/// `println!("Hello, {}", world)`. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct SyntaxContextId(base_db::salsa::InternId); -base_db::impl_intern_key!(SyntaxContextId); - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct SyntaxContext { - outer_expn: HirFileId, - outer_transparency: Transparency, - parent: SyntaxContextId, - /// This context, but with all transparent and semi-transparent expansions filtered away. - opaque: SyntaxContextId, - /// This context, but with all transparent expansions filtered away. - opaque_and_semitransparent: SyntaxContextId, - /// Name of the crate to which `$crate` with this context would resolve. - dollar_crate_name: name::Name, -} - -/// A property of a macro expansion that determines how identifiers -/// produced by that expansion are resolved. -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Hash, Debug)] -pub enum Transparency { - /// Identifier produced by a transparent expansion is always resolved at call-site. - /// Call-site spans in procedural macros, hygiene opt-out in `macro` should use this. - Transparent, - /// Identifier produced by a semi-transparent expansion may be resolved - /// either at call-site or at definition-site. - /// If it's a local variable, label or `$crate` then it's resolved at def-site. - /// Otherwise it's resolved at call-site. - /// `macro_rules` macros behave like this, built-in macros currently behave like this too, - /// but that's an implementation detail. - SemiTransparent, - /// Identifier produced by an opaque expansion is always resolved at definition-site. - /// Def-site spans in procedural macros, identifiers from `macro` by default use this. - Opaque, -} - #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct MacroCallLoc { pub def: MacroDefId, diff --git a/crates/hir-expand/src/quote.rs b/crates/hir-expand/src/quote.rs index 9dd4965c1506..44f20cbd92d2 100644 --- a/crates/hir-expand/src/quote.rs +++ b/crates/hir-expand/src/quote.rs @@ -247,8 +247,8 @@ mod tests { assert_eq!(quoted.to_string(), "hello"); let t = format!("{quoted:?}"); expect![[r#" - SUBTREE $$ SpanData { range: 0..0, anchor: SpanAnchor { file_id: FileId(0), ast_id: Idx::>(0) } } SpanData { range: 0..0, anchor: SpanAnchor { file_id: FileId(0), ast_id: Idx::>(0) } } - IDENT hello SpanData { range: 0..0, anchor: SpanAnchor { file_id: FileId(0), ast_id: Idx::>(0) } }"#]].assert_eq(&t); + SUBTREE $$ SpanData { range: 0..0, anchor: SpanAnchor(FileId(0), 0), ctx: SyntaxContextId(0) } SpanData { range: 0..0, anchor: SpanAnchor(FileId(0), 0), ctx: SyntaxContextId(0) } + IDENT hello SpanData { range: 0..0, anchor: SpanAnchor(FileId(0), 0), ctx: SyntaxContextId(0) }"#]].assert_eq(&t); } #[test] diff --git a/crates/mbe/src/benchmark.rs b/crates/mbe/src/benchmark.rs index 4f60e90773ee..2ead71645639 100644 --- a/crates/mbe/src/benchmark.rs +++ b/crates/mbe/src/benchmark.rs @@ -6,19 +6,27 @@ use syntax::{ AstNode, SmolStr, }; use test_utils::{bench, bench_fixture, skip_slow_tests}; -use tt::{Span, SpanData}; +use tt::{Span, SpanAnchor, SyntaxContext}; use crate::{ parser::{MetaVarKind, Op, RepeatKind, Separator}, syntax_node_to_token_tree, DeclarativeMacro, }; +type SpanData = tt::SpanData; + #[derive(PartialEq, Eq, Clone, Copy, Debug)] struct DummyFile; -impl Span for DummyFile { +impl SpanAnchor for DummyFile { const DUMMY: Self = DummyFile; } +#[derive(PartialEq, Eq, Clone, Copy, Debug)] +struct DummyCtx; +impl SyntaxContext for DummyCtx { + const DUMMY: Self = DummyCtx; +} + #[test] fn benchmark_parse_macro_rules() { if skip_slow_tests() { @@ -54,14 +62,14 @@ fn benchmark_expand_macro_rules() { assert_eq!(hash, 69413); } -fn macro_rules_fixtures() -> FxHashMap>> { +fn macro_rules_fixtures() -> FxHashMap> { macro_rules_fixtures_tt() .into_iter() .map(|(id, tt)| (id, DeclarativeMacro::parse_macro_rules(&tt, true))) .collect() } -fn macro_rules_fixtures_tt() -> FxHashMap>> { +fn macro_rules_fixtures_tt() -> FxHashMap> { let fixture = bench_fixture::numerous_macro_rules(); let source_file = ast::SourceFile::parse(&fixture).ok().unwrap(); @@ -84,8 +92,8 @@ fn macro_rules_fixtures_tt() -> FxHashMap>>, -) -> Vec<(String, tt::Subtree>)> { + rules: &FxHashMap>, +) -> Vec<(String, tt::Subtree)> { let mut seed = 123456789; let mut res = Vec::new(); @@ -130,11 +138,7 @@ fn invocation_fixtures( } return res; - fn collect_from_op( - op: &Op>, - parent: &mut tt::Subtree>, - seed: &mut usize, - ) { + fn collect_from_op(op: &Op, parent: &mut tt::Subtree, seed: &mut usize) { return match op { Op::Var { kind, .. } => match kind.as_ref() { Some(MetaVarKind::Ident) => parent.token_trees.push(make_ident("foo")), @@ -220,20 +224,20 @@ fn invocation_fixtures( *seed = usize::wrapping_add(usize::wrapping_mul(*seed, a), c); *seed } - fn make_ident(ident: &str) -> tt::TokenTree> { + fn make_ident(ident: &str) -> tt::TokenTree { tt::Leaf::Ident(tt::Ident { span: SpanData::DUMMY, text: SmolStr::new(ident) }).into() } - fn make_punct(char: char) -> tt::TokenTree> { + fn make_punct(char: char) -> tt::TokenTree { tt::Leaf::Punct(tt::Punct { span: SpanData::DUMMY, char, spacing: tt::Spacing::Alone }) .into() } - fn make_literal(lit: &str) -> tt::TokenTree> { + fn make_literal(lit: &str) -> tt::TokenTree { tt::Leaf::Literal(tt::Literal { span: SpanData::DUMMY, text: SmolStr::new(lit) }).into() } fn make_subtree( kind: tt::DelimiterKind, - token_trees: Option>>>, - ) -> tt::TokenTree> { + token_trees: Option>>, + ) -> tt::TokenTree { tt::Subtree { delimiter: tt::Delimiter { open: SpanData::DUMMY, close: SpanData::DUMMY, kind }, token_trees: token_trees.unwrap_or_default(), diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index c8c2e5dcd55a..ab272862cdb5 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -9,7 +9,7 @@ use syntax::{ }; use tt::{ buffer::{Cursor, TokenBuffer}, - Span, SpanData, + Span, SpanData, SyntaxContext, }; use crate::{to_parser_input::to_parser_input, tt_iter::TtIter, TokenMap}; @@ -23,33 +23,37 @@ mod tests; /// to relative spans, relative to the passed anchor. /// `map` is used to resolve the converted spans accordingly. /// TODO: Flesh out the doc comment more thoroughly -pub fn syntax_node_to_token_tree( +pub fn syntax_node_to_token_tree( node: &SyntaxNode, - anchor: SpanAnchor, + anchor: Anchor, anchor_offset: TextSize, - map: &TokenMap>, -) -> tt::Subtree> + map: &TokenMap>, +) -> tt::Subtree> where - SpanData: Span, + SpanData: Span, + Anchor: Copy, + Ctx: SyntaxContext, { assert!(anchor_offset <= node.text_range().start()); - let mut c = Converter::new(node, anchor_offset, anchor, vec![], map); - convert_tokens(&mut c) + let mut c = Converter::new(node, anchor_offset, vec![], map); + convert_tokens(&mut c, anchor) } -pub fn syntax_node_to_token_tree_censored( +pub fn syntax_node_to_token_tree_censored( node: &SyntaxNode, - anchor: SpanAnchor, + anchor: Anchor, anchor_offset: TextSize, - map: &TokenMap>, + map: &TokenMap>, censored: Vec, -) -> tt::Subtree> +) -> tt::Subtree> where - SpanData: Span, + SpanData: Span, + Anchor: Copy, + Ctx: SyntaxContext, { assert!(anchor_offset <= node.text_range().start()); - let mut c = Converter::new(node, anchor_offset, anchor, censored, map); - convert_tokens(&mut c) + let mut c = Converter::new(node, anchor_offset, censored, map); + convert_tokens(&mut c, anchor) } // The following items are what `rustc` macro can be parsed into : @@ -64,12 +68,14 @@ where // * AssocItems(SmallVec<[ast::AssocItem; 1]>) // * ForeignItems(SmallVec<[ast::ForeignItem; 1]> -pub fn token_tree_to_syntax_node( - tt: &tt::Subtree>, +pub fn token_tree_to_syntax_node( + tt: &tt::Subtree>, entry_point: parser::TopEntryPoint, -) -> (Parse, TokenMap>) +) -> (Parse, TokenMap>) where - SpanData: Span, + SpanData: Span, + Anchor: Copy, + Ctx: SyntaxContext, { let buffer = match tt { tt::Subtree { @@ -97,36 +103,42 @@ where tree_sink.finish() } -pub fn map_from_syntax_node( +pub fn map_from_syntax_node( node: &SyntaxNode, - anchor: SpanAnchor, + anchor: Anchor, anchor_offset: TextSize, -) -> TokenMap> +) -> TokenMap> where - SpanAnchor: Copy, - SpanData: Span, + Anchor: Copy, + SpanData: Span, + Ctx: SyntaxContext, { let mut map = TokenMap::default(); node.descendants_with_tokens().filter_map(NodeOrToken::into_token).for_each(|t| { - map.insert(t.text_range(), SpanData { range: t.text_range() - anchor_offset, anchor }); + map.insert( + t.text_range(), + SpanData { range: t.text_range() - anchor_offset, anchor, ctx: Ctx::DUMMY }, + ); }); map } /// Convert a string to a `TokenTree` -pub fn parse_to_token_tree( +pub fn parse_to_token_tree( text: &str, - file_id: SpanAnchor, -) -> Option>> + anchor: Anchor, +) -> Option>> where - SpanData: Span, + SpanData: Span, + Anchor: Copy, + Ctx: SyntaxContext, { let lexed = parser::LexedStr::new(text); if lexed.errors().next().is_some() { return None; } - let mut conv = RawConverter { lexed, pos: 0, _offset: TextSize::default(), file_id }; - Some(convert_tokens(&mut conv)) + let mut conv = RawConverter { lexed, pos: 0, _offset: TextSize::default() }; + Some(convert_tokens(&mut conv, anchor)) } /// Split token tree with separate expr: $($e:expr)SEP* @@ -166,134 +178,141 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec>( +fn convert_tokens( conv: &mut C, -) -> tt::Subtree> + anchor: Anchor, +) -> tt::Subtree> where - SpanData: Span, - SpanAnchor: Copy, + C: TokenConverter, + Ctx: SyntaxContext, + SpanData: Span, + Anchor: Copy, { - let entry = tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] }; + let entry = tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: vec![] }; let mut stack = NonEmptyVec::new(entry); - let anchor = conv.anchor(); - loop { - let subtree = stack.last_mut(); - let result = &mut subtree.token_trees; - let Some((token, rel_range, abs_range)) = conv.bump() else { break }; + while let Some((token, rel_range, abs_range)) = conv.bump() { + let tt::Subtree { delimiter, token_trees: result } = stack.last_mut(); + let mk_dummy_span = || SpanData { range: rel_range, anchor, ctx: Ctx::DUMMY }; let kind = token.kind(conv); - if kind == COMMENT { - if let Some(tokens) = conv.convert_doc_comment( - &token, - conv.span_for(abs_range).unwrap_or(SpanData { range: rel_range, anchor }), - ) { - result.extend(tokens); + + let tt = match kind { + // Desugar doc comments into doc attributes + COMMENT => { + let span = conv.span_for(abs_range).unwrap_or_else(mk_dummy_span); + if let Some(tokens) = conv.convert_doc_comment(&token, span) { + result.extend(tokens); + } + continue; } - continue; - } - let tt = if kind.is_punct() && kind != UNDERSCORE { - let expected = match subtree.delimiter.kind { - tt::DelimiterKind::Parenthesis => Some(T![')']), - tt::DelimiterKind::Brace => Some(T!['}']), - tt::DelimiterKind::Bracket => Some(T![']']), - tt::DelimiterKind::Invisible => None, - }; + _ if kind.is_punct() && kind != UNDERSCORE => { + let expected = match delimiter.kind { + tt::DelimiterKind::Parenthesis => Some(T![')']), + tt::DelimiterKind::Brace => Some(T!['}']), + tt::DelimiterKind::Bracket => Some(T![']']), + tt::DelimiterKind::Invisible => None, + }; - if let Some(expected) = expected { - if kind == expected { + // Current token is a closing delimiter that we expect, fix up the closing span + // and end the subtree here + if matches!(expected, Some(expected) if expected == kind) { if let Some(mut subtree) = stack.pop() { - subtree.delimiter.close = conv - .span_for(abs_range) - .unwrap_or(SpanData { range: rel_range, anchor }); + subtree.delimiter.close = + conv.span_for(abs_range).unwrap_or_else(mk_dummy_span); stack.last_mut().token_trees.push(subtree.into()); } continue; } - } - - let delim = match kind { - T!['('] => Some(tt::DelimiterKind::Parenthesis), - T!['{'] => Some(tt::DelimiterKind::Brace), - T!['['] => Some(tt::DelimiterKind::Bracket), - _ => None, - }; - if let Some(kind) = delim { - let subtree = tt::Subtree { - delimiter: tt::Delimiter { - // FIXME: Open and close spans - open: conv - .span_for(abs_range) - .unwrap_or(SpanData { range: rel_range, anchor }), - close: Span::DUMMY, - kind, - }, - token_trees: vec![], + let delim = match kind { + T!['('] => Some(tt::DelimiterKind::Parenthesis), + T!['{'] => Some(tt::DelimiterKind::Brace), + T!['['] => Some(tt::DelimiterKind::Bracket), + _ => None, }; - stack.push(subtree); - continue; - } - let spacing = match conv.peek().map(|next| next.kind(conv)) { - Some(kind) if is_single_token_op(kind) => tt::Spacing::Joint, - _ => tt::Spacing::Alone, - }; - let char = match token.to_char(conv) { - Some(c) => c, - None => { - panic!("Token from lexer must be single char: token = {token:#?}"); + // Start a new subtree + if let Some(kind) = delim { + stack.push(tt::Subtree { + delimiter: tt::Delimiter { + open: conv.span_for(abs_range).unwrap_or_else(mk_dummy_span), + // will be overwritten on subtree close above + close: mk_dummy_span(), + kind, + }, + token_trees: vec![], + }); + continue; } - }; - tt::Leaf::from(tt::Punct { - char, - spacing, - span: conv.span_for(abs_range).unwrap_or(SpanData { range: rel_range, anchor }), - }) - .into() - } else { - macro_rules! make_leaf { - ($i:ident) => { - tt::$i { - span: conv - .span_for(abs_range) - .unwrap_or(SpanData { range: rel_range, anchor }), - text: token.to_text(conv), - } - .into() + + let spacing = match conv.peek().map(|next| next.kind(conv)) { + Some(kind) if is_single_token_op(kind) => tt::Spacing::Joint, + _ => tt::Spacing::Alone, }; + let Some(char) = token.to_char(conv) else { + panic!("Token from lexer must be single char: token = {token:#?}") + }; + tt::Leaf::from(tt::Punct { + char, + spacing, + span: conv.span_for(abs_range).unwrap_or_else(mk_dummy_span), + }) + .into() } - let leaf: tt::Leaf<_> = match kind { - T![true] | T![false] => make_leaf!(Ident), - IDENT => make_leaf!(Ident), - UNDERSCORE => make_leaf!(Ident), - k if k.is_keyword() => make_leaf!(Ident), - k if k.is_literal() => make_leaf!(Literal), - // FIXME: Check whether span splitting works as intended - LIFETIME_IDENT => { - let char_unit = TextSize::of('\''); - let r = TextRange::at(rel_range.start(), char_unit); - let apostrophe = tt::Leaf::from(tt::Punct { - char: '\'', - spacing: tt::Spacing::Joint, - span: conv.span_for(abs_range).unwrap_or(SpanData { range: r, anchor }), - }); - result.push(apostrophe.into()); - - let r = - TextRange::at(rel_range.start() + char_unit, rel_range.len() - char_unit); - let ident = tt::Leaf::from(tt::Ident { - text: SmolStr::new(&token.to_text(conv)[1..]), - span: conv.span_for(abs_range).unwrap_or(SpanData { range: r, anchor }), - }); - result.push(ident.into()); - continue; + _ => { + macro_rules! make_leaf { + ($i:ident) => { + tt::$i { + span: conv.span_for(abs_range).unwrap_or_else(mk_dummy_span), + text: token.to_text(conv), + } + .into() + }; } - _ => continue, - }; + let leaf: tt::Leaf<_> = match kind { + T![true] | T![false] => make_leaf!(Ident), + IDENT => make_leaf!(Ident), + UNDERSCORE => make_leaf!(Ident), + k if k.is_keyword() => make_leaf!(Ident), + k if k.is_literal() => make_leaf!(Literal), + // FIXME: Check whether span splitting works as intended + LIFETIME_IDENT => { + let char_unit = TextSize::of('\''); + let r = TextRange::at(rel_range.start(), char_unit); + let apostrophe = tt::Leaf::from(tt::Punct { + char: '\'', + spacing: tt::Spacing::Joint, + span: conv.span_for(abs_range).unwrap_or(SpanData { + range: r, + anchor, + ctx: Ctx::DUMMY, + }), + }); + result.push(apostrophe.into()); + + let r = TextRange::at( + rel_range.start() + char_unit, + rel_range.len() - char_unit, + ); + let ident = tt::Leaf::from(tt::Ident { + text: SmolStr::new(&token.to_text(conv)[1..]), + span: conv.span_for(abs_range).unwrap_or(SpanData { + range: r, + anchor, + ctx: Ctx::DUMMY, + }), + }); + result.push(ident.into()); + continue; + } + _ => continue, + }; - leaf.into() + leaf.into() + } }; + result.push(tt); } @@ -417,11 +436,10 @@ fn convert_doc_comment( } /// A raw token (straight from lexer) converter -struct RawConverter<'a, SpanAnchor> { +struct RawConverter<'a> { lexed: parser::LexedStr<'a>, pos: usize, _offset: TextSize, - file_id: SpanAnchor, } trait SrcToken: std::fmt::Debug { @@ -432,48 +450,47 @@ trait SrcToken: std::fmt::Debug { fn to_text(&self, ctx: &Ctx) -> SmolStr; } -trait TokenConverter: Sized { +trait TokenConverter: Sized { type Token: SrcToken; fn convert_doc_comment( &self, token: &Self::Token, - span: SpanData, - ) -> Option>>>; + span: SpanData, + ) -> Option>>>; fn bump(&mut self) -> Option<(Self::Token, TextRange, TextRange)>; fn peek(&self) -> Option; - fn anchor(&self) -> SpanAnchor; - fn span_for(&self, range: TextRange) -> Option>; + fn span_for(&self, range: TextRange) -> Option>; } -impl SrcToken> for usize { - fn kind(&self, ctx: &RawConverter<'_, SpanAnchor>) -> SyntaxKind { +impl SrcToken> for usize { + fn kind(&self, ctx: &RawConverter<'_>) -> SyntaxKind { ctx.lexed.kind(*self) } - fn to_char(&self, ctx: &RawConverter<'_, SpanAnchor>) -> Option { + fn to_char(&self, ctx: &RawConverter<'_>) -> Option { ctx.lexed.text(*self).chars().next() } - fn to_text(&self, ctx: &RawConverter<'_, SpanAnchor>) -> SmolStr { + fn to_text(&self, ctx: &RawConverter<'_>) -> SmolStr { ctx.lexed.text(*self).into() } } -impl TokenConverter for RawConverter<'_, SpanAnchor> +impl TokenConverter for RawConverter<'_> where - SpanData: Span, + SpanData: Span, { type Token = usize; fn convert_doc_comment( &self, &token: &usize, - span: SpanData, - ) -> Option>>> { + span: SpanData, + ) -> Option>>> { let text = self.lexed.text(token); convert_doc_comment(&doc_comment(text), span) } @@ -497,34 +514,29 @@ where Some(self.pos) } - fn anchor(&self) -> SpanAnchor { - self.file_id - } - fn span_for(&self, _: TextRange) -> Option> { + fn span_for(&self, _: TextRange) -> Option> { None } } -struct Converter<'a, SpanAnchor> { +struct Converter<'a, Anchor, Ctx> { current: Option, preorder: PreorderWithTokens, range: TextRange, punct_offset: Option<(SyntaxToken, TextSize)>, /// Used to make the emitted text ranges in the spans relative to the span anchor. offset: TextSize, - file_id: SpanAnchor, - map: &'a TokenMap>, + map: &'a TokenMap>, censored: Vec, } -impl<'a, SpanAnchor> Converter<'a, SpanAnchor> { +impl<'a, Anchor, Ctx> Converter<'a, Anchor, Ctx> { fn new( node: &SyntaxNode, anchor_offset: TextSize, - file_id: SpanAnchor, censored: Vec, - map: &'a TokenMap>, - ) -> Converter<'a, SpanAnchor> { + map: &'a TokenMap>, + ) -> Self { let range = node.text_range(); let mut preorder = node.preorder_with_tokens(); let first = Self::next_token(&mut preorder, &censored); @@ -534,7 +546,6 @@ impl<'a, SpanAnchor> Converter<'a, SpanAnchor> { range, punct_offset: None, offset: anchor_offset, - file_id, censored, map, } @@ -569,36 +580,36 @@ impl SynToken { } } -impl SrcToken> for SynToken { - fn kind(&self, ctx: &Converter<'_, SpanAnchor>) -> SyntaxKind { +impl SrcToken> for SynToken { + fn kind(&self, ctx: &Converter<'_, Anchor, Ctx>) -> SyntaxKind { match self { SynToken::Ordinary(token) => token.kind(), SynToken::Punct(..) => SyntaxKind::from_char(self.to_char(ctx).unwrap()).unwrap(), } } - fn to_char(&self, _ctx: &Converter<'_, SpanAnchor>) -> Option { + fn to_char(&self, _ctx: &Converter<'_, Anchor, Ctx>) -> Option { match self { SynToken::Ordinary(_) => None, SynToken::Punct(it, i) => it.text().chars().nth(*i), } } - fn to_text(&self, _ctx: &Converter<'_, SpanAnchor>) -> SmolStr { + fn to_text(&self, _ctx: &Converter<'_, Anchor, Ctx>) -> SmolStr { match self { SynToken::Ordinary(token) | SynToken::Punct(token, _) => token.text().into(), } } } -impl TokenConverter for Converter<'_, SpanAnchor> +impl TokenConverter for Converter<'_, Anchor, Ctx> where - SpanData: Span, + SpanData: Span, { type Token = SynToken; fn convert_doc_comment( &self, token: &Self::Token, - span: SpanData, - ) -> Option>>> { + span: SpanData, + ) -> Option>>> { convert_doc_comment(token.token(), span) } @@ -657,27 +668,24 @@ where Some(token) } - fn anchor(&self) -> SpanAnchor { - self.file_id - } - fn span_for(&self, range: TextRange) -> Option> { + fn span_for(&self, range: TextRange) -> Option> { self.map.span_for_range(range) } } -struct TtTreeSink<'a, SpanAnchor> { +struct TtTreeSink<'a, Anchor, Ctx> { buf: String, - cursor: Cursor<'a, SpanData>, + cursor: Cursor<'a, SpanData>, text_pos: TextSize, inner: SyntaxTreeBuilder, - token_map: TokenMap>, + token_map: TokenMap>, } -impl<'a, SpanAnchor> TtTreeSink<'a, SpanAnchor> +impl<'a, Anchor, Ctx> TtTreeSink<'a, Anchor, Ctx> where - SpanData: Span, + SpanData: Span, { - fn new(cursor: Cursor<'a, SpanData>) -> Self { + fn new(cursor: Cursor<'a, SpanData>) -> Self { TtTreeSink { buf: String::new(), cursor, @@ -687,7 +695,7 @@ where } } - fn finish(mut self) -> (Parse, TokenMap>) { + fn finish(mut self) -> (Parse, TokenMap>) { self.token_map.shrink_to_fit(); (self.inner.finish(), self.token_map) } @@ -705,9 +713,9 @@ fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> Option<&'static str> { Some(&texts[idx..texts.len() - (1 - idx)]) } -impl TtTreeSink<'_, SpanAnchor> +impl TtTreeSink<'_, Anchor, Ctx> where - SpanData: Span, + SpanData: Span, { /// Parses a float literal as if it was a one to two name ref nodes with a dot inbetween. /// This occurs when a float literal is used as a field access. diff --git a/crates/mbe/src/syntax_bridge/tests.rs b/crates/mbe/src/syntax_bridge/tests.rs index 32dfb4d7e0b3..0275e5397c9b 100644 --- a/crates/mbe/src/syntax_bridge/tests.rs +++ b/crates/mbe/src/syntax_bridge/tests.rs @@ -4,20 +4,26 @@ use syntax::{ast, AstNode}; use test_utils::extract_annotations; use tt::{ buffer::{TokenBuffer, TokenTreeRef}, - Leaf, Punct, Spacing, Span, + Leaf, Punct, Spacing, SpanAnchor, SyntaxContext, }; -use crate::syntax_bridge::SpanData; - use super::syntax_node_to_token_tree; fn check_punct_spacing(fixture: &str) { + type SpanData = tt::SpanData; + #[derive(PartialEq, Eq, Clone, Copy, Debug)] struct DummyFile; - impl Span for DummyFile { + impl SpanAnchor for DummyFile { const DUMMY: Self = DummyFile; } + #[derive(PartialEq, Eq, Clone, Copy, Debug)] + struct DummyCtx; + impl SyntaxContext for DummyCtx { + const DUMMY: Self = DummyCtx; + } + let source_file = ast::SourceFile::parse(fixture).ok().unwrap(); let subtree = syntax_node_to_token_tree(source_file.syntax(), DummyFile, 0.into(), &Default::default()); diff --git a/crates/proc-macro-api/src/msg.rs b/crates/proc-macro-api/src/msg.rs index f0719777ab4d..6a7329e322f3 100644 --- a/crates/proc-macro-api/src/msg.rs +++ b/crates/proc-macro-api/src/msg.rs @@ -120,11 +120,13 @@ fn write_json(out: &mut impl Write, msg: &str) -> io::Result<()> { Ok(()) } +/* + #[cfg(test)] mod tests { use tt::{ - Delimiter, DelimiterKind, Ident, Leaf, Literal, Punct, Spacing, Span, Subtree, TokenId, - TokenTree, + Delimiter, DelimiterKind, Ident, Leaf, Literal, Punct, Spacing, SpanAnchor, Subtree, + TokenId, TokenTree, }; use super::*; @@ -176,3 +178,4 @@ mod tests { assert_eq!(tt, back.macro_body.to_subtree(CURRENT_API_VERSION)); } } +*/ diff --git a/crates/proc-macro-api/src/msg/flat.rs b/crates/proc-macro-api/src/msg/flat.rs index bfb3213a25d9..22d9f3952cc2 100644 --- a/crates/proc-macro-api/src/msg/flat.rs +++ b/crates/proc-macro-api/src/msg/flat.rs @@ -39,7 +39,7 @@ use std::collections::{HashMap, VecDeque}; use serde::{Deserialize, Serialize}; use text_size::TextRange; -use tt::Span; +use tt::{Span, SyntaxContext}; use crate::msg::{ENCODE_CLOSE_SPAN_VERSION, VARIABLE_SIZED_SPANS}; @@ -47,25 +47,30 @@ pub trait SerializableSpan: Span { fn into_u32(self) -> [u32; L]; fn from_u32(input: [u32; L]) -> Self; } -impl SerializableSpan<1> for tt::TokenId { - fn into_u32(self) -> [u32; 1] { - [self.0] - } - fn from_u32([input]: [u32; 1]) -> Self { - tt::TokenId(input) - } -} - -impl SerializableSpan<3> for tt::SpanData +// impl SerializableSpan<1> for tt::TokenId { +// fn into_u32(self) -> [u32; 1] { +// [self.0] +// } +// fn from_u32([input]: [u32; 1]) -> Self { +// tt::TokenId(input) +// } +// } + +impl SerializableSpan<3> for tt::SpanData where - FileId: From + Into, + Anchor: From + Into, Self: Span, + Ctx: SyntaxContext, { fn into_u32(self) -> [u32; 3] { [self.anchor.into(), self.range.start().into(), self.range.end().into()] } fn from_u32([file_id, start, end]: [u32; 3]) -> Self { - tt::SpanData { anchor: file_id.into(), range: TextRange::new(start.into(), end.into()) } + tt::SpanData { + anchor: file_id.into(), + range: TextRange::new(start.into(), end.into()), + ctx: Ctx::DUMMY, + } } } diff --git a/crates/rust-analyzer/src/cargo_target_spec.rs b/crates/rust-analyzer/src/cargo_target_spec.rs index b6dcc26de621..4742dc1dfa79 100644 --- a/crates/rust-analyzer/src/cargo_target_spec.rs +++ b/crates/rust-analyzer/src/cargo_target_spec.rs @@ -210,7 +210,7 @@ mod tests { use cfg::CfgExpr; use hir::HirFileId; - use ide_db::base_db::span::{SpanAnchor, ROOT_ERASED_FILE_AST_ID}; + use ide_db::base_db::span::{SpanAnchor, SyntaxContextId, ROOT_ERASED_FILE_AST_ID}; use mbe::syntax_node_to_token_tree; use syntax::{ ast::{self, AstNode}, @@ -221,7 +221,7 @@ mod tests { let cfg_expr = { let source_file = ast::SourceFile::parse(cfg).ok().unwrap(); let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - let tt = syntax_node_to_token_tree( + let tt = syntax_node_to_token_tree::<_, SyntaxContextId>( tt.syntax(), SpanAnchor { file_id: HirFileId::from(0), ast_id: ROOT_ERASED_FILE_AST_ID }, TextSize::new(0), diff --git a/crates/tt/src/lib.rs b/crates/tt/src/lib.rs index 89cb12d2c267..a384af2a9a33 100644 --- a/crates/tt/src/lib.rs +++ b/crates/tt/src/lib.rs @@ -32,27 +32,37 @@ impl TokenId { Self::UNSPECIFIED } } -impl Span for TokenId { - const DUMMY: Self = TokenId(!0); -} #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] -pub struct SpanData { +pub struct SpanData { /// The text range of this span, relative to the anchor. + /// We need the anchor for incrementality, as storing absolute ranges will require + /// recomputation on every change in a file at all times. pub range: TextRange, pub anchor: Anchor, + /// The syntax context of the span. + pub ctx: Ctx, } -impl Span for SpanData { - const DUMMY: Self = - SpanData { range: TextRange::empty(TextSize::new(0)), anchor: Anchor::DUMMY }; +impl Span for SpanData { + const DUMMY: Self = SpanData { + range: TextRange::empty(TextSize::new(0)), + anchor: Anchor::DUMMY, + ctx: Ctx::DUMMY, + }; +} + +pub trait SpanAnchor: std::fmt::Debug + Copy + Sized + Eq { + const DUMMY: Self; } pub trait Span: std::fmt::Debug + Copy + Sized + Eq { const DUMMY: Self; } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct SyntaxContext(pub u32); + +pub trait SyntaxContext: std::fmt::Debug + Copy + Sized + Eq { + const DUMMY: Self; +} #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum TokenTree { From 05f375eae29927565da546c48fc64b843a90197e Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Fri, 17 Nov 2023 19:07:31 +0100 Subject: [PATCH 05/24] hygiene 2.0 --- Cargo.toml | 2 +- crates/base-db/src/span.rs | 11 +- crates/hir-def/src/data.rs | 3 +- crates/hir-def/src/expander.rs | 17 +- crates/hir-def/src/find_path.rs | 4 +- crates/hir-def/src/item_tree.rs | 10 +- crates/hir-def/src/item_tree/lower.rs | 46 ++-- crates/hir-def/src/item_tree/pretty.rs | 2 +- crates/hir-def/src/lib.rs | 23 +- crates/hir-def/src/lower.rs | 22 +- .../hir-def/src/macro_expansion_tests/mbe.rs | 13 +- .../macro_expansion_tests/mbe/metavar_expr.rs | 4 +- .../macro_expansion_tests/mbe/regression.rs | 4 +- .../mbe/tt_conversion.rs | 6 +- .../hir-def/src/macro_expansion_tests/mod.rs | 5 +- crates/hir-def/src/nameres/collector.rs | 69 +++--- crates/hir-def/src/path/lower.rs | 75 +++--- crates/hir-def/src/test_db.rs | 3 +- crates/hir-def/src/visibility.rs | 8 +- crates/hir-expand/src/attrs.rs | 38 ++- crates/hir-expand/src/db.rs | 179 +++++++++----- crates/hir-expand/src/eager.rs | 26 +- crates/hir-expand/src/hygiene.rs | 198 ++++++++++----- crates/hir-expand/src/lib.rs | 9 +- crates/hir-expand/src/mod_path.rs | 93 +++++-- crates/hir-expand/src/name.rs | 1 + crates/hir-ty/src/display.rs | 6 +- crates/hir-ty/src/test_db.rs | 3 +- crates/hir/src/attrs.rs | 4 +- crates/hir/src/db.rs | 2 +- crates/hir/src/lib.rs | 5 +- crates/hir/src/semantics.rs | 4 +- crates/hir/src/source_analyzer.rs | 4 +- crates/ide-db/src/apply_change.rs | 1 - crates/ide-db/src/lib.rs | 1 - crates/mbe/src/benchmark.rs | 4 +- crates/mbe/src/expander.rs | 9 +- crates/mbe/src/expander/matcher.rs | 4 +- crates/mbe/src/expander/transcriber.rs | 233 +++++++++++------- crates/mbe/src/lib.rs | 13 +- crates/mbe/src/syntax_bridge.rs | 6 +- crates/mbe/src/token_map.rs | 18 +- crates/tt/src/lib.rs | 12 +- 43 files changed, 750 insertions(+), 450 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 73bb9c84d2c9..bf98d7ecf9c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ authors = ["rust-analyzer team"] [profile.dev] # Disabling debug info speeds up builds a bunch, # and we don't rely on it for debugging that much. -debug = 0 +debug = 1 [profile.dev.package] # These speed up local tests. diff --git a/crates/base-db/src/span.rs b/crates/base-db/src/span.rs index acc1e5243f73..f430a36ddfb4 100644 --- a/crates/base-db/src/span.rs +++ b/crates/base-db/src/span.rs @@ -17,9 +17,18 @@ pub struct SyntaxContextId(InternId); crate::impl_intern_key!(SyntaxContextId); impl SyntaxContext for SyntaxContextId { + const DUMMY: Self = Self::ROOT; + // veykril(HACK): salsa doesn't allow us fetching the id of the current input to be allocated so + // we need a special value that behaves as the current context. +} +// inherent trait impls please tyvm +impl SyntaxContextId { + // FIXME: This is very much UB, salsa exposes no way to create an InternId in a const context + // currently (which kind of makes sense but we need it here!) + pub const ROOT: Self = SyntaxContextId(unsafe { core::mem::transmute(1) }); // FIXME: This is very much UB, salsa exposes no way to create an InternId in a const context // currently (which kind of makes sense but we need it here!) - const DUMMY: Self = SyntaxContextId(unsafe { core::mem::transmute(1) }); + pub const SELF_REF: Self = SyntaxContextId(unsafe { core::mem::transmute(!0u32) }); } #[derive(Copy, Clone, PartialEq, Eq, Hash)] diff --git a/crates/hir-def/src/data.rs b/crates/hir-def/src/data.rs index 4083d497917c..9986870d9dce 100644 --- a/crates/hir-def/src/data.rs +++ b/crates/hir-def/src/data.rs @@ -706,7 +706,7 @@ impl<'a> AssocItemCollector<'a> { } AssocItem::MacroCall(call) => { let file_id = self.expander.current_file_id(); - let MacroCall { ast_id, expand_to, ref path } = item_tree[call]; + let MacroCall { ast_id, expand_to, call_site, ref path } = item_tree[call]; let module = self.expander.module.local_id; let resolver = |path| { @@ -725,6 +725,7 @@ impl<'a> AssocItemCollector<'a> { match macro_call_as_call_id( self.db.upcast(), &AstIdWithPath::new(file_id, ast_id, Clone::clone(path)), + call_site, expand_to, self.expander.module.krate(), resolver, diff --git a/crates/hir-def/src/expander.rs b/crates/hir-def/src/expander.rs index b7cffb57625e..793c8ddeb504 100644 --- a/crates/hir-def/src/expander.rs +++ b/crates/hir-def/src/expander.rs @@ -7,11 +7,12 @@ use base_db::{ use cfg::CfgOptions; use drop_bomb::DropBomb; use hir_expand::{ - attrs::RawAttrs, hygiene::Hygiene, mod_path::ModPath, ExpandError, ExpandResult, HirFileId, - InFile, MacroCallId, UnresolvedMacro, + attrs::RawAttrs, mod_path::ModPath, ExpandError, ExpandResult, HirFileId, InFile, MacroCallId, + SpanMap, UnresolvedMacro, }; use limit::Limit; use syntax::{ast, Parse, SyntaxNode}; +use triomphe::Arc; use crate::{ attr::Attrs, db::DefDatabase, lower::LowerCtx, macro_id_to_def_id, path::Path, AsMacroCall, @@ -21,7 +22,7 @@ use crate::{ #[derive(Debug)] pub struct Expander { cfg_options: CfgOptions, - hygiene: Hygiene, + hygiene: Arc, krate: CrateId, pub(crate) current_file_id: HirFileId, pub(crate) module: ModuleId, @@ -44,7 +45,7 @@ impl Expander { recursion_depth: 0, recursion_limit, cfg_options: db.crate_graph()[module.krate].cfg_options.clone(), - hygiene: Hygiene::new(db.upcast(), current_file_id), + hygiene: db.span_map(current_file_id), krate: module.krate, } } @@ -98,7 +99,7 @@ impl Expander { } pub fn exit(&mut self, db: &dyn DefDatabase, mut mark: Mark) { - self.hygiene = Hygiene::new(db.upcast(), mark.file_id); + self.hygiene = db.span_map(mark.file_id); self.current_file_id = mark.file_id; if self.recursion_depth == u32::MAX { // Recursion limit has been reached somewhere in the macro expansion tree. Reset the @@ -113,7 +114,7 @@ impl Expander { } pub fn ctx<'a>(&self, db: &'a dyn DefDatabase) -> LowerCtx<'a> { - LowerCtx::new(db, &self.hygiene, self.current_file_id) + LowerCtx::new(db, self.hygiene.clone(), self.current_file_id) } pub(crate) fn to_source(&self, value: T) -> InFile { @@ -143,7 +144,7 @@ impl Expander { } pub(crate) fn parse_path(&mut self, db: &dyn DefDatabase, path: ast::Path) -> Option { - let ctx = LowerCtx::new(db, &self.hygiene, self.current_file_id); + let ctx = LowerCtx::new(db, self.hygiene.clone(), self.current_file_id); Path::from_src(path, &ctx) } @@ -187,7 +188,7 @@ impl Expander { let parse = value.cast::()?; self.recursion_depth += 1; - self.hygiene = Hygiene::new(db.upcast(), file_id); + self.hygiene = db.span_map(file_id); let old_file_id = std::mem::replace(&mut self.current_file_id, file_id); let mark = Mark { file_id: old_file_id, diff --git a/crates/hir-def/src/find_path.rs b/crates/hir-def/src/find_path.rs index 1ebd1ba0e66d..5051884714ba 100644 --- a/crates/hir-def/src/find_path.rs +++ b/crates/hir-def/src/find_path.rs @@ -586,7 +586,7 @@ fn find_local_import_locations( #[cfg(test)] mod tests { use base_db::fixture::WithFixture; - use hir_expand::hygiene::Hygiene; + use hir_expand::SpanMap; use syntax::ast::AstNode; use crate::test_db::TestDB; @@ -608,7 +608,7 @@ mod tests { let parsed_path_file = syntax::SourceFile::parse(&format!("use {path};")); let ast_path = parsed_path_file.syntax_node().descendants().find_map(syntax::ast::Path::cast).unwrap(); - let mod_path = ModPath::from_src(&db, ast_path, &Hygiene::new_unhygienic()).unwrap(); + let mod_path = ModPath::from_src(&db, ast_path, &SpanMap::default()).unwrap(); let def_map = module.def_map(&db); let resolved = def_map diff --git a/crates/hir-def/src/item_tree.rs b/crates/hir-def/src/item_tree.rs index 3bea91ee61c3..901e14a2117b 100644 --- a/crates/hir-def/src/item_tree.rs +++ b/crates/hir-def/src/item_tree.rs @@ -44,14 +44,13 @@ use std::{ use ast::{AstNode, HasName, StructKind}; use base_db::{ - span::{SpanAnchor, ROOT_ERASED_FILE_AST_ID}, + span::{SpanAnchor, SyntaxContextId, ROOT_ERASED_FILE_AST_ID}, CrateId, }; use either::Either; use hir_expand::{ ast_id_map::{AstIdNode, FileAstId}, attrs::RawAttrs, - hygiene::Hygiene, name::{name, AsName, Name}, ExpandTo, HirFileId, InFile, }; @@ -122,7 +121,7 @@ impl ItemTree { let mut item_tree = match_ast! { match syntax { ast::SourceFile(file) => { - top_attrs = Some(RawAttrs::new(db.upcast(), SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, &file, ctx.hygiene())); + top_attrs = Some(RawAttrs::new(db.upcast(), SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, &file, ctx.span_map())); ctx.lower_module_items(&file) }, ast::MacroItems(items) => { @@ -750,6 +749,7 @@ pub struct MacroCall { pub path: Interned, pub ast_id: FileAstId, pub expand_to: ExpandTo, + pub call_site: SyntaxContextId, } #[derive(Debug, Clone, Eq, PartialEq)] @@ -779,7 +779,7 @@ impl Use { // Note: The AST unwraps are fine, since if they fail we should have never obtained `index`. let ast = InFile::new(file_id, self.ast_id).to_node(db.upcast()); let ast_use_tree = ast.use_tree().expect("missing `use_tree`"); - let hygiene = Hygiene::new(db.upcast(), file_id); + let hygiene = db.span_map(file_id); let (_, source_map) = lower::lower_use_tree(db, &hygiene, ast_use_tree).expect("failed to lower use tree"); source_map[index].clone() @@ -794,7 +794,7 @@ impl Use { // Note: The AST unwraps are fine, since if they fail we should have never obtained `index`. let ast = InFile::new(file_id, self.ast_id).to_node(db.upcast()); let ast_use_tree = ast.use_tree().expect("missing `use_tree`"); - let hygiene = Hygiene::new(db.upcast(), file_id); + let hygiene = db.span_map(file_id); lower::lower_use_tree(db, &hygiene, ast_use_tree).expect("failed to lower use tree").1 } } diff --git a/crates/hir-def/src/item_tree/lower.rs b/crates/hir-def/src/item_tree/lower.rs index 807b2a7bf752..0b3def6d756b 100644 --- a/crates/hir-def/src/item_tree/lower.rs +++ b/crates/hir-def/src/item_tree/lower.rs @@ -3,7 +3,7 @@ use std::collections::hash_map::Entry; use base_db::span::ErasedFileAstId; -use hir_expand::{ast_id_map::AstIdMap, hygiene::Hygiene, HirFileId}; +use hir_expand::{ast_id_map::AstIdMap, HirFileId, SpanMap}; use syntax::ast::{self, HasModuleItem, HasTypeBounds}; use crate::{ @@ -37,8 +37,8 @@ impl<'a> Ctx<'a> { } } - pub(super) fn hygiene(&self) -> &Hygiene { - self.body_ctx.hygiene() + pub(super) fn span_map(&self) -> &SpanMap { + self.body_ctx.span_map() } pub(super) fn lower_module_items(mut self, item_owner: &dyn HasModuleItem) -> ItemTree { @@ -90,7 +90,7 @@ impl<'a> Ctx<'a> { ast_id: self.source_ast_id_map.ast_id(block).erase(), }, block, - self.hygiene(), + self.span_map(), ), ); self.tree.top_level = block @@ -145,7 +145,7 @@ impl<'a> Ctx<'a> { self.db.upcast(), SpanAnchor { file_id: self.file, ast_id: mod_item.ast_id(&self.tree).erase() }, item, - self.hygiene(), + self.span_map(), ); self.add_attrs(mod_item.into(), attrs); @@ -174,7 +174,7 @@ impl<'a> Ctx<'a> { self.db.upcast(), SpanAnchor { file_id: self.file, ast_id: item.ast_id(&self.tree).erase() }, item_node, - self.hygiene(), + self.span_map(), ); self.add_attrs( match item { @@ -227,7 +227,7 @@ impl<'a> Ctx<'a> { self.db.upcast(), SpanAnchor { file_id: self.file, ast_id }, &field, - self.hygiene(), + self.span_map(), ), ); } @@ -260,7 +260,7 @@ impl<'a> Ctx<'a> { self.db.upcast(), SpanAnchor { file_id: self.file, ast_id }, &field, - self.hygiene(), + self.span_map(), ), ); } @@ -314,7 +314,7 @@ impl<'a> Ctx<'a> { self.db.upcast(), SpanAnchor { file_id: self.file, ast_id }, &variant, - self.hygiene(), + self.span_map(), ), ); } @@ -370,7 +370,7 @@ impl<'a> Ctx<'a> { self.db.upcast(), SpanAnchor { file_id: self.file, ast_id: ast_id.erase() }, &self_param, - self.hygiene(), + self.span_map(), ), ); has_self_param = true; @@ -396,7 +396,7 @@ impl<'a> Ctx<'a> { self.db.upcast(), SpanAnchor { file_id: self.file, ast_id: ast_id.erase() }, ¶m, - self.hygiene(), + self.span_map(), ), ); } @@ -585,7 +585,7 @@ impl<'a> Ctx<'a> { fn lower_use(&mut self, use_item: &ast::Use) -> Option> { let visibility = self.lower_visibility(use_item); let ast_id = self.source_ast_id_map.ast_id(use_item); - let (use_tree, _) = lower_use_tree(self.db, self.hygiene(), use_item.use_tree()?)?; + let (use_tree, _) = lower_use_tree(self.db, self.span_map(), use_item.use_tree()?)?; let res = Use { visibility, ast_id, use_tree }; Some(id(self.data().uses.alloc(res))) @@ -607,10 +607,18 @@ impl<'a> Ctx<'a> { } fn lower_macro_call(&mut self, m: &ast::MacroCall) -> Option> { - let path = Interned::new(ModPath::from_src(self.db.upcast(), m.path()?, self.hygiene())?); + let span_map = self.span_map(); + let path = Interned::new(ModPath::from_src(self.db.upcast(), m.path()?, span_map)?); let ast_id = self.source_ast_id_map.ast_id(m); let expand_to = hir_expand::ExpandTo::from_call_site(m); - let res = MacroCall { path, ast_id, expand_to }; + let res = MacroCall { + path, + ast_id, + expand_to, + call_site: span_map + .span_for_range(m.syntax().text_range()) + .map_or(SyntaxContextId::ROOT, |s| s.ctx), + }; Some(id(self.data().macro_calls.alloc(res))) } @@ -655,7 +663,7 @@ impl<'a> Ctx<'a> { ast_id: mod_item.ast_id(&self.tree).erase(), }, &item, - self.hygiene(), + self.span_map(), ); self.add_attrs(mod_item.into(), attrs); Some(mod_item) @@ -697,7 +705,7 @@ impl<'a> Ctx<'a> { self.db.upcast(), SpanAnchor { file_id: self.file, ast_id: owner_ast_id }, ¶m, - self.body_ctx.hygiene(), + self.body_ctx.span_map(), ); // This is identical to the body of `Ctx::add_attrs()` but we can't call that here // because it requires `&mut self` and the call to `generics.fill()` below also @@ -731,7 +739,7 @@ impl<'a> Ctx<'a> { } fn lower_visibility(&mut self, item: &dyn ast::HasVisibility) -> RawVisibilityId { - let vis = RawVisibility::from_ast_with_hygiene(self.db, item.visibility(), self.hygiene()); + let vis = RawVisibility::from_ast_with_hygiene(self.db, item.visibility(), self.span_map()); self.data().vis.alloc(vis) } @@ -809,7 +817,7 @@ fn lower_abi(abi: ast::Abi) -> Interned { struct UseTreeLowering<'a> { db: &'a dyn DefDatabase, - hygiene: &'a Hygiene, + hygiene: &'a SpanMap, mapping: Arena, } @@ -877,7 +885,7 @@ impl UseTreeLowering<'_> { pub(crate) fn lower_use_tree( db: &dyn DefDatabase, - hygiene: &Hygiene, + hygiene: &SpanMap, tree: ast::UseTree, ) -> Option<(UseTree, Arena)> { let mut lowering = UseTreeLowering { db, hygiene, mapping: Arena::new() }; diff --git a/crates/hir-def/src/item_tree/pretty.rs b/crates/hir-def/src/item_tree/pretty.rs index ca3785bf28df..244111d202ce 100644 --- a/crates/hir-def/src/item_tree/pretty.rs +++ b/crates/hir-def/src/item_tree/pretty.rs @@ -457,7 +457,7 @@ impl Printer<'_> { } } ModItem::MacroCall(it) => { - let MacroCall { path, ast_id: _, expand_to: _ } = &self.tree[it]; + let MacroCall { path, ast_id: _, expand_to: _, call_site: _ } = &self.tree[it]; wln!(self, "{}!(...);", path.display(self.db.upcast())); } ModItem::MacroRules(it) => { diff --git a/crates/hir-def/src/lib.rs b/crates/hir-def/src/lib.rs index 5f09d7c48134..0da605819f9e 100644 --- a/crates/hir-def/src/lib.rs +++ b/crates/hir-def/src/lib.rs @@ -63,7 +63,7 @@ use std::{ panic::{RefUnwindSafe, UnwindSafe}, }; -use base_db::{impl_intern_key, salsa, CrateId, ProcMacroKind}; +use base_db::{impl_intern_key, salsa, span::SyntaxContextId, CrateId, ProcMacroKind}; use hir_expand::{ ast_id_map::{AstIdNode, FileAstId}, attrs::{Attr, AttrId, AttrInput}, @@ -72,7 +72,6 @@ use hir_expand::{ builtin_fn_macro::{BuiltinFnLikeExpander, EagerExpander}, db::ExpandDatabase, eager::expand_eager_macro_input, - hygiene::Hygiene, name::Name, proc_macro::ProcMacroExpander, AstId, ExpandError, ExpandResult, ExpandTo, HirFileId, InFile, MacroCallId, MacroCallKind, @@ -82,7 +81,7 @@ use item_tree::ExternBlock; use la_arena::Idx; use nameres::DefMap; use stdx::impl_from; -use syntax::ast; +use syntax::{ast, AstNode}; pub use hir_expand::tt; @@ -1166,16 +1165,21 @@ impl AsMacroCall for InFile<&ast::MacroCall> { ) -> Result>, UnresolvedMacro> { let expands_to = hir_expand::ExpandTo::from_call_site(self.value); let ast_id = AstId::new(self.file_id, db.ast_id_map(self.file_id).ast_id(self.value)); - let h = Hygiene::new(db, self.file_id); - let path = self.value.path().and_then(|path| path::ModPath::from_src(db, path, &h)); + let span_map = db.span_map(self.file_id); + let path = self.value.path().and_then(|path| path::ModPath::from_src(db, path, &span_map)); let Some(path) = path else { return Ok(ExpandResult::only_err(ExpandError::other("malformed macro invocation"))); }; + let call_site = span_map + .span_for_range(self.value.syntax().text_range()) + .map_or(SyntaxContextId::ROOT, |s| s.ctx); + macro_call_as_call_id_with_eager( db, &AstIdWithPath::new(ast_id.file_id, ast_id.value, path), + call_site, expands_to, krate, resolver, @@ -1200,17 +1204,19 @@ impl AstIdWithPath { fn macro_call_as_call_id( db: &dyn ExpandDatabase, call: &AstIdWithPath, + call_site: SyntaxContextId, expand_to: ExpandTo, krate: CrateId, resolver: impl Fn(path::ModPath) -> Option + Copy, ) -> Result, UnresolvedMacro> { - macro_call_as_call_id_with_eager(db, call, expand_to, krate, resolver, resolver) + macro_call_as_call_id_with_eager(db, call, call_site, expand_to, krate, resolver, resolver) .map(|res| res.value) } fn macro_call_as_call_id_with_eager( db: &dyn ExpandDatabase, call: &AstIdWithPath, + call_site: SyntaxContextId, expand_to: ExpandTo, krate: CrateId, resolver: impl FnOnce(path::ModPath) -> Option, @@ -1231,6 +1237,7 @@ fn macro_call_as_call_id_with_eager( db, krate, MacroCallKind::FnLike { ast_id: call.ast_id, expand_to }, + call_site, )), err: None, }, @@ -1329,6 +1336,8 @@ fn derive_macro_as_call_id( derive_index: derive_pos, derive_attr_index, }, + //FIXME + SyntaxContextId::ROOT, ); Ok((macro_id, def_id, call_id)) } @@ -1358,6 +1367,8 @@ fn attr_macro_as_call_id( attr_args: Arc::new(arg), invoc_attr_index: macro_attr.id, }, + //FIXME + SyntaxContextId::ROOT, ) } intern::impl_internable!( diff --git a/crates/hir-def/src/lower.rs b/crates/hir-def/src/lower.rs index 52781d988921..28a652a60a76 100644 --- a/crates/hir-def/src/lower.rs +++ b/crates/hir-def/src/lower.rs @@ -3,8 +3,7 @@ use std::cell::OnceCell; use hir_expand::{ ast_id_map::{AstIdMap, AstIdNode}, - hygiene::Hygiene, - AstId, HirFileId, InFile, + AstId, HirFileId, InFile, SpanMap, }; use syntax::ast; use triomphe::Arc; @@ -13,28 +12,25 @@ use crate::{db::DefDatabase, path::Path}; pub struct LowerCtx<'a> { pub db: &'a dyn DefDatabase, - hygiene: Hygiene, + hygiene: Arc, + // FIXME: This optimization is probably pointless, ast id map should pretty much always exist anyways. ast_id_map: Option<(HirFileId, OnceCell>)>, } impl<'a> LowerCtx<'a> { - pub fn new(db: &'a dyn DefDatabase, hygiene: &Hygiene, file_id: HirFileId) -> Self { - LowerCtx { db, hygiene: hygiene.clone(), ast_id_map: Some((file_id, OnceCell::new())) } + pub fn new(db: &'a dyn DefDatabase, hygiene: Arc, file_id: HirFileId) -> Self { + LowerCtx { db, hygiene, ast_id_map: Some((file_id, OnceCell::new())) } } pub fn with_file_id(db: &'a dyn DefDatabase, file_id: HirFileId) -> Self { - LowerCtx { - db, - hygiene: Hygiene::new(db.upcast(), file_id), - ast_id_map: Some((file_id, OnceCell::new())), - } + LowerCtx { db, hygiene: db.span_map(file_id), ast_id_map: Some((file_id, OnceCell::new())) } } - pub fn with_hygiene(db: &'a dyn DefDatabase, hygiene: &Hygiene) -> Self { - LowerCtx { db, hygiene: hygiene.clone(), ast_id_map: None } + pub fn with_hygiene(db: &'a dyn DefDatabase, hygiene: Arc) -> Self { + LowerCtx { db, hygiene, ast_id_map: None } } - pub(crate) fn hygiene(&self) -> &Hygiene { + pub(crate) fn span_map(&self) -> &SpanMap { &self.hygiene } diff --git a/crates/hir-def/src/macro_expansion_tests/mbe.rs b/crates/hir-def/src/macro_expansion_tests/mbe.rs index b5d70052a89e..af4b3e12b9f3 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe.rs @@ -26,6 +26,7 @@ macro_rules! f { // +tokenids f!(struct MyTraitMap2); "#, + // FIXME: #SpanAnchor(FileId(0), 1)@91..92 why is there whitespace annotated with a span here? expect![[r#" macro_rules! f { ( struct $ident:ident ) => { @@ -36,7 +37,7 @@ macro_rules! f { } struct#SpanAnchor(FileId(0), 1)@58..64 MyTraitMap2#SpanAnchor(FileId(0), 2)@23..34 {#SpanAnchor(FileId(0), 1)@72..73 - map#SpanAnchor(FileId(0), 1)@86..89:#SpanAnchor(FileId(0), 1)@89..90 ::std#SpanAnchor(FileId(0), 1)@93..96::collections#SpanAnchor(FileId(0), 1)@98..109::HashSet#SpanAnchor(FileId(0), 1)@111..118<#SpanAnchor(FileId(0), 1)@118..119(#SpanAnchor(FileId(0), 1)@119..120)#SpanAnchor(FileId(0), 1)@120..121>#SpanAnchor(FileId(0), 1)@121..122,#SpanAnchor(FileId(0), 1)@122..123 + map#SpanAnchor(FileId(0), 1)@86..89:#SpanAnchor(FileId(0), 1)@89..90 #SpanAnchor(FileId(0), 1)@91..92::#SpanAnchor(FileId(0), 1)@92..93std#SpanAnchor(FileId(0), 1)@93..96::#SpanAnchor(FileId(0), 1)@97..98collections#SpanAnchor(FileId(0), 1)@98..109::#SpanAnchor(FileId(0), 1)@110..111HashSet#SpanAnchor(FileId(0), 1)@111..118<#SpanAnchor(FileId(0), 1)@118..119(#SpanAnchor(FileId(0), 1)@119..120)#SpanAnchor(FileId(0), 1)@120..121>#SpanAnchor(FileId(0), 1)@121..122,#SpanAnchor(FileId(0), 1)@122..123 }#SpanAnchor(FileId(0), 1)@132..133 "#]], ); @@ -938,9 +939,9 @@ macro_rules! vec { fn f() { { let mut v = Vec::new(); - v.push(1); - v.push(2); - v.push(3); + v.push((1)); + v.push((2)); + v.push((3)); v }; } @@ -1409,8 +1410,8 @@ macro_rules! matches { }; } fn main() { - match 0 { - 0|1 if true =>true , _=>false + match (0) { + 0|1 if (true )=>true , _=>false }; } "#]], diff --git a/crates/hir-def/src/macro_expansion_tests/mbe/metavar_expr.rs b/crates/hir-def/src/macro_expansion_tests/mbe/metavar_expr.rs index 967b5ad36bab..dd83e5c04d45 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe/metavar_expr.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe/metavar_expr.rs @@ -62,10 +62,10 @@ macro_rules !implement_methods { struct Foo; impl Foo { fn alpha() -> &'static[u32] { - &[1, 2, 3] + &[(1), (2), (3)] } fn beta() -> &'static[u32] { - &[1, 2, 3] + &[(1), (2), (3)] } } "#]], diff --git a/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs b/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs index 2886b2a366c0..71dbb400b548 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs @@ -39,8 +39,8 @@ fn main() { }; { let mut v = Vec::new(); - v.push(1u32); - v.push(2); + v.push((1u32)); + v.push((2)); v }; } diff --git a/crates/hir-def/src/macro_expansion_tests/mbe/tt_conversion.rs b/crates/hir-def/src/macro_expansion_tests/mbe/tt_conversion.rs index ae56934f632f..b2ac7eb4094d 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe/tt_conversion.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe/tt_conversion.rs @@ -192,9 +192,9 @@ macro_rules! constant { ($e:expr ;) => {$e}; } -const _: () = 0.0; -const _: () = 0.; -const _: () = 0e0; +const _: () = (0.0); +const _: () = (0.); +const _: () = (0e0); "#]], ); } diff --git a/crates/hir-def/src/macro_expansion_tests/mod.rs b/crates/hir-def/src/macro_expansion_tests/mod.rs index 1a672b4605c6..d4902c52e748 100644 --- a/crates/hir-def/src/macro_expansion_tests/mod.rs +++ b/crates/hir-def/src/macro_expansion_tests/mod.rs @@ -123,8 +123,9 @@ pub fn identity_when_valid(_attr: TokenStream, item: TokenStream) -> TokenStream } else { assert!( parse.errors().is_empty(), - "parse errors in expansion: \n{:#?}", - parse.errors() + "parse errors in expansion: \n{:#?}\n```\n{}\n```", + parse.errors(), + parse.syntax_node(), ); } let pp = pretty_print_macro_expansion( diff --git a/crates/hir-def/src/nameres/collector.rs b/crates/hir-def/src/nameres/collector.rs index 659e7ed5033e..360bf0f93e30 100644 --- a/crates/hir-def/src/nameres/collector.rs +++ b/crates/hir-def/src/nameres/collector.rs @@ -6,7 +6,7 @@ use std::{cmp::Ordering, iter, mem}; use ::tt::Span; -use base_db::{CrateId, Dependency, Edition, FileId}; +use base_db::{span::SyntaxContextId, CrateId, Dependency, Edition, FileId}; use cfg::{CfgExpr, CfgOptions}; use either::Either; use hir_expand::{ @@ -15,7 +15,6 @@ use hir_expand::{ builtin_attr_macro::find_builtin_attr, builtin_derive_macro::find_builtin_derive, builtin_fn_macro::find_builtin_macro, - hygiene::Hygiene, name::{name, AsName, Name}, proc_macro::ProcMacroExpander, ExpandResult, ExpandTo, HirFileId, InFile, MacroCallId, MacroCallKind, MacroCallLoc, @@ -112,7 +111,6 @@ pub(super) fn collect_defs(db: &dyn DefDatabase, def_map: DefMap, tree_id: TreeI from_glob_import: Default::default(), skip_attrs: Default::default(), is_proc_macro, - hygienes: FxHashMap::default(), }; if tree_id.is_block() { collector.seed_with_inner(tree_id); @@ -212,9 +210,22 @@ struct MacroDirective { #[derive(Clone, Debug, Eq, PartialEq)] enum MacroDirectiveKind { - FnLike { ast_id: AstIdWithPath, expand_to: ExpandTo }, - Derive { ast_id: AstIdWithPath, derive_attr: AttrId, derive_pos: usize }, - Attr { ast_id: AstIdWithPath, attr: Attr, mod_item: ModItem, tree: TreeId }, + FnLike { + ast_id: AstIdWithPath, + expand_to: ExpandTo, + call_site: SyntaxContextId, + }, + Derive { + ast_id: AstIdWithPath, + derive_attr: AttrId, + derive_pos: usize, + }, + Attr { + ast_id: AstIdWithPath, + attr: Attr, + mod_item: ModItem, + /* is this needed? */ tree: TreeId, + }, } /// Walks the tree of module recursively @@ -242,12 +253,6 @@ struct DefCollector<'a> { /// This also stores the attributes to skip when we resolve derive helpers and non-macro /// non-builtin attributes in general. skip_attrs: FxHashMap, AttrId>, - /// `Hygiene` cache, because `Hygiene` construction is expensive. - /// - /// Almost all paths should have been lowered to `ModPath` during `ItemTree` construction. - /// However, `DefCollector` still needs to lower paths in attributes, in particular those in - /// derive meta item list. - hygienes: FxHashMap, } impl DefCollector<'_> { @@ -315,9 +320,8 @@ impl DefCollector<'_> { } if *attr_name == hir_expand::name![feature] { - let hygiene = &Hygiene::new_unhygienic(); let features = attr - .parse_path_comma_token_tree(self.db.upcast(), hygiene) + .parse_path_comma_token_tree(self.db.upcast()) .into_iter() .flatten() .filter_map(|feat| match feat.segments() { @@ -1119,10 +1123,11 @@ impl DefCollector<'_> { let resolver_def_id = |path| resolver(path).map(|(_, it)| it); match &directive.kind { - MacroDirectiveKind::FnLike { ast_id, expand_to } => { + MacroDirectiveKind::FnLike { ast_id, expand_to, call_site } => { let call_id = macro_call_as_call_id( self.db.upcast(), ast_id, + *call_site, *expand_to, self.def_map.krate, resolver_def_id, @@ -1234,19 +1239,7 @@ impl DefCollector<'_> { }; let ast_id = ast_id.with_value(ast_adt_id); - let extend_unhygenic; - let hygiene = if file_id.is_macro() { - self.hygienes - .entry(file_id) - .or_insert_with(|| Hygiene::new(self.db.upcast(), file_id)) - } else { - // Avoid heap allocation (`Hygiene` embraces `Arc`) and hash map entry - // when we're in an oridinary (non-macro) file. - extend_unhygenic = Hygiene::new_unhygienic(); - &extend_unhygenic - }; - - match attr.parse_path_comma_token_tree(self.db.upcast(), hygiene) { + match attr.parse_path_comma_token_tree(self.db.upcast()) { Some(derive_macros) => { let mut len = 0; for (idx, path) in derive_macros.enumerate() { @@ -1414,11 +1407,12 @@ impl DefCollector<'_> { for directive in &self.unresolved_macros { match &directive.kind { - MacroDirectiveKind::FnLike { ast_id, expand_to } => { + MacroDirectiveKind::FnLike { ast_id, expand_to, call_site } => { // FIXME: we shouldn't need to re-resolve the macro here just to get the unresolved error! let macro_call_as_call_id = macro_call_as_call_id( self.db.upcast(), ast_id, + *call_site, *expand_to, self.def_map.krate, |path| { @@ -1823,9 +1817,8 @@ impl ModCollector<'_, '_> { cov_mark::hit!(macro_rules_from_other_crates_are_visible_with_macro_use); let mut single_imports = Vec::new(); - let hygiene = Hygiene::new_unhygienic(); for attr in macro_use_attrs { - let Some(paths) = attr.parse_path_comma_token_tree(db.upcast(), &hygiene) else { + let Some(paths) = attr.parse_path_comma_token_tree(db.upcast()) else { // `#[macro_use]` (without any paths) found, forget collected names and just import // all visible macros. self.def_collector.import_macros_from_extern_crate( @@ -2209,8 +2202,12 @@ impl ModCollector<'_, '_> { } } - fn collect_macro_call(&mut self, mac: &MacroCall, container: ItemContainerId) { - let ast_id = AstIdWithPath::new(self.file_id(), mac.ast_id, ModPath::clone(&mac.path)); + fn collect_macro_call( + &mut self, + &MacroCall { ref path, ast_id, expand_to, call_site }: &MacroCall, + container: ItemContainerId, + ) { + let ast_id = AstIdWithPath::new(self.file_id(), ast_id, ModPath::clone(&path)); let db = self.def_collector.db; // FIXME: Immediately expanding in "Case 1" is insufficient since "Case 2" may also define @@ -2221,7 +2218,8 @@ impl ModCollector<'_, '_> { if let Ok(res) = macro_call_as_call_id_with_eager( db.upcast(), &ast_id, - mac.expand_to, + call_site, + expand_to, self.def_collector.def_map.krate, |path| { path.as_ident().and_then(|name| { @@ -2275,7 +2273,7 @@ impl ModCollector<'_, '_> { self.def_collector.unresolved_macros.push(MacroDirective { module_id: self.module_id, depth: self.macro_depth + 1, - kind: MacroDirectiveKind::FnLike { ast_id, expand_to: mac.expand_to }, + kind: MacroDirectiveKind::FnLike { ast_id, expand_to: expand_to, call_site }, container, }); } @@ -2362,7 +2360,6 @@ mod tests { from_glob_import: Default::default(), skip_attrs: Default::default(), is_proc_macro: false, - hygienes: FxHashMap::default(), }; collector.seed_with_top_level(); collector.collect(); diff --git a/crates/hir-def/src/path/lower.rs b/crates/hir-def/src/path/lower.rs index abd817893cc4..ee49dfa44c80 100644 --- a/crates/hir-def/src/path/lower.rs +++ b/crates/hir-def/src/path/lower.rs @@ -4,8 +4,11 @@ use std::iter; use crate::{lower::LowerCtx, type_ref::ConstRef}; -use either::Either; -use hir_expand::name::{name, AsName}; +use base_db::span::SyntaxContextId; +use hir_expand::{ + mod_path::resolve_crate_root, + name::{name, AsName}, +}; use intern::Interned; use syntax::ast::{self, AstNode, HasTypeBounds}; @@ -14,14 +17,17 @@ use crate::{ type_ref::{LifetimeRef, TypeBound, TypeRef}, }; +// fn resolve_crate_root + /// Converts an `ast::Path` to `Path`. Works with use trees. /// It correctly handles `$crate` based path from macro call. +// FIXME: flip the params pub(super) fn lower_path(mut path: ast::Path, ctx: &LowerCtx<'_>) -> Option { let mut kind = PathKind::Plain; let mut type_anchor = None; let mut segments = Vec::new(); let mut generic_args = Vec::new(); - let hygiene = ctx.hygiene(); + let hygiene = ctx.span_map(); loop { let segment = path.segment()?; @@ -31,31 +37,34 @@ pub(super) fn lower_path(mut path: ast::Path, ctx: &LowerCtx<'_>) -> Option { - // FIXME: this should just return name - match hygiene.name_ref_to_name(ctx.db.upcast(), name_ref) { - Either::Left(name) => { - let args = segment - .generic_arg_list() - .and_then(|it| lower_generic_args(ctx, it)) - .or_else(|| { - lower_generic_args_from_fn_path( - ctx, - segment.param_list(), - segment.ret_type(), - ) - }) - .map(Interned::new); - if let Some(_) = args { - generic_args.resize(segments.len(), None); - generic_args.push(args); - } - segments.push(name); - } - Either::Right(crate_id) => { - kind = PathKind::DollarCrate(crate_id); - break; - } + let name = if name_ref.text() == "$crate" { + kind = resolve_crate_root( + ctx.db.upcast(), + hygiene + .span_for_range(name_ref.syntax().text_range()) + .map_or(SyntaxContextId::ROOT, |s| s.ctx), + ) + .map(PathKind::DollarCrate)?; + break; + } else { + name_ref.as_name() + }; + let args = segment + .generic_arg_list() + .and_then(|it| lower_generic_args(ctx, it)) + .or_else(|| { + lower_generic_args_from_fn_path( + ctx, + segment.param_list(), + segment.ret_type(), + ) + }) + .map(Interned::new); + if let Some(_) = args { + generic_args.resize(segments.len(), None); + generic_args.push(args); } + segments.push(name); } ast::PathSegmentKind::SelfTypeKw => { segments.push(name![Self]); @@ -151,8 +160,16 @@ pub(super) fn lower_path(mut path: ast::Path, ctx: &LowerCtx<'_>) -> Option Self { let mut this = Self { storage: Default::default(), events: Default::default() }; + this.intern_syntax_context(SyntaxContextData::root()); this.set_expand_proc_attr_macros_with_durability(true, Durability::HIGH); this } diff --git a/crates/hir-def/src/visibility.rs b/crates/hir-def/src/visibility.rs index 30f48de61f2e..b341b8cfbd96 100644 --- a/crates/hir-def/src/visibility.rs +++ b/crates/hir-def/src/visibility.rs @@ -2,7 +2,7 @@ use std::iter; -use hir_expand::{hygiene::Hygiene, InFile}; +use hir_expand::{InFile, SpanMap}; use la_arena::ArenaMap; use syntax::ast; use triomphe::Arc; @@ -34,13 +34,13 @@ impl RawVisibility { db: &dyn DefDatabase, node: InFile>, ) -> RawVisibility { - Self::from_ast_with_hygiene(db, node.value, &Hygiene::new(db.upcast(), node.file_id)) + Self::from_ast_with_hygiene(db, node.value, &db.span_map(node.file_id)) } pub(crate) fn from_ast_with_hygiene( db: &dyn DefDatabase, node: Option, - hygiene: &Hygiene, + hygiene: &SpanMap, ) -> RawVisibility { Self::from_ast_with_hygiene_and_default(db, node, RawVisibility::private(), hygiene) } @@ -49,7 +49,7 @@ impl RawVisibility { db: &dyn DefDatabase, node: Option, default: RawVisibility, - hygiene: &Hygiene, + hygiene: &SpanMap, ) -> RawVisibility { let node = match node { None => return default, diff --git a/crates/hir-expand/src/attrs.rs b/crates/hir-expand/src/attrs.rs index 01a66cd03ab1..5ce12d2f6e71 100644 --- a/crates/hir-expand/src/attrs.rs +++ b/crates/hir-expand/src/attrs.rs @@ -13,10 +13,9 @@ use triomphe::Arc; use crate::{ db::ExpandDatabase, - hygiene::Hygiene, mod_path::ModPath, tt::{self, Subtree}, - InFile, + InFile, SpanMap, }; /// Syntactical attributes, without filtering of `cfg_attr`s. @@ -44,7 +43,7 @@ impl RawAttrs { db: &dyn ExpandDatabase, span_anchor: SpanAnchor, owner: &dyn ast::HasAttrs, - hygiene: &Hygiene, + hygiene: &SpanMap, ) -> Self { let entries = collect_attrs(owner) .filter_map(|(id, attr)| match attr { @@ -69,8 +68,7 @@ impl RawAttrs { span_anchor: SpanAnchor, owner: InFile<&dyn ast::HasAttrs>, ) -> Self { - let hygiene = Hygiene::new(db, owner.file_id); - Self::new(db, span_anchor, owner.value, &hygiene) + Self::new(db, span_anchor, owner.value, &db.span_map(owner.file_id)) } pub fn merge(&self, other: Self) -> Self { @@ -135,9 +133,7 @@ impl RawAttrs { delimiter: tt::Delimiter::unspecified(), token_trees: attr.to_vec(), }; - // FIXME hygiene - let hygiene = Hygiene::new_unhygienic(); - Attr::from_tt(db, &tree, &hygiene, index.with_cfg_attr(idx)) + Attr::from_tt(db, &tree, index.with_cfg_attr(idx)) }, ); @@ -220,7 +216,7 @@ impl Attr { db: &dyn ExpandDatabase, span_anchor: SpanAnchor, ast: ast::Meta, - hygiene: &Hygiene, + hygiene: &SpanMap, id: AttrId, ) -> Option { let path = Interned::new(ModPath::from_src(db, ast.path()?, hygiene)?); @@ -234,9 +230,7 @@ impl Attr { // FIXME: We could also allocate ids for attributes and use the attribute itself as an anchor let offset = db.ast_id_map(span_anchor.file_id).get_raw(span_anchor.ast_id).text_range().start(); - // FIXME: Spanmap - let tree = - syntax_node_to_token_tree(tt.syntax(), span_anchor, offset, &Default::default()); + let tree = syntax_node_to_token_tree(tt.syntax(), span_anchor, offset, hygiene); Some(Interned::new(AttrInput::TokenTree(Box::new(tree)))) } else { None @@ -244,18 +238,13 @@ impl Attr { Some(Attr { id, path, input }) } - fn from_tt( - db: &dyn ExpandDatabase, - tt: &tt::Subtree, - hygiene: &Hygiene, - id: AttrId, - ) -> Option { + fn from_tt(db: &dyn ExpandDatabase, tt: &tt::Subtree, id: AttrId) -> Option { // FIXME: Unecessary roundtrip tt -> ast -> tt let (parse, _map) = mbe::token_tree_to_syntax_node(tt, mbe::TopEntryPoint::MetaItem); let ast = ast::Meta::cast(parse.syntax_node())?; // FIXME: we discard spans here! - Self::from_src(db, SpanAnchor::DUMMY, ast, hygiene, id) + Self::from_src(db, SpanAnchor::DUMMY, ast, &SpanMap::default(), id) } pub fn path(&self) -> &ModPath { @@ -295,9 +284,9 @@ impl Attr { pub fn parse_path_comma_token_tree<'a>( &'a self, db: &'a dyn ExpandDatabase, - hygiene: &'a Hygiene, ) -> Option + 'a> { let args = self.token_tree_value()?; + dbg!(args); if args.delimiter.kind != DelimiterKind::Parenthesis { return None; @@ -309,12 +298,13 @@ impl Attr { if tts.is_empty() { return None; } - // FIXME: This is necessarily a hack. It'd be nice if we could avoid allocation here. + // FIXME: This is necessarily a hack. It'd be nice if we could avoid allocation + // here. let subtree = tt::Subtree { delimiter: tt::Delimiter::unspecified(), - token_trees: tts.into_iter().cloned().collect(), + token_trees: tts.to_vec(), }; - let (parse, _) = + let (parse, span_map) = mbe::token_tree_to_syntax_node(&subtree, mbe::TopEntryPoint::MetaItem); let meta = ast::Meta::cast(parse.syntax_node())?; // Only simple paths are allowed. @@ -323,7 +313,7 @@ impl Attr { return None; } let path = meta.path()?; - ModPath::from_src(db, path, hygiene) + ModPath::from_src(db, path, &span_map) }); Some(paths) diff --git a/crates/hir-expand/src/db.rs b/crates/hir-expand/src/db.rs index 1a68653a6fcd..7dd69099a6e3 100644 --- a/crates/hir-expand/src/db.rs +++ b/crates/hir-expand/src/db.rs @@ -1,6 +1,6 @@ //! Defines database & queries for macro expansion. -use ::tt::SyntaxContext; +use ::tt::{SpanAnchor as _, SyntaxContext}; use base_db::{ salsa, span::{SpanAnchor, SyntaxContextId, ROOT_ERASED_FILE_AST_ID}, @@ -17,9 +17,10 @@ use triomphe::Arc; use crate::{ ast_id_map::AstIdMap, + attrs::RawAttrs, builtin_attr_macro::pseudo_derive_attr_expansion, builtin_fn_macro::EagerExpander, - hygiene::{self, HygieneFrame, SyntaxContextData}, + hygiene::{self, SyntaxContextData, Transparency}, tt, AstId, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander, EagerCallInfo, ExpandError, ExpandResult, ExpandTo, HirFileId, HirFileIdRepr, MacroCallId, MacroCallKind, MacroCallLoc, MacroDefId, MacroDefKind, MacroFile, ProcMacroExpander, SpanMap, @@ -37,16 +38,35 @@ static TOKEN_LIMIT: Limit = Limit::new(1_048_576); /// Old-style `macro_rules` or the new macros 2.0 pub struct DeclarativeMacroExpander { pub mac: mbe::DeclarativeMacro, + pub transparency: Transparency, } impl DeclarativeMacroExpander { - pub fn expand(&self, tt: tt::Subtree) -> ExpandResult { + pub fn expand( + &self, + db: &dyn ExpandDatabase, + tt: tt::Subtree, + call_id: MacroCallId, + ) -> ExpandResult { + match self.mac.err() { + Some(e) => ExpandResult::new( + tt::Subtree::empty(), + ExpandError::other(format!("invalid macro definition: {e}")), + ), + None => self + .mac + .expand(&tt, |s| s.ctx = db.apply_mark(s.ctx, call_id, self.transparency)) + .map_err(Into::into), + } + } + + pub fn expand_unhygienic(&self, tt: tt::Subtree) -> ExpandResult { match self.mac.err() { Some(e) => ExpandResult::new( tt::Subtree::empty(), ExpandError::other(format!("invalid macro definition: {e}")), ), - None => self.mac.expand(&tt).map_err(Into::into), + None => self.mac.expand(&tt, |_| ()).map_err(Into::into), } } } @@ -83,6 +103,9 @@ pub trait ExpandDatabase: SourceDatabase { &self, macro_file: MacroFile, ) -> ExpandResult<(Parse, Arc)>; + // TODO: transparent? + #[salsa::transparent] + fn span_map(&self, file_id: HirFileId) -> Arc; /// Macro ids. That's probably the tricksiest bit in rust-analyzer, and the /// reason why we use salsa at all. @@ -97,8 +120,8 @@ pub trait ExpandDatabase: SourceDatabase { #[salsa::invoke(hygiene::apply_mark)] fn apply_mark( &self, - ctxt: SyntaxContextData, - file_id: HirFileId, + ctxt: SyntaxContextId, + call_id: MacroCallId, transparency: hygiene::Transparency, ) -> SyntaxContextId; @@ -137,8 +160,13 @@ pub trait ExpandDatabase: SourceDatabase { &self, macro_call: MacroCallId, ) -> ExpandResult>; +} - fn hygiene_frame(&self, file_id: HirFileId) -> Arc; +fn span_map(db: &dyn ExpandDatabase, file_id: HirFileId) -> Arc { + match file_id.repr() { + HirFileIdRepr::FileId(_) => Arc::new(Default::default()), + HirFileIdRepr::MacroFile(m) => db.parse_macro_expansion(m).value.1, + } } /// This expands the given macro call, but with different arguments. This is @@ -220,7 +248,9 @@ pub fn expand_speculative( ), ) } - MacroDefKind::Declarative(it) => db.decl_macro_expander(loc.krate, it).expand(tt), + MacroDefKind::Declarative(it) => { + db.decl_macro_expander(loc.krate, it).expand_unhygienic(tt) + } MacroDefKind::BuiltIn(it, _) => it.expand(db, actual_macro_call, &tt).map_err(Into::into), MacroDefKind::BuiltInEager(it, _) => { it.expand(db, actual_macro_call, &tt).map_err(Into::into) @@ -229,7 +259,9 @@ pub fn expand_speculative( }; let expand_to = macro_expand_to(db, actual_macro_call); - let (node, rev_tmap) = token_tree_to_syntax_node(db, &speculative_expansion.value, expand_to); + let (node, mut rev_tmap) = + token_tree_to_syntax_node(db, &speculative_expansion.value, expand_to); + rev_tmap.real_file = false; let syntax_node = node.syntax_node(); let token = rev_tmap @@ -285,7 +317,8 @@ fn parse_macro_expansion( tracing::debug!("expanded = {}", tt.as_debug_string()); tracing::debug!("kind = {:?}", expand_to); - let (parse, rev_token_map) = token_tree_to_syntax_node(db, &tt, expand_to); + let (parse, mut rev_token_map) = token_tree_to_syntax_node(db, &tt, expand_to); + rev_token_map.real_file = false; ExpandResult { value: (parse, Arc::new(rev_token_map)), err } } @@ -464,41 +497,70 @@ fn decl_macro_expander( (parse.syntax_node(), map) } }; - let mac = match id.to_ptr(db).to_node(&root) { - ast::Macro::MacroRules(macro_rules) => match macro_rules.token_tree() { - Some(arg) => { - let tt = mbe::syntax_node_to_token_tree( - arg.syntax(), - SpanAnchor { file_id: id.file_id, ast_id: id.value.erase() }, - macro_rules.syntax().text_range().start(), - &map, - ); - let mac = mbe::DeclarativeMacro::parse_macro_rules(&tt, is_2021); - mac - } - None => mbe::DeclarativeMacro::from_err( - mbe::ParseError::Expected("expected a token tree".into()), - is_2021, - ), - }, - ast::Macro::MacroDef(macro_def) => match macro_def.body() { - Some(arg) => { - let tt = mbe::syntax_node_to_token_tree( - arg.syntax(), - SpanAnchor { file_id: id.file_id, ast_id: id.value.erase() }, - macro_def.syntax().text_range().start(), - &map, - ); - let mac = mbe::DeclarativeMacro::parse_macro2(&tt, is_2021); - mac - } - None => mbe::DeclarativeMacro::from_err( - mbe::ParseError::Expected("expected a token tree".into()), - is_2021, - ), - }, + + let transparency = |node| { + // ... would be nice to have the item tree here + let attrs = + RawAttrs::new(db, SpanAnchor::DUMMY, node, &Default::default()).filter(db, def_crate); + match &*attrs + .iter() + .find(|it| { + it.path.as_ident().and_then(|it| it.as_str()) == Some("rustc_macro_transparency") + })? + .token_tree_value()? + .token_trees + { + [tt::TokenTree::Leaf(tt::Leaf::Ident(i)), ..] => match &*i.text { + "transparent" => Some(Transparency::Transparent), + "semitransparent" => Some(Transparency::SemiTransparent), + "opaque" => Some(Transparency::Opaque), + _ => None, + }, + _ => None, + } + }; + + let (mac, transparency) = match id.to_ptr(db).to_node(&root) { + ast::Macro::MacroRules(macro_rules) => ( + match macro_rules.token_tree() { + Some(arg) => { + let tt = mbe::syntax_node_to_token_tree( + arg.syntax(), + SpanAnchor { file_id: id.file_id, ast_id: id.value.erase() }, + macro_rules.syntax().text_range().start(), + &map, + ); + let mac = mbe::DeclarativeMacro::parse_macro_rules(&tt, is_2021); + mac + } + None => mbe::DeclarativeMacro::from_err( + mbe::ParseError::Expected("expected a token tree".into()), + is_2021, + ), + }, + transparency(¯o_rules).unwrap_or(Transparency::SemiTransparent), + ), + ast::Macro::MacroDef(macro_def) => ( + match macro_def.body() { + Some(arg) => { + let tt = mbe::syntax_node_to_token_tree( + arg.syntax(), + SpanAnchor { file_id: id.file_id, ast_id: id.value.erase() }, + macro_def.syntax().text_range().start(), + &map, + ); + let mac = mbe::DeclarativeMacro::parse_macro2(&tt, is_2021); + mac + } + None => mbe::DeclarativeMacro::from_err( + mbe::ParseError::Expected("expected a token tree".into()), + is_2021, + ), + }, + transparency(¯o_def).unwrap_or(Transparency::Opaque), + ), }; - Arc::new(DeclarativeMacroExpander { mac }) + Arc::new(DeclarativeMacroExpander { mac, transparency }) } fn macro_expander(db: &dyn ExpandDatabase, id: MacroDefId) -> TokenExpander { @@ -514,12 +576,15 @@ fn macro_expander(db: &dyn ExpandDatabase, id: MacroDefId) -> TokenExpander { } } -fn macro_expand(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult> { +fn macro_expand( + db: &dyn ExpandDatabase, + macro_call_id: MacroCallId, +) -> ExpandResult> { let _p = profile::span("macro_expand"); - let loc = db.lookup_intern_macro_call(id); + let loc = db.lookup_intern_macro_call(macro_call_id); let ExpandResult { value: tt, mut err } = match loc.def.kind { - MacroDefKind::ProcMacro(..) => return db.expand_proc_macro(id), + MacroDefKind::ProcMacro(..) => return db.expand_proc_macro(macro_call_id), MacroDefKind::BuiltInDerive(expander, ..) => { // FIXME: add firewall query for this? let hir_file_id = loc.kind.file_id(); @@ -538,7 +603,7 @@ fn macro_expand(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult map, @@ -554,7 +619,7 @@ fn macro_expand(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult { - let ValueResult { value, err } = db.macro_arg(id); + let ValueResult { value, err } = db.macro_arg(macro_call_id); let Some(macro_arg) = value else { return ExpandResult { value: Arc::new(tt::Subtree { @@ -570,9 +635,11 @@ fn macro_expand(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult { - db.decl_macro_expander(loc.def.krate, id).expand(arg.clone()) + db.decl_macro_expander(loc.def.krate, id).expand(db, arg.clone(), macro_call_id) + } + MacroDefKind::BuiltIn(it, _) => { + it.expand(db, macro_call_id, &arg).map_err(Into::into) } - MacroDefKind::BuiltIn(it, _) => it.expand(db, id, &arg).map_err(Into::into), // This might look a bit odd, but we do not expand the inputs to eager macros here. // Eager macros inputs are expanded, well, eagerly when we collect the macro calls. // That kind of expansion uses the ast id map of an eager macros input though which goes through @@ -594,8 +661,10 @@ fn macro_expand(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult it.expand(db, id, &arg).map_err(Into::into), - MacroDefKind::BuiltInAttr(it, _) => it.expand(db, id, &arg), + MacroDefKind::BuiltInEager(it, _) => { + it.expand(db, macro_call_id, &arg).map_err(Into::into) + } + MacroDefKind::BuiltInAttr(it, _) => it.expand(db, macro_call_id, &arg), _ => unreachable!(), } } @@ -653,10 +722,6 @@ fn expand_proc_macro(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult Arc { - Arc::new(HygieneFrame::new(db, file_id)) -} - fn macro_expand_to(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandTo { db.lookup_intern_macro_call(id).expand_to() } diff --git a/crates/hir-expand/src/eager.rs b/crates/hir-expand/src/eager.rs index ae5f26b5d36a..dc6507a92de5 100644 --- a/crates/hir-expand/src/eager.rs +++ b/crates/hir-expand/src/eager.rs @@ -19,7 +19,7 @@ //! //! See the full discussion : use base_db::{ - span::{SpanAnchor, ROOT_ERASED_FILE_AST_ID}, + span::{SpanAnchor, SyntaxContextId, ROOT_ERASED_FILE_AST_ID}, CrateId, }; use rustc_hash::FxHashMap; @@ -29,7 +29,6 @@ use triomphe::Arc; use crate::{ ast::{self, AstNode}, db::ExpandDatabase, - hygiene::Hygiene, mod_path::ModPath, EagerCallInfo, ExpandError, ExpandResult, ExpandTo, InFile, MacroCallId, MacroCallKind, MacroCallLoc, MacroDefId, MacroDefKind, SpanMap, @@ -56,8 +55,10 @@ pub fn expand_eager_macro_input( krate, eager: None, kind: MacroCallKind::FnLike { ast_id: call_id, expand_to: ExpandTo::Expr }, + // FIXME + call_site: SyntaxContextId::ROOT, }); - let ExpandResult { value: (arg_exp, _arg_exp_map), err: parse_err } = + let ExpandResult { value: (arg_exp, arg_exp_map), err: parse_err } = db.parse_macro_expansion(arg_id.as_macro_file()); // we need this map here as the expansion of the eager input fake file loses whitespace ... // let mut ws_mapping = FxHashMap::default(); @@ -70,7 +71,7 @@ pub fn expand_eager_macro_input( let ExpandResult { value: expanded_eager_input, err } = { eager_macro_recur( db, - &Hygiene::new(db, macro_call.file_id), + &arg_exp_map, InFile::new(arg_id.as_file(), arg_exp.syntax_node()), krate, resolver, @@ -131,6 +132,8 @@ pub fn expand_eager_macro_input( error: err.clone(), })), kind: MacroCallKind::FnLike { ast_id: call_id, expand_to }, + // FIXME + call_site: SyntaxContextId::ROOT, }; ExpandResult { value: Some(db.intern_macro_call(loc)), err } @@ -146,7 +149,13 @@ fn lazy_expand( let expand_to = ExpandTo::from_call_site(¯o_call.value); let ast_id = macro_call.with_value(ast_id); - let id = def.as_lazy_macro(db, krate, MacroCallKind::FnLike { ast_id, expand_to }); + let id = def.as_lazy_macro( + db, + krate, + MacroCallKind::FnLike { ast_id, expand_to }, + // FIXME + SyntaxContextId::ROOT, + ); let macro_file = id.as_macro_file(); db.parse_macro_expansion(macro_file) @@ -155,7 +164,7 @@ fn lazy_expand( fn eager_macro_recur( db: &dyn ExpandDatabase, - hygiene: &Hygiene, + hygiene: &SpanMap, curr: InFile, krate: CrateId, macro_resolver: &dyn Fn(ModPath) -> Option, @@ -250,14 +259,13 @@ fn eager_macro_recur( | MacroDefKind::BuiltInAttr(..) | MacroDefKind::BuiltInDerive(..) | MacroDefKind::ProcMacro(..) => { - let ExpandResult { value: (parse, _tm), err } = + let ExpandResult { value: (parse, tm), err } = lazy_expand(db, &def, curr.with_value(call.clone()), krate); // replace macro inside - let hygiene = Hygiene::new(db, parse.file_id); let ExpandResult { value, err: error } = eager_macro_recur( db, - &hygiene, + &tm, // FIXME: We discard parse errors here parse.as_ref().map(|it| it.syntax_node()), krate, diff --git a/crates/hir-expand/src/hygiene.rs b/crates/hir-expand/src/hygiene.rs index e0688178ffb9..f83a9bf2d657 100644 --- a/crates/hir-expand/src/hygiene.rs +++ b/crates/hir-expand/src/hygiene.rs @@ -2,32 +2,31 @@ //! //! Specifically, `ast` + `Hygiene` allows you to create a `Name`. Note that, at //! this moment, this is horribly incomplete and handles only `$crate`. -use base_db::{span::SyntaxContextId, CrateId}; -use either::Either; -use syntax::{ - ast::{self}, - TextRange, -}; -use triomphe::Arc; +use base_db::span::{MacroCallId, SyntaxContextId}; -use crate::{ - db::ExpandDatabase, - name::{AsName, Name}, - HirFileId, InFile, -}; +use crate::db::ExpandDatabase; #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct SyntaxContextData { - // FIXME: This might only need to be Option? - outer_expn: HirFileId, - outer_transparency: Transparency, - parent: SyntaxContextId, + pub outer_expn: Option, + pub outer_transparency: Transparency, + pub parent: SyntaxContextId, /// This context, but with all transparent and semi-transparent expansions filtered away. - opaque: SyntaxContextId, + pub opaque: SyntaxContextId, /// This context, but with all transparent expansions filtered away. - opaque_and_semitransparent: SyntaxContextId, - /// Name of the crate to which `$crate` with this context would resolve. - dollar_crate_name: Name, + pub opaque_and_semitransparent: SyntaxContextId, +} + +impl SyntaxContextData { + pub fn root() -> Self { + SyntaxContextData { + outer_expn: None, + outer_transparency: Transparency::Opaque, + parent: SyntaxContextId::ROOT, + opaque: SyntaxContextId::ROOT, + opaque_and_semitransparent: SyntaxContextId::ROOT, + } + } } /// A property of a macro expansion that determines how identifiers @@ -50,64 +49,135 @@ pub enum Transparency { } pub(super) fn apply_mark( - _db: &dyn ExpandDatabase, - _ctxt: SyntaxContextData, - _file_id: HirFileId, - _transparency: Transparency, + db: &dyn ExpandDatabase, + ctxt: SyntaxContextId, + call_id: MacroCallId, + transparency: Transparency, ) -> SyntaxContextId { - _db.intern_syntax_context(_ctxt) -} - -// pub(super) fn with_ctxt_from_mark(db: &ExpandDatabase, file_id: HirFileId) { -// self.with_ctxt_from_mark(expn_id, Transparency::Transparent) -// } -// pub(super) fn with_call_site_ctxt(db: &ExpandDatabase, file_id: HirFileId) { -// self.with_ctxt_from_mark(expn_id, Transparency::Transparent) -// } - -#[derive(Clone, Debug)] -pub struct Hygiene {} - -impl Hygiene { - pub fn new(_: &dyn ExpandDatabase, _: HirFileId) -> Hygiene { - Hygiene {} + if transparency == Transparency::Opaque { + return apply_mark_internal(db, ctxt, Some(call_id), transparency); } - pub fn new_unhygienic() -> Hygiene { - Hygiene {} - } + let call_site_ctxt = db.lookup_intern_macro_call(call_id).call_site; + let mut call_site_ctxt = if transparency == Transparency::SemiTransparent { + call_site_ctxt.normalize_to_macros_2_0(db) + } else { + call_site_ctxt.normalize_to_macro_rules(db) + }; - // FIXME: this should just return name - pub fn name_ref_to_name( - &self, - _: &dyn ExpandDatabase, - name_ref: ast::NameRef, - ) -> Either { - Either::Left(name_ref.as_name()) + if call_site_ctxt.is_root(db) { + return apply_mark_internal(db, ctxt, Some(call_id), transparency); } - pub fn local_inner_macros(&self, _: &dyn ExpandDatabase, _: ast::Path) -> Option { - None + // Otherwise, `expn_id` is a macros 1.0 definition and the call site is in a + // macros 2.0 expansion, i.e., a macros 1.0 invocation is in a macros 2.0 definition. + // + // In this case, the tokens from the macros 1.0 definition inherit the hygiene + // at their invocation. That is, we pretend that the macros 1.0 definition + // was defined at its invocation (i.e., inside the macros 2.0 definition) + // so that the macros 2.0 definition remains hygienic. + // + // See the example at `test/ui/hygiene/legacy_interaction.rs`. + for (call_id, transparency) in ctxt.marks(db) { + call_site_ctxt = apply_mark_internal(db, call_site_ctxt, call_id, transparency); } + apply_mark_internal(db, call_site_ctxt, Some(call_id), transparency) } -#[derive(Clone, Debug)] -struct HygieneFrames(Arc); +fn apply_mark_internal( + db: &dyn ExpandDatabase, + ctxt: SyntaxContextId, + call_id: Option, + transparency: Transparency, +) -> SyntaxContextId { + let syntax_context_data = db.lookup_intern_syntax_context(ctxt); + let mut opaque = syntax_context_data.opaque; + let mut opaque_and_semitransparent = syntax_context_data.opaque_and_semitransparent; + + if transparency >= Transparency::Opaque { + let parent = opaque; + let new_opaque = SyntaxContextId::SELF_REF; + // But we can't just grab the to be allocated ID either as that would not deduplicate + // things! + // So we need a new salsa store type here ... + opaque = db.intern_syntax_context(SyntaxContextData { + outer_expn: call_id, + outer_transparency: transparency, + parent, + opaque: new_opaque, + opaque_and_semitransparent: new_opaque, + }); + } -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct HygieneFrame {} + if transparency >= Transparency::SemiTransparent { + let parent = opaque_and_semitransparent; + let new_opaque_and_semitransparent = SyntaxContextId::SELF_REF; + opaque_and_semitransparent = db.intern_syntax_context(SyntaxContextData { + outer_expn: call_id, + outer_transparency: transparency, + parent, + opaque, + opaque_and_semitransparent: new_opaque_and_semitransparent, + }); + } -#[derive(Debug, Clone, PartialEq, Eq)] -struct HygieneInfo {} + let parent = ctxt; + db.intern_syntax_context(SyntaxContextData { + outer_expn: call_id, + outer_transparency: transparency, + parent, + opaque, + opaque_and_semitransparent, + }) +} +pub trait SyntaxContextExt { + fn is_root(self, db: &dyn ExpandDatabase) -> bool; + fn normalize_to_macro_rules(self, db: &dyn ExpandDatabase) -> Self; + fn normalize_to_macros_2_0(self, db: &dyn ExpandDatabase) -> Self; + fn parent_ctxt(self, db: &dyn ExpandDatabase) -> Self; + fn outer_mark(self, db: &dyn ExpandDatabase) -> (Option, Transparency); + fn marks(self, db: &dyn ExpandDatabase) -> Vec<(Option, Transparency)>; +} -impl HygieneInfo { - fn _map_ident_up(&self, _: &dyn ExpandDatabase, _: TextRange) -> Option> { - None +#[inline(always)] +fn handle_self_ref(p: SyntaxContextId, n: SyntaxContextId) -> SyntaxContextId { + match n { + SyntaxContextId::SELF_REF => p, + _ => n, } } -impl HygieneFrame { - pub(crate) fn new(_: &dyn ExpandDatabase, _: HirFileId) -> HygieneFrame { - HygieneFrame {} +impl SyntaxContextExt for SyntaxContextId { + fn is_root(self, db: &dyn ExpandDatabase) -> bool { + db.lookup_intern_syntax_context(self).outer_expn.is_none() + } + fn normalize_to_macro_rules(self, db: &dyn ExpandDatabase) -> Self { + handle_self_ref(self, db.lookup_intern_syntax_context(self).opaque_and_semitransparent) + } + fn normalize_to_macros_2_0(self, db: &dyn ExpandDatabase) -> Self { + handle_self_ref(self, db.lookup_intern_syntax_context(self).opaque) + } + fn parent_ctxt(self, db: &dyn ExpandDatabase) -> Self { + db.lookup_intern_syntax_context(self).parent + } + fn outer_mark(self, db: &dyn ExpandDatabase) -> (Option, Transparency) { + let data = db.lookup_intern_syntax_context(self); + (data.outer_expn, data.outer_transparency) + } + fn marks(mut self, db: &dyn ExpandDatabase) -> Vec<(Option, Transparency)> { + let mut marks = Vec::new(); + while self != SyntaxContextId::ROOT { + marks.push(self.outer_mark(db)); + self = self.parent_ctxt(db); + } + marks.reverse(); + marks } } + +// pub(super) fn with_ctxt_from_mark(db: &ExpandDatabase, file_id: HirFileId) { +// self.with_ctxt_from_mark(expn_id, Transparency::Transparent) +// } +// pub(super) fn with_call_site_ctxt(db: &ExpandDatabase, file_id: HirFileId) { +// self.with_ctxt_from_mark(expn_id, Transparency::Transparent) +// } diff --git a/crates/hir-expand/src/lib.rs b/crates/hir-expand/src/lib.rs index ae07cf4b151b..6864f477ae8d 100644 --- a/crates/hir-expand/src/lib.rs +++ b/crates/hir-expand/src/lib.rs @@ -24,7 +24,10 @@ use triomphe::Arc; use std::{fmt, hash::Hash, iter}; -use base_db::{span::HirFileIdRepr, CrateId, FileId, FileRange, ProcMacroKind}; +use base_db::{ + span::{HirFileIdRepr, SyntaxContextId}, + CrateId, FileId, FileRange, ProcMacroKind, +}; use either::Either; use syntax::{ algo::{self, skip_trivia_token}, @@ -105,6 +108,7 @@ pub struct MacroCallLoc { /// for the eager input macro file. eager: Option>, pub kind: MacroCallKind, + pub call_site: SyntaxContextId, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -330,8 +334,9 @@ impl MacroDefId { db: &dyn db::ExpandDatabase, krate: CrateId, kind: MacroCallKind, + call_site: SyntaxContextId, ) -> MacroCallId { - db.intern_macro_call(MacroCallLoc { def: self, krate, eager: None, kind }) + db.intern_macro_call(MacroCallLoc { def: self, krate, eager: None, kind, call_site }) } pub fn ast_id(&self) -> Either, AstId> { diff --git a/crates/hir-expand/src/mod_path.rs b/crates/hir-expand/src/mod_path.rs index 69aa09c4a521..9396b08b2800 100644 --- a/crates/hir-expand/src/mod_path.rs +++ b/crates/hir-expand/src/mod_path.rs @@ -7,11 +7,11 @@ use std::{ use crate::{ db::ExpandDatabase, - hygiene::Hygiene, - name::{known, Name}, + hygiene::{SyntaxContextExt, Transparency}, + name::{known, AsName, Name}, + SpanMap, }; -use base_db::CrateId; -use either::Either; +use base_db::{span::SyntaxContextId, CrateId}; use smallvec::SmallVec; use syntax::{ast, AstNode}; @@ -38,6 +38,7 @@ pub enum PathKind { Crate, /// Absolute path (::foo) Abs, + // FIXME: Remove this /// `$crate` from macro expansion DollarCrate(CrateId), } @@ -46,7 +47,7 @@ impl ModPath { pub fn from_src( db: &dyn ExpandDatabase, path: ast::Path, - hygiene: &Hygiene, + hygiene: &SpanMap, ) -> Option { convert_path(db, None, path, hygiene) } @@ -193,7 +194,7 @@ fn convert_path( db: &dyn ExpandDatabase, prefix: Option, path: ast::Path, - hygiene: &Hygiene, + hygiene: &SpanMap, ) -> Option { let prefix = match path.qualifier() { Some(qual) => Some(convert_path(db, prefix, qual, hygiene)?), @@ -203,23 +204,26 @@ fn convert_path( let segment = path.segment()?; let mut mod_path = match segment.kind()? { ast::PathSegmentKind::Name(name_ref) => { - match hygiene.name_ref_to_name(db, name_ref) { - Either::Left(name) => { - // no type args in use - let mut res = prefix.unwrap_or_else(|| { - ModPath::from_kind( - segment.coloncolon_token().map_or(PathKind::Plain, |_| PathKind::Abs), - ) - }); - res.segments.push(name); - res - } - Either::Right(crate_id) => { - return Some(ModPath::from_segments( - PathKind::DollarCrate(crate_id), - iter::empty(), - )) + if name_ref.text() == "$crate" { + if prefix.is_some() { + return None; } + resolve_crate_root( + db, + hygiene + .span_for_range(name_ref.syntax().text_range()) + .map_or(SyntaxContextId::ROOT, |s| s.ctx), + ) + .map(PathKind::DollarCrate) + .map(ModPath::from_kind)? + } else { + let mut res = prefix.unwrap_or_else(|| { + ModPath::from_kind( + segment.coloncolon_token().map_or(PathKind::Plain, |_| PathKind::Abs), + ) + }); + res.segments.push(name_ref.as_name()); + res } } ast::PathSegmentKind::SelfTypeKw => { @@ -261,8 +265,15 @@ fn convert_path( // We follow what it did anyway :) if mod_path.segments.len() == 1 && mod_path.kind == PathKind::Plain { if let Some(_macro_call) = path.syntax().parent().and_then(ast::MacroCall::cast) { - if let Some(crate_id) = hygiene.local_inner_macros(db, path) { - mod_path.kind = PathKind::DollarCrate(crate_id); + let syn_ctx = hygiene + .span_for_range(segment.syntax().text_range()) + .map_or(SyntaxContextId::ROOT, |s| s.ctx); + if let Some(macro_call_id) = db.lookup_intern_syntax_context(syn_ctx).outer_expn { + if db.lookup_intern_macro_call(macro_call_id).def.local_inner { + if let Some(crate_root) = resolve_crate_root(db, syn_ctx) { + mod_path.kind = PathKind::DollarCrate(crate_root); + } + } } } } @@ -270,6 +281,40 @@ fn convert_path( Some(mod_path) } +pub fn resolve_crate_root(db: &dyn ExpandDatabase, mut ctxt: SyntaxContextId) -> Option { + // When resolving `$crate` from a `macro_rules!` invoked in a `macro`, + // we don't want to pretend that the `macro_rules!` definition is in the `macro` + // as described in `SyntaxContext::apply_mark`, so we ignore prepended opaque marks. + // FIXME: This is only a guess and it doesn't work correctly for `macro_rules!` + // definitions actually produced by `macro` and `macro` definitions produced by + // `macro_rules!`, but at least such configurations are not stable yet. + ctxt = ctxt.normalize_to_macro_rules(db); + let mut iter = ctxt.marks(db).into_iter().rev().peekable(); + let mut result_mark = None; + // Find the last opaque mark from the end if it exists. + while let Some(&(mark, transparency)) = iter.peek() { + if transparency == Transparency::Opaque { + result_mark = Some(mark); + iter.next(); + } else { + break; + } + } + // Then find the last semi-transparent mark from the end if it exists. + for (mark, transparency) in iter { + if transparency == Transparency::SemiTransparent { + result_mark = Some(mark); + } else { + break; + } + } + + match result_mark { + Some(Some(call)) => Some(db.lookup_intern_macro_call(call.into()).def.krate), + Some(None) | None => None, + } +} + pub use crate::name as __name; #[macro_export] diff --git a/crates/hir-expand/src/name.rs b/crates/hir-expand/src/name.rs index a876f48bda4e..a321f94cd755 100644 --- a/crates/hir-expand/src/name.rs +++ b/crates/hir-expand/src/name.rs @@ -470,6 +470,7 @@ pub mod known { pub const SELF_TYPE: super::Name = super::Name::new_inline("Self"); pub const STATIC_LIFETIME: super::Name = super::Name::new_inline("'static"); + pub const DOLLAR_CRATE: super::Name = super::Name::new_inline("$crate"); #[macro_export] macro_rules! name { diff --git a/crates/hir-ty/src/display.rs b/crates/hir-ty/src/display.rs index 9ccf467358ec..0712fc1c50c7 100644 --- a/crates/hir-ty/src/display.rs +++ b/crates/hir-ty/src/display.rs @@ -23,7 +23,7 @@ use hir_def::{ EnumVariantId, HasModule, ItemContainerId, LocalFieldId, Lookup, ModuleDefId, ModuleId, TraitId, }; -use hir_expand::{hygiene::Hygiene, name::Name}; +use hir_expand::name::Name; use intern::{Internable, Interned}; use itertools::Itertools; use la_arena::ArenaMap; @@ -1732,11 +1732,11 @@ impl HirDisplay for TypeRef { f.write_joined(bounds, " + ")?; } TypeRef::Macro(macro_call) => { - let macro_call = macro_call.to_node(f.db.upcast()); let ctx = hir_def::lower::LowerCtx::with_hygiene( f.db.upcast(), - &Hygiene::new_unhygienic(), + f.db.span_map(macro_call.file_id), ); + let macro_call = macro_call.to_node(f.db.upcast()); match macro_call.path() { Some(path) => match Path::from_src(path, &ctx) { Some(path) => path.hir_fmt(f)?, diff --git a/crates/hir-ty/src/test_db.rs b/crates/hir-ty/src/test_db.rs index 7d19e0a19169..a3383b2b5dfb 100644 --- a/crates/hir-ty/src/test_db.rs +++ b/crates/hir-ty/src/test_db.rs @@ -7,7 +7,7 @@ use base_db::{ AnchoredPath, CrateId, FileId, FileLoader, FileLoaderDelegate, SourceDatabase, Upcast, }; use hir_def::{db::DefDatabase, ModuleId}; -use hir_expand::db::ExpandDatabase; +use hir_expand::{db::ExpandDatabase, hygiene::SyntaxContextData}; use nohash_hasher::IntMap; use rustc_hash::FxHashSet; use syntax::TextRange; @@ -30,6 +30,7 @@ pub(crate) struct TestDB { impl Default for TestDB { fn default() -> Self { let mut this = Self { storage: Default::default(), events: Default::default() }; + this.intern_syntax_context(SyntaxContextData::root()); this.set_expand_proc_attr_macros_with_durability(true, Durability::HIGH); this } diff --git a/crates/hir/src/attrs.rs b/crates/hir/src/attrs.rs index 796490abd7f2..bb02620208bf 100644 --- a/crates/hir/src/attrs.rs +++ b/crates/hir/src/attrs.rs @@ -8,7 +8,7 @@ use hir_def::{ resolver::{HasResolver, Resolver, TypeNs}, AssocItemId, AttrDefId, ModuleDefId, }; -use hir_expand::{hygiene::Hygiene, name::Name}; +use hir_expand::name::Name; use hir_ty::db::HirDatabase; use syntax::{ast, AstNode}; @@ -234,7 +234,7 @@ fn modpath_from_str(db: &dyn HirDatabase, link: &str) -> Option { if ast_path.syntax().text() != link { return None; } - ModPath::from_src(db.upcast(), ast_path, &Hygiene::new_unhygienic()) + ModPath::from_src(db.upcast(), ast_path, &Default::default()) }; let full = try_get_modpath(link); diff --git a/crates/hir/src/db.rs b/crates/hir/src/db.rs index f4129e736ee1..cc3e869aa7ce 100644 --- a/crates/hir/src/db.rs +++ b/crates/hir/src/db.rs @@ -6,7 +6,7 @@ pub use hir_def::db::*; pub use hir_expand::db::{ AstIdMapQuery, DeclMacroExpanderQuery, ExpandDatabase, ExpandDatabaseStorage, - ExpandProcMacroQuery, HygieneFrameQuery, InternMacroCallQuery, MacroArgQuery, MacroExpandQuery, + ExpandProcMacroQuery, InternMacroCallQuery, MacroArgQuery, MacroExpandQuery, ParseMacroExpansionErrorQuery, ParseMacroExpansionQuery, }; pub use hir_ty::db::*; diff --git a/crates/hir/src/lib.rs b/crates/hir/src/lib.rs index 50d88b4cf838..b224b4b3a97c 100644 --- a/crates/hir/src/lib.rs +++ b/crates/hir/src/lib.rs @@ -137,10 +137,7 @@ pub use { // These are negative re-exports: pub using these names is forbidden, they // should remain private to hir internals. #[allow(unused)] -use { - hir_def::path::Path, - hir_expand::{hygiene::Hygiene, name::AsName}, -}; +use {hir_def::path::Path, hir_expand::name::AsName}; /// hir::Crate describes a single crate. It's the main interface with which /// a crate's dependencies interact. Mostly, it should be just a proxy for the diff --git a/crates/hir/src/semantics.rs b/crates/hir/src/semantics.rs index 758e6118aa12..825d68cdbaed 100644 --- a/crates/hir/src/semantics.rs +++ b/crates/hir/src/semantics.rs @@ -842,8 +842,8 @@ impl<'db> SemanticsImpl<'db> { pub fn resolve_trait(&self, path: &ast::Path) -> Option { let analyze = self.analyze(path.syntax())?; - let hygiene = hir_expand::hygiene::Hygiene::new(self.db.upcast(), analyze.file_id); - let ctx = LowerCtx::with_hygiene(self.db.upcast(), &hygiene); + let hygiene = self.db.span_map(analyze.file_id); + let ctx = LowerCtx::with_hygiene(self.db.upcast(), hygiene); let hir_path = Path::from_src(path.clone(), &ctx)?; match analyze.resolver.resolve_path_in_type_ns_fully(self.db.upcast(), &hir_path)? { TypeNs::TraitId(id) => Some(Trait { id }), diff --git a/crates/hir/src/source_analyzer.rs b/crates/hir/src/source_analyzer.rs index 8fc7f2c05d74..aaad82e12876 100644 --- a/crates/hir/src/source_analyzer.rs +++ b/crates/hir/src/source_analyzer.rs @@ -26,7 +26,6 @@ use hir_def::{ }; use hir_expand::{ builtin_fn_macro::BuiltinFnLikeExpander, - hygiene::Hygiene, mod_path::path, name, name::{AsName, Name}, @@ -596,8 +595,7 @@ impl SourceAnalyzer { } // This must be a normal source file rather than macro file. - let hygiene = Hygiene::new(db.upcast(), self.file_id); - let ctx = LowerCtx::with_hygiene(db.upcast(), &hygiene); + let ctx = LowerCtx::with_hygiene(db.upcast(), db.span_map(self.file_id)); let hir_path = Path::from_src(path.clone(), &ctx)?; // Case where path is a qualifier of a use tree, e.g. foo::bar::{Baz, Qux} where we are diff --git a/crates/ide-db/src/apply_change.rs b/crates/ide-db/src/apply_change.rs index 65eaa6510f0e..dc77424c1314 100644 --- a/crates/ide-db/src/apply_change.rs +++ b/crates/ide-db/src/apply_change.rs @@ -103,7 +103,6 @@ impl RootDatabase { hir::db::DeclMacroExpanderQuery hir::db::MacroExpandQuery hir::db::ExpandProcMacroQuery - hir::db::HygieneFrameQuery // DefDatabase hir::db::FileItemTreeQuery diff --git a/crates/ide-db/src/lib.rs b/crates/ide-db/src/lib.rs index 258d893b47da..957c9ad26ca2 100644 --- a/crates/ide-db/src/lib.rs +++ b/crates/ide-db/src/lib.rs @@ -208,7 +208,6 @@ impl RootDatabase { hir_db::DeclMacroExpanderQuery // hir_db::MacroExpandQuery hir_db::ExpandProcMacroQuery - hir_db::HygieneFrameQuery hir_db::ParseMacroExpansionErrorQuery // DefDatabase diff --git a/crates/mbe/src/benchmark.rs b/crates/mbe/src/benchmark.rs index 2ead71645639..eb0773570b32 100644 --- a/crates/mbe/src/benchmark.rs +++ b/crates/mbe/src/benchmark.rs @@ -53,7 +53,7 @@ fn benchmark_expand_macro_rules() { invocations .into_iter() .map(|(id, tt)| { - let res = rules[&id].expand(&tt); + let res = rules[&id].expand(&tt, |_| ()); assert!(res.err.is_none()); res.value.token_trees.len() }) @@ -124,7 +124,7 @@ fn invocation_fixtures( for op in rule.lhs.iter() { collect_from_op(op, &mut subtree, &mut seed); } - if it.expand(&subtree).err.is_none() { + if it.expand(&subtree, |_| ()).err.is_none() { res.push((name.clone(), subtree)); break; } diff --git a/crates/mbe/src/expander.rs b/crates/mbe/src/expander.rs index fac2b3375810..487e8b35980d 100644 --- a/crates/mbe/src/expander.rs +++ b/crates/mbe/src/expander.rs @@ -14,6 +14,7 @@ use crate::{parser::MetaVarKind, ExpandError, ExpandResult}; pub(crate) fn expand_rules( rules: &[crate::Rule], input: &tt::Subtree, + marker: impl Fn(&mut S) + Copy, is_2021: bool, ) -> ExpandResult> { let mut match_: Option<(matcher::Match, &crate::Rule)> = None; @@ -25,7 +26,7 @@ pub(crate) fn expand_rules( // Unconditionally returning the transcription here makes the // `test_repeat_bad_var` test fail. let ExpandResult { value, err: transcribe_err } = - transcriber::transcribe(&rule.rhs, &new_match.bindings); + transcriber::transcribe(&rule.rhs, &new_match.bindings, marker); if transcribe_err.is_none() { return ExpandResult::ok(value); } @@ -44,7 +45,7 @@ pub(crate) fn expand_rules( if let Some((match_, rule)) = match_ { // if we got here, there was no match without errors let ExpandResult { value, err: transcribe_err } = - transcriber::transcribe(&rule.rhs, &match_.bindings); + transcriber::transcribe(&rule.rhs, &match_.bindings, marker); ExpandResult { value, err: match_.err.or(transcribe_err) } } else { ExpandResult::new( @@ -129,7 +130,7 @@ enum Fragment { /// At one point in time, we tried to use "fake" delimiters here à la /// proc-macro delimiter=none. As we later discovered, "none" delimiters are /// tricky to handle in the parser, and rustc doesn't handle those either. - Expr(tt::TokenTree), + Expr(tt::Subtree), /// There are roughly two types of paths: paths in expression context, where a /// separator `::` between an identifier and its following generic argument list /// is mandatory, and paths in type context, where `::` can be omitted. @@ -139,5 +140,5 @@ enum Fragment { /// and is trasncribed as an expression-context path, verbatim transcription /// would cause a syntax error. We need to fix it up just before transcribing; /// see `transcriber::fix_up_and_push_path_tt()`. - Path(tt::TokenTree), + Path(tt::Subtree), } diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs index 796c9f2eb321..14b32599091e 100644 --- a/crates/mbe/src/expander/matcher.rs +++ b/crates/mbe/src/expander/matcher.rs @@ -765,7 +765,7 @@ fn match_meta_var( MetaVarKind::Path => { return input .expect_fragment(parser::PrefixEntryPoint::Path) - .map(|it| it.map(Fragment::Path)); + .map(|it| it.map(tt::TokenTree::subtree_or_wrap).map(Fragment::Path)); } MetaVarKind::Ty => parser::PrefixEntryPoint::Ty, MetaVarKind::Pat if is_2021 => parser::PrefixEntryPoint::PatTop, @@ -793,7 +793,7 @@ fn match_meta_var( }; return input .expect_fragment(parser::PrefixEntryPoint::Expr) - .map(|tt| tt.map(Fragment::Expr)); + .map(|tt| tt.map(tt::TokenTree::subtree_or_wrap).map(Fragment::Expr)); } MetaVarKind::Ident | MetaVarKind::Tt | MetaVarKind::Lifetime | MetaVarKind::Literal => { let tt_result = match kind { diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs index 4f5cd0480c78..c8b326fa6c53 100644 --- a/crates/mbe/src/expander/transcriber.rs +++ b/crates/mbe/src/expander/transcriber.rs @@ -11,21 +11,19 @@ use crate::{ }; impl Bindings { - fn contains(&self, name: &str) -> bool { - self.inner.contains_key(name) - } - fn get(&self, name: &str) -> Result<&Binding, ExpandError> { match self.inner.get(name) { Some(binding) => Ok(binding), - None => Err(ExpandError::binding_error(format!("could not find binding `{name}`"))), + None => Err(ExpandError::UnresolvedBinding(Box::new(Box::from(name)))), } } fn get_fragment( &self, name: &str, + mut span: S, nesting: &mut [NestingState], + marker: impl Fn(&mut S), ) -> Result, ExpandError> { macro_rules! binding_err { ($($arg:tt)*) => { ExpandError::binding_error(format!($($arg)*)) }; @@ -48,54 +46,75 @@ impl Bindings { }; } match b { - Binding::Fragment(it) => Ok(it.clone()), - // emit some reasonable default expansion for missing bindings, - // this gives better recovery than emitting the `$fragment-name` verbatim - Binding::Missing(it) => Ok(match it { - MetaVarKind::Stmt => { - Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { - span: S::DUMMY, - char: ';', - spacing: tt::Spacing::Alone, - }))) - } - MetaVarKind::Block => Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { + Binding::Fragment(f @ (Fragment::Path(sub) | Fragment::Expr(sub))) => { + let tt::Subtree { delimiter, token_trees } = sub; + marker(&mut span); + let subtree = tt::Subtree { delimiter: tt::Delimiter { - open: S::DUMMY, - close: S::DUMMY, - kind: tt::DelimiterKind::Brace, + // TODO split span + open: span, + close: span, + kind: delimiter.kind, }, - token_trees: vec![], - })), - // FIXME: Meta and Item should get proper defaults - MetaVarKind::Meta | MetaVarKind::Item | MetaVarKind::Tt | MetaVarKind::Vis => { - Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { - delimiter: tt::Delimiter::UNSPECIFIED, + token_trees: token_trees.clone(), + }; + Ok(match f { + Fragment::Tokens(_) => unreachable!(), + Fragment::Expr(_) => Fragment::Expr, + Fragment::Path(_) => Fragment::Path, + }(subtree)) + } + Binding::Fragment(it @ Fragment::Tokens(_)) => Ok(it.clone()), + // emit some reasonable default expansion for missing bindings, + // this gives better recovery than emitting the `$fragment-name` verbatim + Binding::Missing(it) => Ok({ + marker(&mut span); + match it { + MetaVarKind::Stmt => { + Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { + span, + char: ';', + spacing: tt::Spacing::Alone, + }))) + } + MetaVarKind::Block => Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { + delimiter: tt::Delimiter { + open: S::DUMMY, + close: S::DUMMY, + kind: tt::DelimiterKind::Brace, + }, token_trees: vec![], - })) - } - MetaVarKind::Path - | MetaVarKind::Ty - | MetaVarKind::Pat - | MetaVarKind::PatParam - | MetaVarKind::Expr - | MetaVarKind::Ident => { - Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - text: SmolStr::new_inline("missing"), - span: S::DUMMY, - }))) - } - MetaVarKind::Lifetime => { - Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - text: SmolStr::new_inline("'missing"), - span: S::DUMMY, - }))) - } - MetaVarKind::Literal => { - Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - text: SmolStr::new_inline("\"missing\""), - span: S::DUMMY, - }))) + })), + // FIXME: Meta and Item should get proper defaults + MetaVarKind::Meta | MetaVarKind::Item | MetaVarKind::Tt | MetaVarKind::Vis => { + Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { + delimiter: tt::Delimiter::UNSPECIFIED, + token_trees: vec![], + })) + } + MetaVarKind::Path + | MetaVarKind::Ty + | MetaVarKind::Pat + | MetaVarKind::PatParam + | MetaVarKind::Expr + | MetaVarKind::Ident => { + Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + text: SmolStr::new_inline("missing"), + span, + }))) + } + MetaVarKind::Lifetime => { + Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + text: SmolStr::new_inline("'missing"), + span, + }))) + } + MetaVarKind::Literal => { + Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + text: SmolStr::new_inline("\"missing\""), + span, + }))) + } } }), Binding::Nested(_) => { @@ -111,10 +130,11 @@ impl Bindings { pub(super) fn transcribe( template: &MetaTemplate, bindings: &Bindings, + marker: impl Fn(&mut S) + Copy, ) -> ExpandResult> { let mut ctx = ExpandCtx { bindings, nesting: Vec::new() }; let mut arena: Vec> = Vec::new(); - expand_subtree(&mut ctx, template, None, &mut arena) + expand_subtree(&mut ctx, template, None, &mut arena, marker) } #[derive(Debug)] @@ -139,40 +159,65 @@ fn expand_subtree( template: &MetaTemplate, delimiter: Option>, arena: &mut Vec>, + marker: impl Fn(&mut S) + Copy, ) -> ExpandResult> { // remember how many elements are in the arena now - when returning, we want to drain exactly how many elements we added. This way, the recursive uses of the arena get their own "view" of the arena, but will reuse the allocation let start_elements = arena.len(); let mut err = None; 'ops: for op in template.iter() { match op { - Op::Literal(it) => arena.push(tt::Leaf::from(it.clone()).into()), - Op::Ident(it) => arena.push(tt::Leaf::from(it.clone()).into()), + Op::Literal(it) => arena.push( + tt::Leaf::from({ + let mut it = it.clone(); + marker(&mut it.span); + it + }) + .into(), + ), + Op::Ident(it) => arena.push( + tt::Leaf::from({ + let mut it = it.clone(); + marker(&mut it.span); + it + }) + .into(), + ), Op::Punct(puncts) => { for punct in puncts { - arena.push(tt::Leaf::from(*punct).into()); + arena.push( + tt::Leaf::from({ + let mut it = punct.clone(); + marker(&mut it.span); + it + }) + .into(), + ); } } Op::Subtree { tokens, delimiter } => { + let mut delimiter = *delimiter; + marker(&mut delimiter.open); + marker(&mut delimiter.close); let ExpandResult { value: tt, err: e } = - expand_subtree(ctx, tokens, Some(*delimiter), arena); + expand_subtree(ctx, tokens, Some(delimiter), arena, marker); err = err.or(e); arena.push(tt.into()); } Op::Var { name, id, .. } => { - let ExpandResult { value: fragment, err: e } = expand_var(ctx, name, *id); + let ExpandResult { value: fragment, err: e } = expand_var(ctx, name, *id, marker); err = err.or(e); push_fragment(arena, fragment); } Op::Repeat { tokens: subtree, kind, separator } => { let ExpandResult { value: fragment, err: e } = - expand_repeat(ctx, subtree, *kind, separator, arena); + expand_repeat(ctx, subtree, *kind, separator, arena, marker); err = err.or(e); push_fragment(arena, fragment) } Op::Ignore { name, id } => { // Expand the variable, but ignore the result. This registers the repetition count. // FIXME: Any emitted errors are dropped. - expand_var(ctx, name, *id); + expand_var(ctx, name, *id, marker); } Op::Index { depth } => { let index = @@ -258,42 +303,42 @@ fn expand_var( ctx: &mut ExpandCtx<'_, S>, v: &SmolStr, id: S, + marker: impl Fn(&mut S), ) -> ExpandResult> { // We already handle $crate case in mbe parser debug_assert!(v != "crate"); - if !ctx.bindings.contains(v) { - // Note that it is possible to have a `$var` inside a macro which is not bound. - // For example: - // ``` - // macro_rules! foo { - // ($a:ident, $b:ident, $c:tt) => { - // macro_rules! bar { - // ($bi:ident) => { - // fn $bi() -> u8 {$c} - // } - // } - // } - // ``` - // We just treat it a normal tokens - let tt = tt::Subtree { - delimiter: tt::Delimiter::UNSPECIFIED, - token_trees: vec![ - tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, span: id }) - .into(), - tt::Leaf::from(tt::Ident { text: v.clone(), span: id }).into(), - ], + match ctx.bindings.get_fragment(v, id, &mut ctx.nesting, marker) { + Ok(it) => ExpandResult::ok(it), + Err(ExpandError::UnresolvedBinding(_)) => { + // Note that it is possible to have a `$var` inside a macro which is not bound. + // For example: + // ``` + // macro_rules! foo { + // ($a:ident, $b:ident, $c:tt) => { + // macro_rules! bar { + // ($bi:ident) => { + // fn $bi() -> u8 {$c} + // } + // } + // } + // ``` + // We just treat it a normal tokens + let tt = tt::Subtree { + delimiter: tt::Delimiter::UNSPECIFIED, + token_trees: vec![ + tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, span: id }) + .into(), + tt::Leaf::from(tt::Ident { text: v.clone(), span: id }).into(), + ], + } + .into(); + ExpandResult::ok(Fragment::Tokens(tt)) } - .into(); - ExpandResult::ok(Fragment::Tokens(tt)) - } else { - ctx.bindings.get_fragment(v, &mut ctx.nesting).map_or_else( - |e| ExpandResult { - value: Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree::empty())), - err: Some(e), - }, - ExpandResult::ok, - ) + Err(e) => ExpandResult { + value: Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree::empty())), + err: Some(e), + }, } } @@ -303,6 +348,7 @@ fn expand_repeat( kind: RepeatKind, separator: &Option>, arena: &mut Vec>, + marker: impl Fn(&mut S) + Copy, ) -> ExpandResult> { let mut buf: Vec> = Vec::new(); ctx.nesting.push(NestingState { idx: 0, at_end: false, hit: false }); @@ -314,7 +360,8 @@ fn expand_repeat( let mut err = None; loop { - let ExpandResult { value: mut t, err: e } = expand_subtree(ctx, template, None, arena); + let ExpandResult { value: mut t, err: e } = + expand_subtree(ctx, template, None, arena, marker); let nesting_state = ctx.nesting.last_mut().unwrap(); if nesting_state.at_end || !nesting_state.hit { break; @@ -391,7 +438,7 @@ fn expand_repeat( fn push_fragment(buf: &mut Vec>, fragment: Fragment) { match fragment { Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt), - Fragment::Expr(tt::TokenTree::Subtree(mut tt)) => { + Fragment::Expr(mut tt) => { if tt.delimiter.kind == tt::DelimiterKind::Invisible { tt.delimiter = tt::Delimiter { open: S::DUMMY, @@ -401,8 +448,8 @@ fn push_fragment(buf: &mut Vec>, fragment: Fragment } buf.push(tt.into()) } - Fragment::Path(tt::TokenTree::Subtree(tt)) => fix_up_and_push_path_tt(buf, tt), - Fragment::Tokens(tt) | Fragment::Expr(tt) | Fragment::Path(tt) => buf.push(tt), + Fragment::Path(tt) => fix_up_and_push_path_tt(buf, tt), + Fragment::Tokens(tt) => buf.push(tt), } } diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index 43543479eb94..482d0157b2db 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -72,6 +72,7 @@ impl fmt::Display for ParseError { #[derive(Debug, PartialEq, Eq, Clone, Hash)] pub enum ExpandError { BindingError(Box>), + UnresolvedBinding(Box>), LeftoverTokens, ConversionError, LimitExceeded, @@ -94,6 +95,10 @@ impl fmt::Display for ExpandError { ExpandError::NoMatchingRule => f.write_str("no rule matches input tokens"), ExpandError::UnexpectedToken => f.write_str("unexpected token in input"), ExpandError::BindingError(e) => f.write_str(e), + ExpandError::UnresolvedBinding(binding) => { + f.write_str("could not find binding ")?; + f.write_str(binding) + } ExpandError::ConversionError => f.write_str("could not convert tokens"), ExpandError::LimitExceeded => f.write_str("Expand exceed limit"), ExpandError::LeftoverTokens => f.write_str("leftover tokens"), @@ -233,8 +238,12 @@ impl DeclarativeMacro { self.err.as_deref() } - pub fn expand(&self, tt: &tt::Subtree) -> ExpandResult> { - expander::expand_rules(&self.rules, &tt, self.is_2021) + pub fn expand( + &self, + tt: &tt::Subtree, + marker: impl Fn(&mut S) + Copy, + ) -> ExpandResult> { + expander::expand_rules(&self.rules, &tt, marker, self.is_2021) } } diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index ab272862cdb5..b843db510e39 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -155,10 +155,7 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec break, - Some(tt @ tt::TokenTree::Leaf(_)) => { - tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![tt] } - } - Some(tt::TokenTree::Subtree(tt)) => tt, + Some(tt) => tt.subtree_or_wrap(), }); let mut fork = iter.clone(); @@ -720,6 +717,7 @@ where /// Parses a float literal as if it was a one to two name ref nodes with a dot inbetween. /// This occurs when a float literal is used as a field access. fn float_split(&mut self, has_pseudo_dot: bool) { + // TODO: FIXME this breaks the hygiene map let (text, _span) = match self.cursor.token_tree() { Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Literal(lit), _)) => { (lit.text.as_str(), lit.span) diff --git a/crates/mbe/src/token_map.rs b/crates/mbe/src/token_map.rs index 1af50c8b3b95..c825bd01bcd6 100644 --- a/crates/mbe/src/token_map.rs +++ b/crates/mbe/src/token_map.rs @@ -20,11 +20,12 @@ pub struct TokenMap { // then a bin search on the ast id pub span_map: Vec<(TextRange, S)>, // span_map2: rustc_hash::FxHashMap, + pub real_file: bool, } impl Default for TokenMap { fn default() -> Self { - Self { span_map: Vec::new() } + Self { span_map: Vec::new(), real_file: true } } } @@ -49,8 +50,21 @@ impl TokenMap { ) } + // FIXME: Should be infallible pub fn span_for_range(&self, range: TextRange) -> Option { - self.span_map.iter().find_map(|(r, s)| if r == &range { Some(s.clone()) } else { None }) + // TODO FIXME: make this proper + self.span_map + .iter() + .filter_map(|(r, s)| Some((r, s, r.intersect(range)?))) + .max_by_key(|(_, _, intersection)| intersection.len()) + .map(|(_, &s, _)| s) + .or_else(|| { + if self.real_file { + None + } else { + panic!("no span for range {range:?} in {:#?}", self.span_map) + } + }) } // pub fn ranges_by_token( diff --git a/crates/tt/src/lib.rs b/crates/tt/src/lib.rs index a384af2a9a33..73e75a051bd6 100644 --- a/crates/tt/src/lib.rs +++ b/crates/tt/src/lib.rs @@ -56,6 +56,7 @@ pub trait SpanAnchor: std::fmt::Debug + Copy + Sized + Eq { const DUMMY: Self; } +// FIXME: Get rid of this trait? pub trait Span: std::fmt::Debug + Copy + Sized + Eq { const DUMMY: Self; } @@ -72,7 +73,16 @@ pub enum TokenTree { impl_from!(Leaf, Subtree for TokenTree); impl TokenTree { pub const fn empty() -> Self { - Self::Subtree(Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] }) + Self::Subtree(Subtree { delimiter: Delimiter::UNSPECIFIED, token_trees: vec![] }) + } + + pub fn subtree_or_wrap(self) -> Subtree { + match self { + TokenTree::Leaf(_) => { + Subtree { delimiter: Delimiter::UNSPECIFIED, token_trees: vec![self] } + } + TokenTree::Subtree(s) => s, + } } } From 8423893d1ce33659adc42e26850069a8e7087c30 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sun, 19 Nov 2023 12:27:22 +0100 Subject: [PATCH 06/24] More incremental tests --- .../hir-def/src/nameres/tests/incremental.rs | 226 ++++++++++++++---- 1 file changed, 179 insertions(+), 47 deletions(-) diff --git a/crates/hir-def/src/nameres/tests/incremental.rs b/crates/hir-def/src/nameres/tests/incremental.rs index 977745c476f1..3d7784bdd542 100644 --- a/crates/hir-def/src/nameres/tests/incremental.rs +++ b/crates/hir-def/src/nameres/tests/incremental.rs @@ -1,13 +1,18 @@ -use base_db::SourceDatabaseExt; +use base_db::{SourceDatabase, SourceDatabaseExt}; use triomphe::Arc; -use crate::{db::DefDatabase, AdtId, ModuleDefId}; - -use super::*; +use crate::{ + db::DefDatabase, + nameres::tests::{TestDB, WithFixture}, + AdtId, ModuleDefId, +}; fn check_def_map_is_not_recomputed(ra_fixture_initial: &str, ra_fixture_change: &str) { let (mut db, pos) = TestDB::with_position(ra_fixture_initial); - let krate = db.test_crate(); + let krate = { + let crate_graph = db.crate_graph(); + crate_graph.iter().last().unwrap() + }; { let events = db.log_executed(|| { db.crate_def_map(krate); @@ -28,39 +33,39 @@ fn check_def_map_is_not_recomputed(ra_fixture_initial: &str, ra_fixture_change: fn typing_inside_a_function_should_not_invalidate_def_map() { check_def_map_is_not_recomputed( r" - //- /lib.rs - mod foo;$0 +//- /lib.rs +mod foo;$0 - use crate::foo::bar::Baz; +use crate::foo::bar::Baz; - enum E { A, B } - use E::*; +enum E { A, B } +use E::*; - fn foo() -> i32 { - 1 + 1 - } +fn foo() -> i32 { + 1 + 1 +} - #[cfg(never)] - fn no() {} - //- /foo/mod.rs - pub mod bar; +#[cfg(never)] +fn no() {} +//- /foo/mod.rs +pub mod bar; - //- /foo/bar.rs - pub struct Baz; - ", +//- /foo/bar.rs +pub struct Baz; +", r" - mod foo; +mod foo; - use crate::foo::bar::Baz; +use crate::foo::bar::Baz; - enum E { A, B } - use E::*; +enum E { A, B } +use E::*; - fn foo() -> i32 { 92 } +fn foo() -> i32 { 92 } - #[cfg(never)] - fn no() {} - ", +#[cfg(never)] +fn no() {} +", ); } @@ -68,30 +73,157 @@ fn typing_inside_a_function_should_not_invalidate_def_map() { fn typing_inside_a_macro_should_not_invalidate_def_map() { check_def_map_is_not_recomputed( r" - //- /lib.rs - macro_rules! m { - ($ident:ident) => { - fn f() { - $ident + $ident; - }; - } - } - mod foo; +//- /lib.rs +macro_rules! m { + ($ident:ident) => { + fn f() { + $ident + $ident; + }; + } +} +mod foo; + +//- /foo/mod.rs +pub mod bar; + +//- /foo/bar.rs +$0 +m!(X); + +pub struct S {} +", + r" +m!(Y); + +pub struct S {} +", + ); +} - //- /foo/mod.rs - pub mod bar; +#[test] +fn typing_inside_an_attribute_should_not_invalidate_def_map() { + check_def_map_is_not_recomputed( + r" +//- proc_macros: identity +//- /lib.rs +mod foo; - //- /foo/bar.rs - $0 - m!(X); +//- /foo/mod.rs +pub mod bar; - pub struct S {} - ", +//- /foo/bar.rs +$0 +#[proc_macros::identity] +fn f() {} +", r" - m!(Y); +#[proc_macros::identity] +fn f() { foo } +", + ); +} + +#[test] +fn typing_inside_an_attribute_arg_should_not_invalidate_def_map() { + check_def_map_is_not_recomputed( + r" +//- proc_macros: identity +//- /lib.rs +mod foo; + +//- /foo/mod.rs +pub mod bar; - pub struct S {} - ", +//- /foo/bar.rs +$0 +#[proc_macros::identity] +fn f() {} +", + r" +#[proc_macros::identity(foo)] +fn f() {} +", + ); +} +#[test] +fn typing_inside_macro_heavy_file_should_not_invalidate_def_map() { + check_def_map_is_not_recomputed( + r" +//- proc_macros: identity, derive_identity +//- /lib.rs +macro_rules! m { + ($ident:ident) => { + fn fm() { + $ident + $ident; + }; + } +} +mod foo; + +//- /foo/mod.rs +pub mod bar; + +//- /foo/bar.rs +$0 +fn f() {} + +m!(X); +macro_rules! m2 { + ($ident:ident) => { + fn f2() { + $ident + $ident; + }; + } +} +m2!(X); + +#[proc_macros::identity] +#[derive(proc_macros::DeriveIdentity)] +pub struct S {} +", + r" +fn f() {0} + +m!(X); +macro_rules! m2 { + ($ident:ident) => { + fn f2() { + $ident + $ident; + }; + } +} +m2!(X); + +#[proc_macros::identity] +#[derive(proc_macros::DeriveIdentity)] +pub struct S {} +", + ); +} + +#[test] +fn typing_inside_a_derive_should_not_invalidate_def_map() { + check_def_map_is_not_recomputed( + r" +//- proc_macros: derive_identity +//- minicore:derive +//- /lib.rs +mod foo; + +//- /foo/mod.rs +pub mod bar; + +//- /foo/bar.rs +$0 +#[derive(proc_macros::DeriveIdentity)] +#[allow()] +struct S; +", + r" +#[derive(proc_macros::DeriveIdentity)] +#[allow(dead_code)] +struct S; +", ); } From 394d11b0fa2cb40c92f5cfa7e664d5fc425073aa Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sun, 19 Nov 2023 18:42:25 +0100 Subject: [PATCH 07/24] Fix float-split hack not setting up spans correctly --- crates/base-db/src/span.rs | 6 ++ crates/hir-def/src/lib.rs | 7 +-- .../hir-def/src/macro_expansion_tests/mbe.rs | 57 +++++++++++++----- .../hir-def/src/macro_expansion_tests/mod.rs | 30 +++++++--- .../src/macro_expansion_tests/proc_macros.rs | 59 +++++++++++++++++++ crates/hir-def/src/nameres/collector.rs | 13 ++-- crates/hir-def/src/nameres/tests.rs | 4 +- crates/hir-def/src/test_db.rs | 4 +- crates/hir-expand/src/attrs.rs | 30 ++++++++-- crates/hir-expand/src/db.rs | 8 ++- crates/hir-ty/src/test_db.rs | 4 +- crates/hir-ty/src/tests/macros.rs | 12 ++-- crates/ide-db/src/lib.rs | 1 + crates/mbe/src/syntax_bridge.rs | 12 +++- crates/mbe/src/token_map.rs | 8 +-- 15 files changed, 197 insertions(+), 58 deletions(-) diff --git a/crates/base-db/src/span.rs b/crates/base-db/src/span.rs index f430a36ddfb4..600aa079333e 100644 --- a/crates/base-db/src/span.rs +++ b/crates/base-db/src/span.rs @@ -16,6 +16,12 @@ pub type SpanData = tt::SpanData; pub struct SyntaxContextId(InternId); crate::impl_intern_key!(SyntaxContextId); +impl fmt::Display for SyntaxContextId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0.as_u32()) + } +} + impl SyntaxContext for SyntaxContextId { const DUMMY: Self = Self::ROOT; // veykril(HACK): salsa doesn't allow us fetching the id of the current input to be allocated so diff --git a/crates/hir-def/src/lib.rs b/crates/hir-def/src/lib.rs index 0da605819f9e..65f1dcc85019 100644 --- a/crates/hir-def/src/lib.rs +++ b/crates/hir-def/src/lib.rs @@ -1322,6 +1322,7 @@ fn derive_macro_as_call_id( item_attr: &AstIdWithPath, derive_attr_index: AttrId, derive_pos: u32, + call_site: SyntaxContextId, krate: CrateId, resolver: impl Fn(path::ModPath) -> Option<(MacroId, MacroDefId)>, ) -> Result<(MacroId, MacroDefId, MacroCallId), UnresolvedMacro> { @@ -1336,8 +1337,7 @@ fn derive_macro_as_call_id( derive_index: derive_pos, derive_attr_index, }, - //FIXME - SyntaxContextId::ROOT, + call_site, ); Ok((macro_id, def_id, call_id)) } @@ -1367,8 +1367,7 @@ fn attr_macro_as_call_id( attr_args: Arc::new(arg), invoc_attr_index: macro_attr.id, }, - //FIXME - SyntaxContextId::ROOT, + macro_attr.ctxt, ) } intern::impl_internable!( diff --git a/crates/hir-def/src/macro_expansion_tests/mbe.rs b/crates/hir-def/src/macro_expansion_tests/mbe.rs index af4b3e12b9f3..dcecec4e8e4e 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe.rs @@ -23,10 +23,12 @@ macro_rules! f { }; } -// +tokenids +// +spans f!(struct MyTraitMap2); "#, - // FIXME: #SpanAnchor(FileId(0), 1)@91..92 why is there whitespace annotated with a span here? + // FIXME: #SpanAnchor(FileId(0), 1)@91..92\2# why is there whitespace annotated with a span + // here? Presumably because the leading `::` is getting two spans instead of one? Sounds + // liek glueing might be failing here expect![[r#" macro_rules! f { ( struct $ident:ident ) => { @@ -36,9 +38,9 @@ macro_rules! f { }; } -struct#SpanAnchor(FileId(0), 1)@58..64 MyTraitMap2#SpanAnchor(FileId(0), 2)@23..34 {#SpanAnchor(FileId(0), 1)@72..73 - map#SpanAnchor(FileId(0), 1)@86..89:#SpanAnchor(FileId(0), 1)@89..90 #SpanAnchor(FileId(0), 1)@91..92::#SpanAnchor(FileId(0), 1)@92..93std#SpanAnchor(FileId(0), 1)@93..96::#SpanAnchor(FileId(0), 1)@97..98collections#SpanAnchor(FileId(0), 1)@98..109::#SpanAnchor(FileId(0), 1)@110..111HashSet#SpanAnchor(FileId(0), 1)@111..118<#SpanAnchor(FileId(0), 1)@118..119(#SpanAnchor(FileId(0), 1)@119..120)#SpanAnchor(FileId(0), 1)@120..121>#SpanAnchor(FileId(0), 1)@121..122,#SpanAnchor(FileId(0), 1)@122..123 -}#SpanAnchor(FileId(0), 1)@132..133 +struct#FileId(0):1@58..64\2# MyTraitMap2#FileId(0):2@20..31\0# {#FileId(0):1@72..73\2# + map#FileId(0):1@86..89\2#:#FileId(0):1@89..90\2# #FileId(0):1@91..92\2#::#FileId(0):1@92..93\2#std#FileId(0):1@93..96\2#::#FileId(0):1@97..98\2#collections#FileId(0):1@98..109\2#::#FileId(0):1@110..111\2#HashSet#FileId(0):1@111..118\2#<#FileId(0):1@118..119\2#(#FileId(0):1@119..120\2#)#FileId(0):1@120..121\2#>#FileId(0):1@121..122\2#,#FileId(0):1@122..123\2# +}#FileId(0):1@132..133\2# "#]], ); } @@ -49,18 +51,19 @@ fn token_mapping_floats() { // (and related issues) check( r#" -// +tokenids +// +spans macro_rules! f { ($($tt:tt)*) => { $($tt)* }; } -// +tokenids +// +spans f! { fn main() { 1; 1.0; + ((1,),).0.0; let x = 1; } } @@ -68,18 +71,19 @@ f! { "#, expect![[r#" -// +tokenids +// +spans macro_rules! f { ($($tt:tt)*) => { $($tt)* }; } -fn#SpanAnchor(FileId(0), 2)@22..24 main#SpanAnchor(FileId(0), 2)@25..29(#SpanAnchor(FileId(0), 2)@29..30)#SpanAnchor(FileId(0), 2)@30..31 {#SpanAnchor(FileId(0), 2)@32..33 - 1#SpanAnchor(FileId(0), 2)@42..43;#SpanAnchor(FileId(0), 2)@43..44 - 1.0#SpanAnchor(FileId(0), 2)@53..56;#SpanAnchor(FileId(0), 2)@56..57 - let#SpanAnchor(FileId(0), 2)@66..69 x#SpanAnchor(FileId(0), 2)@70..71 =#SpanAnchor(FileId(0), 2)@72..73 1#SpanAnchor(FileId(0), 2)@74..75;#SpanAnchor(FileId(0), 2)@75..76 -}#SpanAnchor(FileId(0), 2)@81..82 +fn#FileId(0):2@19..21\0# main#FileId(0):2@22..26\0#(#FileId(0):2@26..27\0#)#FileId(0):2@27..28\0# {#FileId(0):2@29..30\0# + 1#FileId(0):2@39..40\0#;#FileId(0):2@40..41\0# + 1.0#FileId(0):2@50..53\0#;#FileId(0):2@53..54\0# + (#FileId(0):2@63..64\0#(#FileId(0):2@64..65\0#1#FileId(0):2@65..66\0#,#FileId(0):2@66..67\0# )#FileId(0):2@67..68\0#,#FileId(0):2@68..69\0# )#FileId(0):2@69..70\0#.#FileId(0):2@70..71\0#0#FileId(0):2@71..74\0#.#FileId(0):2@71..74\0#0#FileId(0):2@71..74\0#;#FileId(0):2@74..75\0# + let#FileId(0):2@84..87\0# x#FileId(0):2@88..89\0# =#FileId(0):2@90..91\0# 1#FileId(0):2@92..93\0#;#FileId(0):2@93..94\0# +}#FileId(0):2@99..100\0# "#]], @@ -123,7 +127,7 @@ macro_rules! identity { } fn main(foo: ()) { - format_args/*+tokenids*/!("{} {} {}", format_args!("{}", 0), foo, identity!(10), "bar") + format_args/*+spans*/!("{} {} {}", format_args!("{}", 0), foo, identity!(10), "bar") } "#, @@ -137,13 +141,36 @@ macro_rules! identity { } fn main(foo: ()) { - builtin#SpanAnchor(FileId(0), 0)@0..0 ##SpanAnchor(FileId(0), 0)@0..0format_args#SpanAnchor(FileId(0), 0)@0..0 (#SpanAnchor(FileId(0), 6)@25..26"{} {} {}"#SpanAnchor(FileId(0), 6)@26..36,#SpanAnchor(FileId(0), 6)@36..37 format_args#SpanAnchor(FileId(0), 6)@38..49!#SpanAnchor(FileId(0), 6)@49..50(#SpanAnchor(FileId(0), 6)@50..51"{}"#SpanAnchor(FileId(0), 6)@51..55,#SpanAnchor(FileId(0), 6)@55..56 0#SpanAnchor(FileId(0), 6)@57..58)#SpanAnchor(FileId(0), 6)@58..59,#SpanAnchor(FileId(0), 6)@59..60 foo#SpanAnchor(FileId(0), 6)@61..64,#SpanAnchor(FileId(0), 6)@64..65 identity#SpanAnchor(FileId(0), 6)@66..74!#SpanAnchor(FileId(0), 6)@74..75(#SpanAnchor(FileId(0), 6)@75..7610#SpanAnchor(FileId(0), 6)@76..78)#SpanAnchor(FileId(0), 6)@78..79,#SpanAnchor(FileId(0), 6)@79..80 "bar"#SpanAnchor(FileId(0), 6)@81..86)#SpanAnchor(FileId(0), 6)@86..87 + builtin#FileId(0):0@0..0\0# ##FileId(0):0@0..0\0#format_args#FileId(0):0@0..0\0# (#FileId(0):6@22..23\0#"{} {} {}"#FileId(0):6@23..33\0#,#FileId(0):6@33..34\0# format_args#FileId(0):6@35..46\0#!#FileId(0):6@46..47\0#(#FileId(0):6@47..48\0#"{}"#FileId(0):6@48..52\0#,#FileId(0):6@52..53\0# 0#FileId(0):6@54..55\0#)#FileId(0):6@55..56\0#,#FileId(0):6@56..57\0# foo#FileId(0):6@58..61\0#,#FileId(0):6@61..62\0# identity#FileId(0):6@63..71\0#!#FileId(0):6@71..72\0#(#FileId(0):6@72..73\0#10#FileId(0):6@73..75\0#)#FileId(0):6@75..76\0#,#FileId(0):6@76..77\0# "bar"#FileId(0):6@78..83\0#)#FileId(0):6@83..84\0# } "##]], ); } +#[test] +fn token_mapping_across_files() { + check( + r#" +//- /lib.rs +#[macro_use] +mod foo; + +mk_struct/*+spans*/!(Foo with u32); +//- /foo.rs +macro_rules! mk_struct { + ($foo:ident with $ty:ty) => { struct $foo($ty); } +} +"#, + expect![[r#" +#[macro_use] +mod foo; + +struct#FileId(1):1@59..65\2# Foo#FileId(0):2@21..24\0#(#FileId(1):1@70..71\2#u32#FileId(0):2@30..33\0#)#FileId(1):1@74..75\2#;#FileId(1):1@75..76\2# +"#]], + ); +} + #[test] fn float_field_access_macro_input() { check( diff --git a/crates/hir-def/src/macro_expansion_tests/mod.rs b/crates/hir-def/src/macro_expansion_tests/mod.rs index d4902c52e748..f770d2832efb 100644 --- a/crates/hir-def/src/macro_expansion_tests/mod.rs +++ b/crates/hir-def/src/macro_expansion_tests/mod.rs @@ -103,11 +103,11 @@ pub fn identity_when_valid(_attr: TokenStream, item: TokenStream) -> TokenStream for (call, exp) in expansions.into_iter().rev() { let mut tree = false; let mut expect_errors = false; - let mut show_token_ids = false; + let mut show_spans = false; for comment in call.syntax().children_with_tokens().filter(|it| it.kind() == COMMENT) { tree |= comment.to_string().contains("+tree"); expect_errors |= comment.to_string().contains("+errors"); - show_token_ids |= comment.to_string().contains("+tokenids"); + show_spans |= comment.to_string().contains("+spans"); } let mut expn_text = String::new(); @@ -128,10 +128,8 @@ pub fn identity_when_valid(_attr: TokenStream, item: TokenStream) -> TokenStream parse.syntax_node(), ); } - let pp = pretty_print_macro_expansion( - parse.syntax_node(), - show_token_ids.then_some(&*token_map), - ); + let pp = + pretty_print_macro_expansion(parse.syntax_node(), show_spans.then_some(&*token_map)); let indent = IndentLevel::from_node(call.syntax()); let pp = reindent(indent, pp); format_to!(expn_text, "{}", pp); @@ -166,9 +164,18 @@ pub fn identity_when_valid(_attr: TokenStream, item: TokenStream) -> TokenStream } _ => None, }; + if let Some(src) = src { if src.file_id.is_attr_macro(&db) || src.file_id.is_custom_derive(&db) { - let pp = pretty_print_macro_expansion(src.value, None); + let call = src.file_id.call_node(&db).expect("macro file"); + let mut show_spans = false; + for comment in call.value.children_with_tokens().filter(|it| it.kind() == COMMENT) { + show_spans |= comment.to_string().contains("+spans"); + } + let pp = pretty_print_macro_expansion( + src.value, + show_spans.then_some(&db.span_map(src.file_id)), + ); format_to!(expanded_text, "\n{}", pp) } } @@ -250,7 +257,14 @@ fn pretty_print_macro_expansion(expn: SyntaxNode, map: Option<&SpanMap>) -> Stri format_to!(res, "{}", token); if let Some(map) = map { if let Some(span) = map.span_for_range(token.text_range()) { - format_to!(res, "#{:?}@{:?}", span.anchor, span.range); + format_to!( + res, + "#{:?}:{:?}@{:?}\\{}#", + span.anchor.file_id, + span.anchor.ast_id.into_raw(), + span.range, + span.ctx + ); } } } diff --git a/crates/hir-def/src/macro_expansion_tests/proc_macros.rs b/crates/hir-def/src/macro_expansion_tests/proc_macros.rs index 822bdcc122dc..29374945f647 100644 --- a/crates/hir-def/src/macro_expansion_tests/proc_macros.rs +++ b/crates/hir-def/src/macro_expansion_tests/proc_macros.rs @@ -93,6 +93,41 @@ fn foo() { ); } +#[test] +fn macro_rules_in_attr() { + // Regression test for https://github.com/rust-lang/rust-analyzer/issues/12211 + check( + r#" +//- proc_macros: identity +macro_rules! id { + ($($t:tt)*) => { + $($t)* + }; +} +id! { + #[proc_macros::identity] + impl Foo for WrapBj { + async fn foo(&self) { + self.id().await; + } + } +} +"#, + expect![[r#" +macro_rules! id { + ($($t:tt)*) => { + $($t)* + }; +} +#[proc_macros::identity] impl Foo for WrapBj { + async fn foo(&self ) { + self .id().await ; + } +} +"#]], + ); +} + #[test] fn float_parsing_panic() { // Regression test for https://github.com/rust-lang/rust-analyzer/issues/12211 @@ -127,3 +162,27 @@ macro_rules! id { "#]], ); } + +#[test] +fn float_attribute_mapping() { + check( + r#" +//- proc_macros: identity +//+spans +#[proc_macros::identity] +fn foo(&self) { + self.0. 1; +} +"#, + expect![[r#" +//+spans +#[proc_macros::identity] +fn foo(&self) { + self.0. 1; +} + +fn#FileId(0):1@34..36\0# foo#FileId(0):1@37..40\0#(#FileId(0):1@40..41\0#&#FileId(0):1@41..42\0#self#FileId(0):1@42..46\0# )#FileId(0):1@46..47\0# {#FileId(0):1@48..49\0# + self#FileId(0):1@54..58\0# .#FileId(0):1@58..59\0#0#FileId(0):1@59..60\0#.#FileId(0):1@60..61\0#1#FileId(0):1@62..63\0#;#FileId(0):1@63..64\0# +}#FileId(0):1@65..66\0#"#]], + ); +} diff --git a/crates/hir-def/src/nameres/collector.rs b/crates/hir-def/src/nameres/collector.rs index 360bf0f93e30..fef13604225a 100644 --- a/crates/hir-def/src/nameres/collector.rs +++ b/crates/hir-def/src/nameres/collector.rs @@ -219,6 +219,7 @@ enum MacroDirectiveKind { ast_id: AstIdWithPath, derive_attr: AttrId, derive_pos: usize, + call_site: SyntaxContextId, }, Attr { ast_id: AstIdWithPath, @@ -324,7 +325,7 @@ impl DefCollector<'_> { .parse_path_comma_token_tree(self.db.upcast()) .into_iter() .flatten() - .filter_map(|feat| match feat.segments() { + .filter_map(|(feat, _)| match feat.segments() { [name] => Some(name.to_smol_str()), _ => None, }); @@ -1139,12 +1140,13 @@ impl DefCollector<'_> { return false; } } - MacroDirectiveKind::Derive { ast_id, derive_attr, derive_pos } => { + MacroDirectiveKind::Derive { ast_id, derive_attr, derive_pos, call_site } => { let id = derive_macro_as_call_id( self.db, ast_id, *derive_attr, *derive_pos as u32, + *call_site, self.def_map.krate, resolver, ); @@ -1242,7 +1244,7 @@ impl DefCollector<'_> { match attr.parse_path_comma_token_tree(self.db.upcast()) { Some(derive_macros) => { let mut len = 0; - for (idx, path) in derive_macros.enumerate() { + for (idx, (path, call_site)) in derive_macros.enumerate() { let ast_id = AstIdWithPath::new(file_id, ast_id.value, path); self.unresolved_macros.push(MacroDirective { module_id: directive.module_id, @@ -1251,6 +1253,7 @@ impl DefCollector<'_> { ast_id, derive_attr: attr.id, derive_pos: idx, + call_site, }, container: directive.container, }); @@ -1438,7 +1441,7 @@ impl DefCollector<'_> { )); } } - MacroDirectiveKind::Derive { ast_id, derive_attr, derive_pos } => { + MacroDirectiveKind::Derive { ast_id, derive_attr, derive_pos, call_site: _ } => { self.def_map.diagnostics.push(DefDiagnostic::unresolved_macro_call( directive.module_id, MacroCallKind::Derive { @@ -1828,7 +1831,7 @@ impl ModCollector<'_, '_> { ); return; }; - for path in paths { + for (path, _) in paths { if let Some(name) = path.as_ident() { single_imports.push(name.clone()); } diff --git a/crates/hir-def/src/nameres/tests.rs b/crates/hir-def/src/nameres/tests.rs index e7cc44b04da8..b2ffbbe4c5d8 100644 --- a/crates/hir-def/src/nameres/tests.rs +++ b/crates/hir-def/src/nameres/tests.rs @@ -8,9 +8,7 @@ use base_db::{fixture::WithFixture, SourceDatabase}; use expect_test::{expect, Expect}; use triomphe::Arc; -use crate::{db::DefDatabase, test_db::TestDB}; - -use super::DefMap; +use crate::{db::DefDatabase, nameres::DefMap, test_db::TestDB}; fn compute_crate_def_map(ra_fixture: &str) -> Arc { let db = TestDB::with_files(ra_fixture); diff --git a/crates/hir-def/src/test_db.rs b/crates/hir-def/src/test_db.rs index db64e7cd538d..f4a6b61f7af5 100644 --- a/crates/hir-def/src/test_db.rs +++ b/crates/hir-def/src/test_db.rs @@ -7,7 +7,7 @@ use base_db::{ AnchoredPath, CrateId, FileId, FileLoader, FileLoaderDelegate, FilePosition, SourceDatabase, Upcast, }; -use hir_expand::{db::ExpandDatabase, hygiene::SyntaxContextData, InFile}; +use hir_expand::{db::ExpandDatabase, InFile}; use rustc_hash::FxHashSet; use syntax::{algo, ast, AstNode}; use triomphe::Arc; @@ -34,7 +34,7 @@ pub(crate) struct TestDB { impl Default for TestDB { fn default() -> Self { let mut this = Self { storage: Default::default(), events: Default::default() }; - this.intern_syntax_context(SyntaxContextData::root()); + this.setup_syntax_context_root(); this.set_expand_proc_attr_macros_with_durability(true, Durability::HIGH); this } diff --git a/crates/hir-expand/src/attrs.rs b/crates/hir-expand/src/attrs.rs index 5ce12d2f6e71..76c787721b55 100644 --- a/crates/hir-expand/src/attrs.rs +++ b/crates/hir-expand/src/attrs.rs @@ -2,13 +2,16 @@ use std::{fmt, ops}; use ::tt::SpanAnchor as _; -use base_db::{span::SpanAnchor, CrateId}; +use base_db::{ + span::{SpanAnchor, SyntaxContextId}, + CrateId, +}; use cfg::CfgExpr; use either::Either; use intern::Interned; use mbe::{syntax_node_to_token_tree, DelimiterKind, Punct}; use smallvec::{smallvec, SmallVec}; -use syntax::{ast, match_ast, AstNode, SmolStr, SyntaxNode}; +use syntax::{ast, match_ast, AstNode, AstToken, SmolStr, SyntaxNode}; use triomphe::Arc; use crate::{ @@ -54,6 +57,9 @@ impl RawAttrs { id, input: Some(Interned::new(AttrInput::Literal(SmolStr::new(doc)))), path: Interned::new(ModPath::from(crate::name!(doc))), + ctxt: hygiene + .span_for_range(comment.syntax().text_range()) + .map_or(SyntaxContextId::ROOT, |s| s.ctx), }), }) .collect::>(); @@ -191,6 +197,7 @@ pub struct Attr { pub id: AttrId, pub path: Interned, pub input: Option>, + pub ctxt: SyntaxContextId, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -235,7 +242,14 @@ impl Attr { } else { None }; - Some(Attr { id, path, input }) + Some(Attr { + id, + path, + input, + ctxt: hygiene + .span_for_range(ast.syntax().text_range()) + .map_or(SyntaxContextId::ROOT, |s| s.ctx), + }) } fn from_tt(db: &dyn ExpandDatabase, tt: &tt::Subtree, id: AttrId) -> Option { @@ -284,9 +298,8 @@ impl Attr { pub fn parse_path_comma_token_tree<'a>( &'a self, db: &'a dyn ExpandDatabase, - ) -> Option + 'a> { + ) -> Option + 'a> { let args = self.token_tree_value()?; - dbg!(args); if args.delimiter.kind != DelimiterKind::Parenthesis { return None; @@ -298,6 +311,11 @@ impl Attr { if tts.is_empty() { return None; } + // FIXME: Absolutely wrong + let call_site = match tts.first().unwrap() { + tt::TokenTree::Leaf(l) => l.span().ctx, + tt::TokenTree::Subtree(s) => s.delimiter.open.ctx, + }; // FIXME: This is necessarily a hack. It'd be nice if we could avoid allocation // here. let subtree = tt::Subtree { @@ -313,7 +331,7 @@ impl Attr { return None; } let path = meta.path()?; - ModPath::from_src(db, path, &span_map) + Some((ModPath::from_src(db, path, &span_map)?, call_site)) }); Some(paths) diff --git a/crates/hir-expand/src/db.rs b/crates/hir-expand/src/db.rs index 7dd69099a6e3..e176bef78b5a 100644 --- a/crates/hir-expand/src/db.rs +++ b/crates/hir-expand/src/db.rs @@ -103,7 +103,7 @@ pub trait ExpandDatabase: SourceDatabase { &self, macro_file: MacroFile, ) -> ExpandResult<(Parse, Arc)>; - // TODO: transparent? + // FIXME: This always allocates one for non macro files which is wasteful. #[salsa::transparent] fn span_map(&self, file_id: HirFileId) -> Arc; @@ -117,6 +117,8 @@ pub trait ExpandDatabase: SourceDatabase { #[salsa::interned] fn intern_syntax_context(&self, ctx: SyntaxContextData) -> SyntaxContextId; #[salsa::transparent] + fn setup_syntax_context_root(&self) -> (); + #[salsa::transparent] #[salsa::invoke(hygiene::apply_mark)] fn apply_mark( &self, @@ -770,3 +772,7 @@ fn check_tt_count(tt: &tt::Subtree) -> Result<(), ExpandResult> Ok(()) } } + +fn setup_syntax_context_root(db: &dyn ExpandDatabase) { + db.intern_syntax_context(SyntaxContextData::root()); +} diff --git a/crates/hir-ty/src/test_db.rs b/crates/hir-ty/src/test_db.rs index a3383b2b5dfb..6f4aef22d2f7 100644 --- a/crates/hir-ty/src/test_db.rs +++ b/crates/hir-ty/src/test_db.rs @@ -7,7 +7,7 @@ use base_db::{ AnchoredPath, CrateId, FileId, FileLoader, FileLoaderDelegate, SourceDatabase, Upcast, }; use hir_def::{db::DefDatabase, ModuleId}; -use hir_expand::{db::ExpandDatabase, hygiene::SyntaxContextData}; +use hir_expand::db::ExpandDatabase; use nohash_hasher::IntMap; use rustc_hash::FxHashSet; use syntax::TextRange; @@ -30,7 +30,7 @@ pub(crate) struct TestDB { impl Default for TestDB { fn default() -> Self { let mut this = Self { storage: Default::default(), events: Default::default() }; - this.intern_syntax_context(SyntaxContextData::root()); + this.setup_syntax_context_root(); this.set_expand_proc_attr_macros_with_durability(true, Durability::HIGH); this } diff --git a/crates/hir-ty/src/tests/macros.rs b/crates/hir-ty/src/tests/macros.rs index d16e0eb0137b..1e10a6fecafb 100644 --- a/crates/hir-ty/src/tests/macros.rs +++ b/crates/hir-ty/src/tests/macros.rs @@ -63,10 +63,10 @@ fn infer_macros_expanded() { } "#, expect![[r#" - !0..17 '{Foo(v...,2,])}': Foo + !0..21 '{Foo(v...2),])}': Foo !1..4 'Foo': Foo({unknown}) -> Foo - !1..16 'Foo(vec![1,2,])': Foo - !5..15 'vec![1,2,]': {unknown} + !1..20 'Foo(ve...(2),])': Foo + !5..19 'vec![(1),(2),]': {unknown} 155..181 '{ ...,2); }': () 165..166 'x': Foo "#]], @@ -96,10 +96,10 @@ fn infer_legacy_textual_scoped_macros_expanded() { } "#, expect![[r#" - !0..17 '{Foo(v...,2,])}': Foo + !0..21 '{Foo(v...2),])}': Foo !1..4 'Foo': Foo({unknown}) -> Foo - !1..16 'Foo(vec![1,2,])': Foo - !5..15 'vec![1,2,]': {unknown} + !1..20 'Foo(ve...(2),])': Foo + !5..19 'vec![(1),(2),]': {unknown} 194..250 '{ ...,2); }': () 204..205 'x': Foo 227..228 'y': {unknown} diff --git a/crates/ide-db/src/lib.rs b/crates/ide-db/src/lib.rs index 957c9ad26ca2..38c9a3538f3a 100644 --- a/crates/ide-db/src/lib.rs +++ b/crates/ide-db/src/lib.rs @@ -144,6 +144,7 @@ impl RootDatabase { db.set_library_roots_with_durability(Default::default(), Durability::HIGH); db.set_expand_proc_attr_macros_with_durability(false, Durability::HIGH); db.update_parse_query_lru_capacity(lru_capacity); + db.setup_syntax_context_root(); db } diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index b843db510e39..f47123336b44 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -717,24 +717,29 @@ where /// Parses a float literal as if it was a one to two name ref nodes with a dot inbetween. /// This occurs when a float literal is used as a field access. fn float_split(&mut self, has_pseudo_dot: bool) { - // TODO: FIXME this breaks the hygiene map - let (text, _span) = match self.cursor.token_tree() { + let (text, span) = match self.cursor.token_tree() { Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Literal(lit), _)) => { (lit.text.as_str(), lit.span) } _ => unreachable!(), }; + // FIXME: Span splitting match text.split_once('.') { Some((left, right)) => { assert!(!left.is_empty()); + self.inner.start_node(SyntaxKind::NAME_REF); self.inner.token(SyntaxKind::INT_NUMBER, left); self.inner.finish_node(); + let range = TextRange::at(self.text_pos, TextSize::of(left)); + self.token_map.insert(range, span); // here we move the exit up, the original exit has been deleted in process self.inner.finish_node(); self.inner.token(SyntaxKind::DOT, "."); + let range = TextRange::at(range.end(), TextSize::of(".")); + self.token_map.insert(range, span); if has_pseudo_dot { assert!(right.is_empty(), "{left}.{right}"); @@ -742,11 +747,14 @@ where assert!(!right.is_empty(), "{left}.{right}"); self.inner.start_node(SyntaxKind::NAME_REF); self.inner.token(SyntaxKind::INT_NUMBER, right); + let range = TextRange::at(range.end(), TextSize::of(right)); + self.token_map.insert(range, span); self.inner.finish_node(); // the parser creates an unbalanced start node, we are required to close it here self.inner.finish_node(); } + self.text_pos += TextSize::of(text); } None => unreachable!(), } diff --git a/crates/mbe/src/token_map.rs b/crates/mbe/src/token_map.rs index c825bd01bcd6..dfbf54410b18 100644 --- a/crates/mbe/src/token_map.rs +++ b/crates/mbe/src/token_map.rs @@ -2,6 +2,7 @@ use std::hash::Hash; +use stdx::never; use syntax::TextRange; use tt::Span; @@ -59,11 +60,10 @@ impl TokenMap { .max_by_key(|(_, _, intersection)| intersection.len()) .map(|(_, &s, _)| s) .or_else(|| { - if self.real_file { - None - } else { - panic!("no span for range {range:?} in {:#?}", self.span_map) + if !self.real_file { + never!("no span for range {:?} in {:#?}", range, self.span_map); } + None }) } From 30093a6d81f49854d0ad068765bf7af9fdb2a101 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Fri, 24 Nov 2023 16:38:48 +0100 Subject: [PATCH 08/24] spans always come from real file --- Cargo.lock | 1 + crates/base-db/src/span.rs | 9 +- crates/cfg/src/tests.rs | 40 +- crates/hir-def/src/attr.rs | 50 +- crates/hir-def/src/attr/tests.rs | 16 +- crates/hir-def/src/expander.rs | 26 +- crates/hir-def/src/find_path.rs | 5 +- crates/hir-def/src/item_scope.rs | 9 + crates/hir-def/src/item_tree.rs | 15 +- crates/hir-def/src/item_tree/lower.rs | 132 +---- crates/hir-def/src/lib.rs | 17 +- crates/hir-def/src/lower.rs | 13 +- .../hir-def/src/macro_expansion_tests/mbe.rs | 35 +- .../hir-def/src/macro_expansion_tests/mod.rs | 45 +- .../src/macro_expansion_tests/proc_macros.rs | 11 +- crates/hir-def/src/nameres/collector.rs | 2 +- crates/hir-def/src/path/lower.rs | 19 +- crates/hir-def/src/visibility.rs | 8 +- crates/hir-expand/src/ast_id_map.rs | 47 +- crates/hir-expand/src/attrs.rs | 49 +- crates/hir-expand/src/builtin_attr_macro.rs | 9 +- crates/hir-expand/src/builtin_derive_macro.rs | 122 ++-- crates/hir-expand/src/builtin_fn_macro.rs | 3 +- crates/hir-expand/src/db.rs | 152 ++--- crates/hir-expand/src/eager.rs | 78 +-- crates/hir-expand/src/files.rs | 293 ++++++++++ crates/hir-expand/src/hygiene.rs | 67 ++- crates/hir-expand/src/lib.rs | 551 +++++------------- crates/hir-expand/src/mod_path.rs | 56 +- crates/hir-expand/src/span.rs | 109 ++++ crates/hir/src/attrs.rs | 12 +- crates/hir/src/lib.rs | 34 +- crates/hir/src/semantics.rs | 38 +- crates/hir/src/symbols.rs | 12 +- .../ide-completion/src/tests/proc_macros.rs | 2 + crates/ide-db/src/rename.rs | 79 ++- .../src/handlers/macro_error.rs | 4 - .../src/handlers/typed_hole.rs | 2 +- .../src/handlers/unresolved_module.rs | 4 +- crates/ide/src/expand_macro.rs | 10 +- crates/ide/src/goto_definition.rs | 4 +- crates/ide/src/goto_implementation.rs | 2 +- crates/ide/src/hover.rs | 1 + .../ide/src/inlay_hints/closure_captures.rs | 5 +- crates/ide/src/navigation_target.rs | 43 +- crates/ide/src/static_index.rs | 1 + .../test_data/highlight_strings.html | 2 +- crates/mbe/src/benchmark.rs | 19 +- crates/mbe/src/lib.rs | 4 +- crates/mbe/src/syntax_bridge.rs | 194 +++--- crates/mbe/src/syntax_bridge/tests.rs | 15 +- crates/mbe/src/token_map.rs | 57 +- crates/proc-macro-srv/src/lib.rs | 2 +- crates/rust-analyzer/src/cargo_target_spec.rs | 22 +- crates/stdx/Cargo.toml | 1 + crates/stdx/src/lib.rs | 19 + crates/tt/src/lib.rs | 14 +- 57 files changed, 1368 insertions(+), 1223 deletions(-) create mode 100644 crates/hir-expand/src/files.rs create mode 100644 crates/hir-expand/src/span.rs diff --git a/Cargo.lock b/Cargo.lock index 90ee0810fa6a..3618d69c749a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1752,6 +1752,7 @@ dependencies = [ "always-assert", "backtrace", "crossbeam-channel", + "itertools 0.12.0", "jod-thread", "libc", "miow", diff --git a/crates/base-db/src/span.rs b/crates/base-db/src/span.rs index 600aa079333e..a78f558759b0 100644 --- a/crates/base-db/src/span.rs +++ b/crates/base-db/src/span.rs @@ -35,11 +35,15 @@ impl SyntaxContextId { // FIXME: This is very much UB, salsa exposes no way to create an InternId in a const context // currently (which kind of makes sense but we need it here!) pub const SELF_REF: Self = SyntaxContextId(unsafe { core::mem::transmute(!0u32) }); + + pub fn is_root(self) -> bool { + self == Self::ROOT + } } #[derive(Copy, Clone, PartialEq, Eq, Hash)] pub struct SpanAnchor { - pub file_id: HirFileId, + pub file_id: FileId, pub ast_id: ErasedFileAstId, } @@ -50,7 +54,7 @@ impl fmt::Debug for SpanAnchor { } impl tt::SpanAnchor for SpanAnchor { - const DUMMY: Self = SpanAnchor { file_id: HirFileId(0), ast_id: ROOT_ERASED_FILE_AST_ID }; + const DUMMY: Self = SpanAnchor { file_id: FileId(0), ast_id: ROOT_ERASED_FILE_AST_ID }; } /// Input to the analyzer is a set of files, where each file is identified by @@ -101,7 +105,6 @@ impl fmt::Debug for HirFileId { pub struct MacroFile { pub macro_call_id: MacroCallId, } - /// `MacroCallId` identifies a particular macro invocation, like /// `println!("Hello, {}", world)`. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] diff --git a/crates/cfg/src/tests.rs b/crates/cfg/src/tests.rs index 0ea176858c94..61cdbded0b99 100644 --- a/crates/cfg/src/tests.rs +++ b/crates/cfg/src/tests.rs @@ -1,12 +1,12 @@ use arbitrary::{Arbitrary, Unstructured}; use expect_test::{expect, Expect}; -use mbe::syntax_node_to_token_tree; +use mbe::{syntax_node_to_token_tree, SpanMapper}; use syntax::{ast, AstNode}; use tt::{SpanAnchor, SyntaxContext}; use crate::{CfgAtom, CfgExpr, CfgOptions, DnfExpr}; -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] struct DummyFile; impl SpanAnchor for DummyFile { const DUMMY: Self = DummyFile; @@ -17,15 +17,18 @@ impl SyntaxContext for DummyCtx { const DUMMY: Self = DummyCtx; } +struct NoOpMap; + +impl SpanMapper> for NoOpMap { + fn span_for(&self, range: syntax::TextRange) -> tt::SpanData { + tt::SpanData { range, anchor: DummyFile, ctx: DummyCtx } + } +} + fn assert_parse_result(input: &str, expected: CfgExpr) { let source_file = ast::SourceFile::parse(input).ok().unwrap(); let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - let tt = syntax_node_to_token_tree::<_, DummyCtx>( - tt.syntax(), - DummyFile, - 0.into(), - &Default::default(), - ); + let tt = syntax_node_to_token_tree(tt.syntax(), NoOpMap); let cfg = CfgExpr::parse(&tt); assert_eq!(cfg, expected); } @@ -33,12 +36,7 @@ fn assert_parse_result(input: &str, expected: CfgExpr) { fn check_dnf(input: &str, expect: Expect) { let source_file = ast::SourceFile::parse(input).ok().unwrap(); let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - let tt = syntax_node_to_token_tree::<_, DummyCtx>( - tt.syntax(), - DummyFile, - 0.into(), - &Default::default(), - ); + let tt = syntax_node_to_token_tree(tt.syntax(), NoOpMap); let cfg = CfgExpr::parse(&tt); let actual = format!("#![cfg({})]", DnfExpr::new(cfg)); expect.assert_eq(&actual); @@ -47,12 +45,7 @@ fn check_dnf(input: &str, expect: Expect) { fn check_why_inactive(input: &str, opts: &CfgOptions, expect: Expect) { let source_file = ast::SourceFile::parse(input).ok().unwrap(); let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - let tt = syntax_node_to_token_tree::<_, DummyCtx>( - tt.syntax(), - DummyFile, - 0.into(), - &Default::default(), - ); + let tt = syntax_node_to_token_tree(tt.syntax(), NoOpMap); let cfg = CfgExpr::parse(&tt); let dnf = DnfExpr::new(cfg); let why_inactive = dnf.why_inactive(opts).unwrap().to_string(); @@ -63,12 +56,7 @@ fn check_why_inactive(input: &str, opts: &CfgOptions, expect: Expect) { fn check_enable_hints(input: &str, opts: &CfgOptions, expected_hints: &[&str]) { let source_file = ast::SourceFile::parse(input).ok().unwrap(); let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - let tt = syntax_node_to_token_tree::<_, DummyCtx>( - tt.syntax(), - DummyFile, - 0.into(), - &Default::default(), - ); + let tt = syntax_node_to_token_tree(tt.syntax(), NoOpMap); let cfg = CfgExpr::parse(&tt); let dnf = DnfExpr::new(cfg); let hints = dnf.compute_enable_hints(opts).map(|diff| diff.to_string()).collect::>(); diff --git a/crates/hir-def/src/attr.rs b/crates/hir-def/src/attr.rs index 35c9d63979a0..45dd981dceec 100644 --- a/crates/hir-def/src/attr.rs +++ b/crates/hir-def/src/attr.rs @@ -7,10 +7,7 @@ mod tests; use std::{hash::Hash, ops, slice::Iter as SliceIter}; -use base_db::{ - span::{ErasedFileAstId, SpanAnchor}, - CrateId, -}; +use base_db::CrateId; use cfg::{CfgExpr, CfgOptions}; use either::Either; use hir_expand::{ @@ -31,8 +28,8 @@ use crate::{ lang_item::LangItem, nameres::{ModuleOrigin, ModuleSource}, src::{HasChildSource, HasSource}, - AdtId, AssocItemLoc, AttrDefId, EnumId, GenericDefId, GenericParamId, ItemLoc, - LocalEnumVariantId, LocalFieldId, Lookup, MacroId, VariantId, + AdtId, AssocItemLoc, AttrDefId, EnumId, GenericParamId, ItemLoc, LocalEnumVariantId, + LocalFieldId, Lookup, MacroId, VariantId, }; #[derive(Default, Debug, Clone, PartialEq, Eq)] @@ -419,43 +416,30 @@ impl AttrsWithOwner { AttrDefId::FunctionId(it) => attrs_from_item_tree_assoc(db, it), AttrDefId::TypeAliasId(it) => attrs_from_item_tree_assoc(db, it), AttrDefId::GenericParamId(it) => { - let ast_id = |p| match p { - GenericDefId::AdtId(AdtId::StructId(it)) => { - erased_ast_id_from_item_tree(db, it) - } - GenericDefId::AdtId(AdtId::EnumId(it)) => erased_ast_id_from_item_tree(db, it), - GenericDefId::AdtId(AdtId::UnionId(it)) => erased_ast_id_from_item_tree(db, it), - GenericDefId::TraitId(it) => erased_ast_id_from_item_tree(db, it), - GenericDefId::TraitAliasId(it) => erased_ast_id_from_item_tree(db, it), - GenericDefId::ImplId(it) => erased_ast_id_from_item_tree(db, it), - GenericDefId::EnumVariantId(it) => erased_ast_id_from_item_tree(db, it.parent), - GenericDefId::TypeAliasId(it) => erased_ast_id_from_item_tree_assoc(db, it), - GenericDefId::FunctionId(it) => erased_ast_id_from_item_tree_assoc(db, it), - GenericDefId::ConstId(it) => erased_ast_id_from_item_tree_assoc(db, it), - }; + // FIXME: we could probably just make these relative to the params? match it { GenericParamId::ConstParamId(it) => { let src = it.parent().child_source(db); RawAttrs::from_attrs_owner( db.upcast(), - SpanAnchor { file_id: src.file_id, ast_id: ast_id(it.parent()) }, src.with_value(&src.value[it.local_id()]), + db.span_map(src.file_id).as_ref(), ) } GenericParamId::TypeParamId(it) => { let src = it.parent().child_source(db); RawAttrs::from_attrs_owner( db.upcast(), - SpanAnchor { file_id: src.file_id, ast_id: ast_id(it.parent()) }, src.with_value(&src.value[it.local_id()]), + db.span_map(src.file_id).as_ref(), ) } GenericParamId::LifetimeParamId(it) => { let src = it.parent.child_source(db); RawAttrs::from_attrs_owner( db.upcast(), - SpanAnchor { file_id: src.file_id, ast_id: ast_id(it.parent) }, src.with_value(&src.value[it.local_id]), + db.span_map(src.file_id).as_ref(), ) } } @@ -663,26 +647,6 @@ fn any_has_attrs( id.lookup(db).source(db).map(ast::AnyHasAttrs::new) } -fn erased_ast_id_from_item_tree( - db: &dyn DefDatabase, - lookup: impl Lookup>, -) -> ErasedFileAstId { - let id = lookup.lookup(db).id; - let tree = id.item_tree(db); - let mod_item = N::id_to_mod_item(id.value); - mod_item.ast_id(&tree).erase() -} - -fn erased_ast_id_from_item_tree_assoc( - db: &dyn DefDatabase, - lookup: impl Lookup>, -) -> ErasedFileAstId { - let id = lookup.lookup(db).id; - let tree = id.item_tree(db); - let mod_item = N::id_to_mod_item(id.value); - mod_item.ast_id(&tree).erase() -} - fn attrs_from_item_tree(db: &dyn DefDatabase, id: ItemTreeId) -> RawAttrs { let tree = id.item_tree(db); let mod_item = N::id_to_mod_item(id.value); diff --git a/crates/hir-def/src/attr/tests.rs b/crates/hir-def/src/attr/tests.rs index 60e5cebd3cb0..0f21dc98299f 100644 --- a/crates/hir-def/src/attr/tests.rs +++ b/crates/hir-def/src/attr/tests.rs @@ -1,27 +1,19 @@ //! This module contains tests for doc-expression parsing. //! Currently, it tests `#[doc(hidden)]` and `#[doc(alias)]`. -use base_db::span::SpanAnchor; +use base_db::FileId; +use hir_expand::span::{RealSpanMap, SpanMapRef}; use mbe::syntax_node_to_token_tree; use syntax::{ast, AstNode}; -use tt::{SpanAnchor as _, SyntaxContext}; use crate::attr::{DocAtom, DocExpr}; -#[derive(Copy, Clone, PartialEq, Eq, Debug)] -struct DummyCtx; -impl SyntaxContext for DummyCtx { - const DUMMY: Self = DummyCtx; -} - fn assert_parse_result(input: &str, expected: DocExpr) { let source_file = ast::SourceFile::parse(input).ok().unwrap(); let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - let tt = syntax_node_to_token_tree::<_, DummyCtx>( + let tt = syntax_node_to_token_tree( tt.syntax(), - SpanAnchor::DUMMY, - 0.into(), - &Default::default(), + SpanMapRef::RealSpanMap(&RealSpanMap::empty(FileId(0))), ); let cfg = DocExpr::parse(&tt); assert_eq!(cfg, expected); diff --git a/crates/hir-def/src/expander.rs b/crates/hir-def/src/expander.rs index 793c8ddeb504..d8ee61a3dca3 100644 --- a/crates/hir-def/src/expander.rs +++ b/crates/hir-def/src/expander.rs @@ -1,28 +1,24 @@ //! Macro expansion utilities. -use base_db::{ - span::{SpanAnchor, ROOT_ERASED_FILE_AST_ID}, - CrateId, -}; +use base_db::CrateId; use cfg::CfgOptions; use drop_bomb::DropBomb; use hir_expand::{ - attrs::RawAttrs, mod_path::ModPath, ExpandError, ExpandResult, HirFileId, InFile, MacroCallId, - SpanMap, UnresolvedMacro, + attrs::RawAttrs, mod_path::ModPath, span::SpanMap, ExpandError, ExpandResult, HirFileId, + InFile, MacroCallId, }; use limit::Limit; use syntax::{ast, Parse, SyntaxNode}; -use triomphe::Arc; use crate::{ attr::Attrs, db::DefDatabase, lower::LowerCtx, macro_id_to_def_id, path::Path, AsMacroCall, - MacroId, ModuleId, + MacroId, ModuleId, UnresolvedMacro, }; #[derive(Debug)] pub struct Expander { cfg_options: CfgOptions, - hygiene: Arc, + hygiene: SpanMap, krate: CrateId, pub(crate) current_file_id: HirFileId, pub(crate) module: ModuleId, @@ -122,17 +118,7 @@ impl Expander { } pub(crate) fn parse_attrs(&self, db: &dyn DefDatabase, owner: &dyn ast::HasAttrs) -> Attrs { - Attrs::filter( - db, - self.krate, - RawAttrs::new( - db.upcast(), - // Usin `ROOT_ERASED_FILE_AST_ID` here is fine as this is only used for cfg checking - SpanAnchor { file_id: self.current_file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, - owner, - &self.hygiene, - ), - ) + Attrs::filter(db, self.krate, RawAttrs::new(db.upcast(), owner, self.hygiene.as_ref())) } pub(crate) fn cfg_options(&self) -> &CfgOptions { diff --git a/crates/hir-def/src/find_path.rs b/crates/hir-def/src/find_path.rs index 5051884714ba..13af0b0218e8 100644 --- a/crates/hir-def/src/find_path.rs +++ b/crates/hir-def/src/find_path.rs @@ -586,7 +586,7 @@ fn find_local_import_locations( #[cfg(test)] mod tests { use base_db::fixture::WithFixture; - use hir_expand::SpanMap; + use hir_expand::db::ExpandDatabase; use syntax::ast::AstNode; use crate::test_db::TestDB; @@ -608,7 +608,8 @@ mod tests { let parsed_path_file = syntax::SourceFile::parse(&format!("use {path};")); let ast_path = parsed_path_file.syntax_node().descendants().find_map(syntax::ast::Path::cast).unwrap(); - let mod_path = ModPath::from_src(&db, ast_path, &SpanMap::default()).unwrap(); + let mod_path = + ModPath::from_src(&db, ast_path, db.span_map(pos.file_id.into()).as_ref()).unwrap(); let def_map = module.def_map(&db); let resolved = def_map diff --git a/crates/hir-def/src/item_scope.rs b/crates/hir-def/src/item_scope.rs index 7c11fb9d1367..ce83cb435e2e 100644 --- a/crates/hir-def/src/item_scope.rs +++ b/crates/hir-def/src/item_scope.rs @@ -112,6 +112,7 @@ pub struct ItemScope { #[derive(Debug, PartialEq, Eq)] struct DeriveMacroInvocation { attr_id: AttrId, + /// The `#[derive]` call attr_call_id: MacroCallId, derive_call_ids: SmallVec<[Option; 1]>, } @@ -401,6 +402,14 @@ impl ItemScope { }) } + pub fn derive_macro_invoc( + &self, + ast_id: AstId, + attr_id: AttrId, + ) -> Option { + Some(self.derive_macros.get(&ast_id)?.iter().find(|it| it.attr_id == attr_id)?.attr_call_id) + } + // FIXME: This is only used in collection, we should move the relevant parts of it out of ItemScope pub(crate) fn unnamed_trait_vis(&self, tr: TraitId) -> Option { self.unnamed_trait_imports.get(&tr).copied().map(|(a, _)| a) diff --git a/crates/hir-def/src/item_tree.rs b/crates/hir-def/src/item_tree.rs index 901e14a2117b..9c61deb423cb 100644 --- a/crates/hir-def/src/item_tree.rs +++ b/crates/hir-def/src/item_tree.rs @@ -43,10 +43,7 @@ use std::{ }; use ast::{AstNode, HasName, StructKind}; -use base_db::{ - span::{SpanAnchor, SyntaxContextId, ROOT_ERASED_FILE_AST_ID}, - CrateId, -}; +use base_db::{span::SyntaxContextId, CrateId}; use either::Either; use hir_expand::{ ast_id_map::{AstIdNode, FileAstId}, @@ -121,7 +118,7 @@ impl ItemTree { let mut item_tree = match_ast! { match syntax { ast::SourceFile(file) => { - top_attrs = Some(RawAttrs::new(db.upcast(), SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, &file, ctx.span_map())); + top_attrs = Some(RawAttrs::new(db.upcast(), &file, ctx.span_map())); ctx.lower_module_items(&file) }, ast::MacroItems(items) => { @@ -780,8 +777,8 @@ impl Use { let ast = InFile::new(file_id, self.ast_id).to_node(db.upcast()); let ast_use_tree = ast.use_tree().expect("missing `use_tree`"); let hygiene = db.span_map(file_id); - let (_, source_map) = - lower::lower_use_tree(db, &hygiene, ast_use_tree).expect("failed to lower use tree"); + let (_, source_map) = lower::lower_use_tree(db, hygiene.as_ref(), ast_use_tree) + .expect("failed to lower use tree"); source_map[index].clone() } /// Maps a `UseTree` contained in this import back to its AST node. @@ -795,7 +792,9 @@ impl Use { let ast = InFile::new(file_id, self.ast_id).to_node(db.upcast()); let ast_use_tree = ast.use_tree().expect("missing `use_tree`"); let hygiene = db.span_map(file_id); - lower::lower_use_tree(db, &hygiene, ast_use_tree).expect("failed to lower use tree").1 + lower::lower_use_tree(db, hygiene.as_ref(), ast_use_tree) + .expect("failed to lower use tree") + .1 } } diff --git a/crates/hir-def/src/item_tree/lower.rs b/crates/hir-def/src/item_tree/lower.rs index 0b3def6d756b..933a59be1534 100644 --- a/crates/hir-def/src/item_tree/lower.rs +++ b/crates/hir-def/src/item_tree/lower.rs @@ -2,8 +2,7 @@ use std::collections::hash_map::Entry; -use base_db::span::ErasedFileAstId; -use hir_expand::{ast_id_map::AstIdMap, HirFileId, SpanMap}; +use hir_expand::{ast_id_map::AstIdMap, span::SpanMapRef, HirFileId}; use syntax::ast::{self, HasModuleItem, HasTypeBounds}; use crate::{ @@ -23,7 +22,6 @@ pub(super) struct Ctx<'a> { tree: ItemTree, source_ast_id_map: Arc, body_ctx: crate::lower::LowerCtx<'a>, - file: HirFileId, } impl<'a> Ctx<'a> { @@ -33,11 +31,10 @@ impl<'a> Ctx<'a> { tree: ItemTree::default(), source_ast_id_map: db.ast_id_map(file), body_ctx: crate::lower::LowerCtx::with_file_id(db, file), - file, } } - pub(super) fn span_map(&self) -> &SpanMap { + pub(super) fn span_map(&self) -> SpanMapRef<'_> { self.body_ctx.span_map() } @@ -81,18 +78,9 @@ impl<'a> Ctx<'a> { } pub(super) fn lower_block(mut self, block: &ast::BlockExpr) -> ItemTree { - self.tree.attrs.insert( - AttrOwner::TopLevel, - RawAttrs::new( - self.db.upcast(), - SpanAnchor { - file_id: self.file, - ast_id: self.source_ast_id_map.ast_id(block).erase(), - }, - block, - self.span_map(), - ), - ); + self.tree + .attrs + .insert(AttrOwner::TopLevel, RawAttrs::new(self.db.upcast(), block, self.span_map())); self.tree.top_level = block .statements() .filter_map(|stmt| match stmt { @@ -141,12 +129,7 @@ impl<'a> Ctx<'a> { ast::Item::MacroDef(ast) => self.lower_macro_def(ast)?.into(), ast::Item::ExternBlock(ast) => self.lower_extern_block(ast).into(), }; - let attrs = RawAttrs::new( - self.db.upcast(), - SpanAnchor { file_id: self.file, ast_id: mod_item.ast_id(&self.tree).erase() }, - item, - self.span_map(), - ); + let attrs = RawAttrs::new(self.db.upcast(), item, self.span_map()); self.add_attrs(mod_item.into(), attrs); Some(mod_item) @@ -170,12 +153,7 @@ impl<'a> Ctx<'a> { ast::AssocItem::Const(ast) => Some(self.lower_const(ast).into()), ast::AssocItem::MacroCall(ast) => self.lower_macro_call(ast).map(Into::into), }?; - let attrs = RawAttrs::new( - self.db.upcast(), - SpanAnchor { file_id: self.file, ast_id: item.ast_id(&self.tree).erase() }, - item_node, - self.span_map(), - ); + let attrs = RawAttrs::new(self.db.upcast(), item_node, self.span_map()); self.add_attrs( match item { AssocItem::Function(it) => AttrOwner::ModItem(ModItem::Function(it)), @@ -192,7 +170,7 @@ impl<'a> Ctx<'a> { let visibility = self.lower_visibility(strukt); let name = strukt.name()?.as_name(); let ast_id = self.source_ast_id_map.ast_id(strukt); - let generic_params = self.lower_generic_params(HasImplicitSelf::No, strukt, ast_id.erase()); + let generic_params = self.lower_generic_params(HasImplicitSelf::No, strukt); let fields = self.lower_fields(&strukt.kind()); let res = Struct { name, visibility, generic_params, fields, ast_id }; Some(id(self.data().structs.alloc(res))) @@ -216,19 +194,10 @@ impl<'a> Ctx<'a> { let start = self.next_field_idx(); for field in fields.fields() { if let Some(data) = self.lower_record_field(&field) { - let ast_id = match data.ast_id { - FieldAstId::Record(it) => it.erase(), - FieldAstId::Tuple(it) => it.erase(), - }; let idx = self.data().fields.alloc(data); self.add_attrs( idx.into(), - RawAttrs::new( - self.db.upcast(), - SpanAnchor { file_id: self.file, ast_id }, - &field, - self.span_map(), - ), + RawAttrs::new(self.db.upcast(), &field, self.span_map()), ); } } @@ -249,20 +218,8 @@ impl<'a> Ctx<'a> { let start = self.next_field_idx(); for (i, field) in fields.fields().enumerate() { let data = self.lower_tuple_field(i, &field); - let ast_id = match data.ast_id { - FieldAstId::Record(it) => it.erase(), - FieldAstId::Tuple(it) => it.erase(), - }; let idx = self.data().fields.alloc(data); - self.add_attrs( - idx.into(), - RawAttrs::new( - self.db.upcast(), - SpanAnchor { file_id: self.file, ast_id }, - &field, - self.span_map(), - ), - ); + self.add_attrs(idx.into(), RawAttrs::new(self.db.upcast(), &field, self.span_map())); } let end = self.next_field_idx(); IdxRange::new(start..end) @@ -280,7 +237,7 @@ impl<'a> Ctx<'a> { let visibility = self.lower_visibility(union); let name = union.name()?.as_name(); let ast_id = self.source_ast_id_map.ast_id(union); - let generic_params = self.lower_generic_params(HasImplicitSelf::No, union, ast_id.erase()); + let generic_params = self.lower_generic_params(HasImplicitSelf::No, union); let fields = match union.record_field_list() { Some(record_field_list) => self.lower_fields(&StructKind::Record(record_field_list)), None => Fields::Record(IdxRange::new(self.next_field_idx()..self.next_field_idx())), @@ -293,7 +250,7 @@ impl<'a> Ctx<'a> { let visibility = self.lower_visibility(enum_); let name = enum_.name()?.as_name(); let ast_id = self.source_ast_id_map.ast_id(enum_); - let generic_params = self.lower_generic_params(HasImplicitSelf::No, enum_, ast_id.erase()); + let generic_params = self.lower_generic_params(HasImplicitSelf::No, enum_); let variants = match &enum_.variant_list() { Some(variant_list) => self.lower_variants(variant_list), None => IdxRange::new(self.next_variant_idx()..self.next_variant_idx()), @@ -306,16 +263,10 @@ impl<'a> Ctx<'a> { let start = self.next_variant_idx(); for variant in variants.variants() { if let Some(data) = self.lower_variant(&variant) { - let ast_id = data.ast_id.erase(); let idx = self.data().variants.alloc(data); self.add_attrs( idx.into(), - RawAttrs::new( - self.db.upcast(), - SpanAnchor { file_id: self.file, ast_id }, - &variant, - self.span_map(), - ), + RawAttrs::new(self.db.upcast(), &variant, self.span_map()), ); } } @@ -366,12 +317,7 @@ impl<'a> Ctx<'a> { }); self.add_attrs( idx.into(), - RawAttrs::new( - self.db.upcast(), - SpanAnchor { file_id: self.file, ast_id: ast_id.erase() }, - &self_param, - self.span_map(), - ), + RawAttrs::new(self.db.upcast(), &self_param, self.span_map()), ); has_self_param = true; } @@ -392,12 +338,7 @@ impl<'a> Ctx<'a> { }; self.add_attrs( idx.into(), - RawAttrs::new( - self.db.upcast(), - SpanAnchor { file_id: self.file, ast_id: ast_id.erase() }, - ¶m, - self.span_map(), - ), + RawAttrs::new(self.db.upcast(), ¶m, self.span_map()), ); } } @@ -455,8 +396,7 @@ impl<'a> Ctx<'a> { ast_id, flags, }; - res.explicit_generic_params = - self.lower_generic_params(HasImplicitSelf::No, func, ast_id.erase()); + res.explicit_generic_params = self.lower_generic_params(HasImplicitSelf::No, func); Some(id(self.data().functions.alloc(res))) } @@ -470,8 +410,7 @@ impl<'a> Ctx<'a> { let visibility = self.lower_visibility(type_alias); let bounds = self.lower_type_bounds(type_alias); let ast_id = self.source_ast_id_map.ast_id(type_alias); - let generic_params = - self.lower_generic_params(HasImplicitSelf::No, type_alias, ast_id.erase()); + let generic_params = self.lower_generic_params(HasImplicitSelf::No, type_alias); let res = TypeAlias { name, visibility, bounds, generic_params, type_ref, ast_id }; Some(id(self.data().type_aliases.alloc(res))) } @@ -520,11 +459,8 @@ impl<'a> Ctx<'a> { let name = trait_def.name()?.as_name(); let visibility = self.lower_visibility(trait_def); let ast_id = self.source_ast_id_map.ast_id(trait_def); - let generic_params = self.lower_generic_params( - HasImplicitSelf::Yes(trait_def.type_bound_list()), - trait_def, - ast_id.erase(), - ); + let generic_params = + self.lower_generic_params(HasImplicitSelf::Yes(trait_def.type_bound_list()), trait_def); let is_auto = trait_def.auto_token().is_some(); let is_unsafe = trait_def.unsafe_token().is_some(); @@ -549,7 +485,6 @@ impl<'a> Ctx<'a> { let generic_params = self.lower_generic_params( HasImplicitSelf::Yes(trait_alias_def.type_bound_list()), trait_alias_def, - ast_id.erase(), ); let alias = TraitAlias { name, visibility, generic_params, ast_id }; @@ -560,8 +495,7 @@ impl<'a> Ctx<'a> { let ast_id = self.source_ast_id_map.ast_id(impl_def); // Note that trait impls don't get implicit `Self` unlike traits, because here they are a // type alias rather than a type parameter, so this is handled by the resolver. - let generic_params = - self.lower_generic_params(HasImplicitSelf::No, impl_def, ast_id.erase()); + let generic_params = self.lower_generic_params(HasImplicitSelf::No, impl_def); // FIXME: If trait lowering fails, due to a non PathType for example, we treat this impl // as if it was an non-trait impl. Ideally we want to create a unique missing ref that only // equals itself. @@ -615,9 +549,7 @@ impl<'a> Ctx<'a> { path, ast_id, expand_to, - call_site: span_map - .span_for_range(m.syntax().text_range()) - .map_or(SyntaxContextId::ROOT, |s| s.ctx), + call_site: span_map.span_for_range(m.syntax().text_range()).ctx, }; Some(id(self.data().macro_calls.alloc(res))) } @@ -656,15 +588,7 @@ impl<'a> Ctx<'a> { ast::ExternItem::TypeAlias(ty) => self.lower_type_alias(ty)?.into(), ast::ExternItem::MacroCall(call) => self.lower_macro_call(call)?.into(), }; - let attrs = RawAttrs::new( - self.db.upcast(), - SpanAnchor { - file_id: self.file, - ast_id: mod_item.ast_id(&self.tree).erase(), - }, - &item, - self.span_map(), - ); + let attrs = RawAttrs::new(self.db.upcast(), &item, self.span_map()); self.add_attrs(mod_item.into(), attrs); Some(mod_item) }) @@ -679,7 +603,6 @@ impl<'a> Ctx<'a> { &mut self, has_implicit_self: HasImplicitSelf, node: &dyn ast::HasGenericParams, - owner_ast_id: ErasedFileAstId, ) -> Interned { let mut generics = GenericParams::default(); @@ -701,12 +624,7 @@ impl<'a> Ctx<'a> { let add_param_attrs = |item: Either, param| { - let attrs = RawAttrs::new( - self.db.upcast(), - SpanAnchor { file_id: self.file, ast_id: owner_ast_id }, - ¶m, - self.body_ctx.span_map(), - ); + let attrs = RawAttrs::new(self.db.upcast(), ¶m, self.body_ctx.span_map()); // This is identical to the body of `Ctx::add_attrs()` but we can't call that here // because it requires `&mut self` and the call to `generics.fill()` below also // references `self`. @@ -817,7 +735,7 @@ fn lower_abi(abi: ast::Abi) -> Interned { struct UseTreeLowering<'a> { db: &'a dyn DefDatabase, - hygiene: &'a SpanMap, + hygiene: SpanMapRef<'a>, mapping: Arena, } @@ -885,7 +803,7 @@ impl UseTreeLowering<'_> { pub(crate) fn lower_use_tree( db: &dyn DefDatabase, - hygiene: &SpanMap, + hygiene: SpanMapRef<'_>, tree: ast::UseTree, ) -> Option<(UseTree, Arena)> { let mut lowering = UseTreeLowering { db, hygiene, mapping: Arena::new() }; diff --git a/crates/hir-def/src/lib.rs b/crates/hir-def/src/lib.rs index 65f1dcc85019..f9374347c264 100644 --- a/crates/hir-def/src/lib.rs +++ b/crates/hir-def/src/lib.rs @@ -75,7 +75,7 @@ use hir_expand::{ name::Name, proc_macro::ProcMacroExpander, AstId, ExpandError, ExpandResult, ExpandTo, HirFileId, InFile, MacroCallId, MacroCallKind, - MacroDefId, MacroDefKind, UnresolvedMacro, + MacroDefId, MacroDefKind, }; use item_tree::ExternBlock; use la_arena::Idx; @@ -1166,15 +1166,14 @@ impl AsMacroCall for InFile<&ast::MacroCall> { let expands_to = hir_expand::ExpandTo::from_call_site(self.value); let ast_id = AstId::new(self.file_id, db.ast_id_map(self.file_id).ast_id(self.value)); let span_map = db.span_map(self.file_id); - let path = self.value.path().and_then(|path| path::ModPath::from_src(db, path, &span_map)); + let path = + self.value.path().and_then(|path| path::ModPath::from_src(db, path, span_map.as_ref())); let Some(path) = path else { return Ok(ExpandResult::only_err(ExpandError::other("malformed macro invocation"))); }; - let call_site = span_map - .span_for_range(self.value.syntax().text_range()) - .map_or(SyntaxContextId::ROOT, |s| s.ctx); + let call_site = span_map.span_for_range(self.value.syntax().text_range()).ctx; macro_call_as_call_id_with_eager( db, @@ -1228,7 +1227,7 @@ fn macro_call_as_call_id_with_eager( let res = match def.kind { MacroDefKind::BuiltInEager(..) => { let macro_call = InFile::new(call.ast_id.file_id, call.ast_id.to_node(db)); - expand_eager_macro_input(db, krate, macro_call, def, &|path| { + expand_eager_macro_input(db, krate, macro_call, def, call_site, &|path| { eager_resolver(path).filter(MacroDefId::is_fn_like) }) } @@ -1370,6 +1369,12 @@ fn attr_macro_as_call_id( macro_attr.ctxt, ) } + +#[derive(Debug)] +pub struct UnresolvedMacro { + pub path: hir_expand::mod_path::ModPath, +} + intern::impl_internable!( crate::type_ref::TypeRef, crate::type_ref::TraitRef, diff --git a/crates/hir-def/src/lower.rs b/crates/hir-def/src/lower.rs index 28a652a60a76..a5c22898245d 100644 --- a/crates/hir-def/src/lower.rs +++ b/crates/hir-def/src/lower.rs @@ -3,7 +3,8 @@ use std::cell::OnceCell; use hir_expand::{ ast_id_map::{AstIdMap, AstIdNode}, - AstId, HirFileId, InFile, SpanMap, + span::{SpanMap, SpanMapRef}, + AstId, HirFileId, InFile, }; use syntax::ast; use triomphe::Arc; @@ -12,13 +13,13 @@ use crate::{db::DefDatabase, path::Path}; pub struct LowerCtx<'a> { pub db: &'a dyn DefDatabase, - hygiene: Arc, + hygiene: SpanMap, // FIXME: This optimization is probably pointless, ast id map should pretty much always exist anyways. ast_id_map: Option<(HirFileId, OnceCell>)>, } impl<'a> LowerCtx<'a> { - pub fn new(db: &'a dyn DefDatabase, hygiene: Arc, file_id: HirFileId) -> Self { + pub fn new(db: &'a dyn DefDatabase, hygiene: SpanMap, file_id: HirFileId) -> Self { LowerCtx { db, hygiene, ast_id_map: Some((file_id, OnceCell::new())) } } @@ -26,12 +27,12 @@ impl<'a> LowerCtx<'a> { LowerCtx { db, hygiene: db.span_map(file_id), ast_id_map: Some((file_id, OnceCell::new())) } } - pub fn with_hygiene(db: &'a dyn DefDatabase, hygiene: Arc) -> Self { + pub fn with_hygiene(db: &'a dyn DefDatabase, hygiene: SpanMap) -> Self { LowerCtx { db, hygiene, ast_id_map: None } } - pub(crate) fn span_map(&self) -> &SpanMap { - &self.hygiene + pub(crate) fn span_map(&self) -> SpanMapRef<'_> { + self.hygiene.as_ref() } pub(crate) fn lower_path(&self, ast: ast::Path) -> Option { diff --git a/crates/hir-def/src/macro_expansion_tests/mbe.rs b/crates/hir-def/src/macro_expansion_tests/mbe.rs index dcecec4e8e4e..fc17dcde9a07 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe.rs @@ -23,12 +23,9 @@ macro_rules! f { }; } -// +spans +// +spans+syntaxctxt f!(struct MyTraitMap2); "#, - // FIXME: #SpanAnchor(FileId(0), 1)@91..92\2# why is there whitespace annotated with a span - // here? Presumably because the leading `::` is getting two spans instead of one? Sounds - // liek glueing might be failing here expect![[r#" macro_rules! f { ( struct $ident:ident ) => { @@ -38,8 +35,8 @@ macro_rules! f { }; } -struct#FileId(0):1@58..64\2# MyTraitMap2#FileId(0):2@20..31\0# {#FileId(0):1@72..73\2# - map#FileId(0):1@86..89\2#:#FileId(0):1@89..90\2# #FileId(0):1@91..92\2#::#FileId(0):1@92..93\2#std#FileId(0):1@93..96\2#::#FileId(0):1@97..98\2#collections#FileId(0):1@98..109\2#::#FileId(0):1@110..111\2#HashSet#FileId(0):1@111..118\2#<#FileId(0):1@118..119\2#(#FileId(0):1@119..120\2#)#FileId(0):1@120..121\2#>#FileId(0):1@121..122\2#,#FileId(0):1@122..123\2# +struct#FileId(0):1@58..64\2# MyTraitMap2#FileId(0):2@31..42\0# {#FileId(0):1@72..73\2# + map#FileId(0):1@86..89\2#:#FileId(0):1@89..90\2# #FileId(0):1@89..90\2#::#FileId(0):1@92..93\2#std#FileId(0):1@93..96\2#::#FileId(0):1@97..98\2#collections#FileId(0):1@98..109\2#::#FileId(0):1@110..111\2#HashSet#FileId(0):1@111..118\2#<#FileId(0):1@118..119\2#(#FileId(0):1@119..120\2#)#FileId(0):1@120..121\2#>#FileId(0):1@121..122\2#,#FileId(0):1@122..123\2# }#FileId(0):1@132..133\2# "#]], ); @@ -51,14 +48,14 @@ fn token_mapping_floats() { // (and related issues) check( r#" -// +spans +// +spans+syntaxctxt macro_rules! f { ($($tt:tt)*) => { $($tt)* }; } -// +spans +// +spans+syntaxctxt f! { fn main() { 1; @@ -71,19 +68,19 @@ f! { "#, expect![[r#" -// +spans +// +spans+syntaxctxt macro_rules! f { ($($tt:tt)*) => { $($tt)* }; } -fn#FileId(0):2@19..21\0# main#FileId(0):2@22..26\0#(#FileId(0):2@26..27\0#)#FileId(0):2@27..28\0# {#FileId(0):2@29..30\0# - 1#FileId(0):2@39..40\0#;#FileId(0):2@40..41\0# - 1.0#FileId(0):2@50..53\0#;#FileId(0):2@53..54\0# - (#FileId(0):2@63..64\0#(#FileId(0):2@64..65\0#1#FileId(0):2@65..66\0#,#FileId(0):2@66..67\0# )#FileId(0):2@67..68\0#,#FileId(0):2@68..69\0# )#FileId(0):2@69..70\0#.#FileId(0):2@70..71\0#0#FileId(0):2@71..74\0#.#FileId(0):2@71..74\0#0#FileId(0):2@71..74\0#;#FileId(0):2@74..75\0# - let#FileId(0):2@84..87\0# x#FileId(0):2@88..89\0# =#FileId(0):2@90..91\0# 1#FileId(0):2@92..93\0#;#FileId(0):2@93..94\0# -}#FileId(0):2@99..100\0# +fn#FileId(0):2@30..32\0# main#FileId(0):2@33..37\0#(#FileId(0):2@37..38\0#)#FileId(0):2@38..39\0# {#FileId(0):2@40..41\0# + 1#FileId(0):2@50..51\0#;#FileId(0):2@51..52\0# + 1.0#FileId(0):2@61..64\0#;#FileId(0):2@64..65\0# + (#FileId(0):2@74..75\0#(#FileId(0):2@75..76\0#1#FileId(0):2@76..77\0#,#FileId(0):2@77..78\0# )#FileId(0):2@78..79\0#,#FileId(0):2@79..80\0# )#FileId(0):2@80..81\0#.#FileId(0):2@81..82\0#0#FileId(0):2@82..85\0#.#FileId(0):2@82..85\0#0#FileId(0):2@82..85\0#;#FileId(0):2@85..86\0# + let#FileId(0):2@95..98\0# x#FileId(0):2@99..100\0# =#FileId(0):2@101..102\0# 1#FileId(0):2@103..104\0#;#FileId(0):2@104..105\0# +}#FileId(0):2@110..111\0# "#]], @@ -127,7 +124,7 @@ macro_rules! identity { } fn main(foo: ()) { - format_args/*+spans*/!("{} {} {}", format_args!("{}", 0), foo, identity!(10), "bar") + format_args/*+spans+syntaxctxt*/!("{} {} {}", format_args!("{}", 0), foo, identity!(10), "bar") } "#, @@ -141,7 +138,7 @@ macro_rules! identity { } fn main(foo: ()) { - builtin#FileId(0):0@0..0\0# ##FileId(0):0@0..0\0#format_args#FileId(0):0@0..0\0# (#FileId(0):6@22..23\0#"{} {} {}"#FileId(0):6@23..33\0#,#FileId(0):6@33..34\0# format_args#FileId(0):6@35..46\0#!#FileId(0):6@46..47\0#(#FileId(0):6@47..48\0#"{}"#FileId(0):6@48..52\0#,#FileId(0):6@52..53\0# 0#FileId(0):6@54..55\0#)#FileId(0):6@55..56\0#,#FileId(0):6@56..57\0# foo#FileId(0):6@58..61\0#,#FileId(0):6@61..62\0# identity#FileId(0):6@63..71\0#!#FileId(0):6@71..72\0#(#FileId(0):6@72..73\0#10#FileId(0):6@73..75\0#)#FileId(0):6@75..76\0#,#FileId(0):6@76..77\0# "bar"#FileId(0):6@78..83\0#)#FileId(0):6@83..84\0# + builtin#FileId(0):0@0..0\0# ##FileId(0):0@0..0\0#format_args#FileId(0):0@0..0\0# (#FileId(0):3@56..57\0#"{} {} {}"#FileId(0):3@57..67\0#,#FileId(0):3@67..68\0# format_args#FileId(0):3@69..80\0#!#FileId(0):3@80..81\0#(#FileId(0):3@81..82\0#"{}"#FileId(0):3@82..86\0#,#FileId(0):3@86..87\0# 0#FileId(0):3@88..89\0#)#FileId(0):3@89..90\0#,#FileId(0):3@90..91\0# foo#FileId(0):3@92..95\0#,#FileId(0):3@95..96\0# identity#FileId(0):3@97..105\0#!#FileId(0):3@105..106\0#(#FileId(0):3@106..107\0#10#FileId(0):3@107..109\0#)#FileId(0):3@109..110\0#,#FileId(0):3@110..111\0# "bar"#FileId(0):3@112..117\0#)#FileId(0):3@117..118\0# } "##]], @@ -156,7 +153,7 @@ fn token_mapping_across_files() { #[macro_use] mod foo; -mk_struct/*+spans*/!(Foo with u32); +mk_struct/*+spans+syntaxctxt*/!(Foo with u32); //- /foo.rs macro_rules! mk_struct { ($foo:ident with $ty:ty) => { struct $foo($ty); } @@ -166,7 +163,7 @@ macro_rules! mk_struct { #[macro_use] mod foo; -struct#FileId(1):1@59..65\2# Foo#FileId(0):2@21..24\0#(#FileId(1):1@70..71\2#u32#FileId(0):2@30..33\0#)#FileId(1):1@74..75\2#;#FileId(1):1@75..76\2# +struct#FileId(1):1@59..65\2# Foo#FileId(0):2@32..35\0#(#FileId(1):1@70..71\2#u32#FileId(0):2@41..44\0#)#FileId(1):1@74..75\2#;#FileId(1):1@75..76\2# "#]], ); } diff --git a/crates/hir-def/src/macro_expansion_tests/mod.rs b/crates/hir-def/src/macro_expansion_tests/mod.rs index f770d2832efb..27ec63d171b0 100644 --- a/crates/hir-def/src/macro_expansion_tests/mod.rs +++ b/crates/hir-def/src/macro_expansion_tests/mod.rs @@ -18,7 +18,7 @@ use std::{iter, ops::Range, sync}; use base_db::{fixture::WithFixture, ProcMacro, SourceDatabase}; use expect_test::Expect; -use hir_expand::{db::ExpandDatabase, HirFileIdExt, InFile, MacroFile, SpanMap}; +use hir_expand::{db::ExpandDatabase, span::SpanMapRef, HirFileIdExt, InFile, MacroFile}; use stdx::format_to; use syntax::{ ast::{self, edit::IndentLevel}, @@ -104,10 +104,12 @@ pub fn identity_when_valid(_attr: TokenStream, item: TokenStream) -> TokenStream let mut tree = false; let mut expect_errors = false; let mut show_spans = false; + let mut show_ctxt = false; for comment in call.syntax().children_with_tokens().filter(|it| it.kind() == COMMENT) { tree |= comment.to_string().contains("+tree"); expect_errors |= comment.to_string().contains("+errors"); show_spans |= comment.to_string().contains("+spans"); + show_ctxt |= comment.to_string().contains("+syntaxctxt"); } let mut expn_text = String::new(); @@ -128,8 +130,12 @@ pub fn identity_when_valid(_attr: TokenStream, item: TokenStream) -> TokenStream parse.syntax_node(), ); } - let pp = - pretty_print_macro_expansion(parse.syntax_node(), show_spans.then_some(&*token_map)); + let pp = pretty_print_macro_expansion( + parse.syntax_node(), + SpanMapRef::ExpansionSpanMap(&token_map), + show_spans, + show_ctxt, + ); let indent = IndentLevel::from_node(call.syntax()); let pp = reindent(indent, pp); format_to!(expn_text, "{}", pp); @@ -169,12 +175,16 @@ pub fn identity_when_valid(_attr: TokenStream, item: TokenStream) -> TokenStream if src.file_id.is_attr_macro(&db) || src.file_id.is_custom_derive(&db) { let call = src.file_id.call_node(&db).expect("macro file"); let mut show_spans = false; + let mut show_ctxt = false; for comment in call.value.children_with_tokens().filter(|it| it.kind() == COMMENT) { show_spans |= comment.to_string().contains("+spans"); + show_ctxt |= comment.to_string().contains("+syntaxctxt"); } let pp = pretty_print_macro_expansion( src.value, - show_spans.then_some(&db.span_map(src.file_id)), + db.span_map(src.file_id).as_ref(), + show_spans, + show_ctxt, ); format_to!(expanded_text, "\n{}", pp) } @@ -184,7 +194,12 @@ pub fn identity_when_valid(_attr: TokenStream, item: TokenStream) -> TokenStream for impl_id in def_map[local_id].scope.impls() { let src = impl_id.lookup(&db).source(&db); if src.file_id.is_builtin_derive(&db) { - let pp = pretty_print_macro_expansion(src.value.syntax().clone(), None); + let pp = pretty_print_macro_expansion( + src.value.syntax().clone(), + db.span_map(src.file_id).as_ref(), + false, + false, + ); format_to!(expanded_text, "\n{}", pp) } } @@ -209,7 +224,12 @@ fn reindent(indent: IndentLevel, pp: String) -> String { res } -fn pretty_print_macro_expansion(expn: SyntaxNode, map: Option<&SpanMap>) -> String { +fn pretty_print_macro_expansion( + expn: SyntaxNode, + map: SpanMapRef<'_>, + show_spans: bool, + show_ctxt: bool, +) -> String { let mut res = String::new(); let mut prev_kind = EOF; let mut indent_level = 0; @@ -255,17 +275,22 @@ fn pretty_print_macro_expansion(expn: SyntaxNode, map: Option<&SpanMap>) -> Stri } prev_kind = curr_kind; format_to!(res, "{}", token); - if let Some(map) = map { - if let Some(span) = map.span_for_range(token.text_range()) { + if show_spans || show_ctxt { + let span = map.span_for_range(token.text_range()); + format_to!(res, "#"); + if show_spans { format_to!( res, - "#{:?}:{:?}@{:?}\\{}#", + "{:?}:{:?}@{:?}", span.anchor.file_id, span.anchor.ast_id.into_raw(), span.range, - span.ctx ); } + if show_ctxt { + format_to!(res, "\\{}", span.ctx); + } + format_to!(res, "#"); } } res diff --git a/crates/hir-def/src/macro_expansion_tests/proc_macros.rs b/crates/hir-def/src/macro_expansion_tests/proc_macros.rs index 29374945f647..548f22499b31 100644 --- a/crates/hir-def/src/macro_expansion_tests/proc_macros.rs +++ b/crates/hir-def/src/macro_expansion_tests/proc_macros.rs @@ -74,6 +74,7 @@ fn foo() { } #[test] +#[ignore] // TODO fn attribute_macro_syntax_completion_2() { // common case of dot completion while typing check( @@ -168,21 +169,21 @@ fn float_attribute_mapping() { check( r#" //- proc_macros: identity -//+spans +//+spans+syntaxctxt #[proc_macros::identity] fn foo(&self) { self.0. 1; } "#, expect![[r#" -//+spans +//+spans+syntaxctxt #[proc_macros::identity] fn foo(&self) { self.0. 1; } -fn#FileId(0):1@34..36\0# foo#FileId(0):1@37..40\0#(#FileId(0):1@40..41\0#&#FileId(0):1@41..42\0#self#FileId(0):1@42..46\0# )#FileId(0):1@46..47\0# {#FileId(0):1@48..49\0# - self#FileId(0):1@54..58\0# .#FileId(0):1@58..59\0#0#FileId(0):1@59..60\0#.#FileId(0):1@60..61\0#1#FileId(0):1@62..63\0#;#FileId(0):1@63..64\0# -}#FileId(0):1@65..66\0#"#]], +fn#FileId(0):1@45..47\0# foo#FileId(0):1@48..51\0#(#FileId(0):1@51..52\0#&#FileId(0):1@52..53\0#self#FileId(0):1@53..57\0# )#FileId(0):1@57..58\0# {#FileId(0):1@59..60\0# + self#FileId(0):1@65..69\0# .#FileId(0):1@69..70\0#0#FileId(0):1@70..71\0#.#FileId(0):1@71..72\0#1#FileId(0):1@73..74\0#;#FileId(0):1@74..75\0# +}#FileId(0):1@76..77\0#"#]], ); } diff --git a/crates/hir-def/src/nameres/collector.rs b/crates/hir-def/src/nameres/collector.rs index fef13604225a..599010e542ca 100644 --- a/crates/hir-def/src/nameres/collector.rs +++ b/crates/hir-def/src/nameres/collector.rs @@ -1219,7 +1219,7 @@ impl DefCollector<'_> { }; if matches!( def, - MacroDefId { kind:MacroDefKind::BuiltInAttr(expander, _),.. } + MacroDefId { kind: MacroDefKind::BuiltInAttr(expander, _),.. } if expander.is_derive() ) { // Resolved to `#[derive]` diff --git a/crates/hir-def/src/path/lower.rs b/crates/hir-def/src/path/lower.rs index ee49dfa44c80..9b499c5619ef 100644 --- a/crates/hir-def/src/path/lower.rs +++ b/crates/hir-def/src/path/lower.rs @@ -4,7 +4,6 @@ use std::iter; use crate::{lower::LowerCtx, type_ref::ConstRef}; -use base_db::span::SyntaxContextId; use hir_expand::{ mod_path::resolve_crate_root, name::{name, AsName}, @@ -40,11 +39,11 @@ pub(super) fn lower_path(mut path: ast::Path, ctx: &LowerCtx<'_>) -> Option) -> Option PathKind::DollarCrate(crate_root), + None => PathKind::Crate, } } } diff --git a/crates/hir-def/src/visibility.rs b/crates/hir-def/src/visibility.rs index b341b8cfbd96..2bf02b49c247 100644 --- a/crates/hir-def/src/visibility.rs +++ b/crates/hir-def/src/visibility.rs @@ -2,7 +2,7 @@ use std::iter; -use hir_expand::{InFile, SpanMap}; +use hir_expand::{span::SpanMapRef, InFile}; use la_arena::ArenaMap; use syntax::ast; use triomphe::Arc; @@ -34,13 +34,13 @@ impl RawVisibility { db: &dyn DefDatabase, node: InFile>, ) -> RawVisibility { - Self::from_ast_with_hygiene(db, node.value, &db.span_map(node.file_id)) + Self::from_ast_with_hygiene(db, node.value, db.span_map(node.file_id).as_ref()) } pub(crate) fn from_ast_with_hygiene( db: &dyn DefDatabase, node: Option, - hygiene: &SpanMap, + hygiene: SpanMapRef<'_>, ) -> RawVisibility { Self::from_ast_with_hygiene_and_default(db, node, RawVisibility::private(), hygiene) } @@ -49,7 +49,7 @@ impl RawVisibility { db: &dyn DefDatabase, node: Option, default: RawVisibility, - hygiene: &SpanMap, + hygiene: SpanMapRef<'_>, ) -> RawVisibility { let node = match node { None => return default, diff --git a/crates/hir-expand/src/ast_id_map.rs b/crates/hir-expand/src/ast_id_map.rs index eb43ae37e08a..2d24496ab701 100644 --- a/crates/hir-expand/src/ast_id_map.rs +++ b/crates/hir-expand/src/ast_id_map.rs @@ -19,6 +19,33 @@ use syntax::{ast, AstNode, AstPtr, SyntaxNode, SyntaxNodePtr}; pub use base_db::span::ErasedFileAstId; +use crate::db; + +/// `AstId` points to an AST node in any file. +/// +/// It is stable across reparses, and can be used as salsa key/value. +pub type AstId = crate::InFile>; + +impl AstId { + pub fn to_node(&self, db: &dyn db::ExpandDatabase) -> N { + self.to_ptr(db).to_node(&db.parse_or_expand(self.file_id)) + } + pub fn to_in_file_node(&self, db: &dyn db::ExpandDatabase) -> crate::InFile { + crate::InFile::new(self.file_id, self.to_ptr(db).to_node(&db.parse_or_expand(self.file_id))) + } + pub fn to_ptr(&self, db: &dyn db::ExpandDatabase) -> AstPtr { + db.ast_id_map(self.file_id).get(self.value) + } +} + +pub type ErasedAstId = crate::InFile; + +impl ErasedAstId { + pub fn to_ptr(&self, db: &dyn db::ExpandDatabase) -> SyntaxNodePtr { + db.ast_id_map(self.file_id).get_erased(self.value) + } +} + /// `AstId` points to an AST node in a specific file. pub struct FileAstId { raw: ErasedFileAstId, @@ -141,9 +168,9 @@ impl AstIdMap { bdfs(node, |it| { if should_alloc_id(it.kind()) { res.alloc(&it); - true + TreeOrder::BreadthFirst } else { - false + TreeOrder::DepthFirst } }); res.map = hashbrown::HashMap::with_capacity_and_hasher(res.arena.len(), ()); @@ -174,7 +201,7 @@ impl AstIdMap { AstPtr::try_from_raw(self.arena[id.raw].clone()).unwrap() } - pub fn get_raw(&self, id: ErasedFileAstId) -> SyntaxNodePtr { + pub fn get_erased(&self, id: ErasedFileAstId) -> SyntaxNodePtr { self.arena[id].clone() } @@ -202,14 +229,20 @@ fn hash_ptr(ptr: &SyntaxNodePtr) -> u64 { hasher.finish() } +#[derive(Copy, Clone, PartialEq, Eq)] +enum TreeOrder { + BreadthFirst, + DepthFirst, +} + /// Walks the subtree in bdfs order, calling `f` for each node. What is bdfs /// order? It is a mix of breadth-first and depth first orders. Nodes for which -/// `f` returns true are visited breadth-first, all the other nodes are explored -/// depth-first. +/// `f` returns [`TreeOrder::BreadthFirst`] are visited breadth-first, all the other nodes are explored +/// [`TreeOrder::DepthFirst`]. /// /// In other words, the size of the bfs queue is bound by the number of "true" /// nodes. -fn bdfs(node: &SyntaxNode, mut f: impl FnMut(SyntaxNode) -> bool) { +fn bdfs(node: &SyntaxNode, mut f: impl FnMut(SyntaxNode) -> TreeOrder) { let mut curr_layer = vec![node.clone()]; let mut next_layer = vec![]; while !curr_layer.is_empty() { @@ -218,7 +251,7 @@ fn bdfs(node: &SyntaxNode, mut f: impl FnMut(SyntaxNode) -> bool) { while let Some(event) = preorder.next() { match event { syntax::WalkEvent::Enter(node) => { - if f(node.clone()) { + if f(node.clone()) == TreeOrder::BreadthFirst { next_layer.extend(node.children()); preorder.skip_subtree(); } diff --git a/crates/hir-expand/src/attrs.rs b/crates/hir-expand/src/attrs.rs index 76c787721b55..c4937ae08b1e 100644 --- a/crates/hir-expand/src/attrs.rs +++ b/crates/hir-expand/src/attrs.rs @@ -1,11 +1,7 @@ //! A higher level attributes based on TokenTree, with also some shortcuts. use std::{fmt, ops}; -use ::tt::SpanAnchor as _; -use base_db::{ - span::{SpanAnchor, SyntaxContextId}, - CrateId, -}; +use base_db::{span::SyntaxContextId, CrateId}; use cfg::CfgExpr; use either::Either; use intern::Interned; @@ -17,8 +13,9 @@ use triomphe::Arc; use crate::{ db::ExpandDatabase, mod_path::ModPath, + span::SpanMapRef, tt::{self, Subtree}, - InFile, SpanMap, + InFile, }; /// Syntactical attributes, without filtering of `cfg_attr`s. @@ -44,22 +41,19 @@ impl RawAttrs { pub fn new( db: &dyn ExpandDatabase, - span_anchor: SpanAnchor, owner: &dyn ast::HasAttrs, - hygiene: &SpanMap, + hygiene: SpanMapRef<'_>, ) -> Self { let entries = collect_attrs(owner) .filter_map(|(id, attr)| match attr { Either::Left(attr) => { - attr.meta().and_then(|meta| Attr::from_src(db, span_anchor, meta, hygiene, id)) + attr.meta().and_then(|meta| Attr::from_src(db, meta, hygiene, id)) } Either::Right(comment) => comment.doc_comment().map(|doc| Attr { id, input: Some(Interned::new(AttrInput::Literal(SmolStr::new(doc)))), path: Interned::new(ModPath::from(crate::name!(doc))), - ctxt: hygiene - .span_for_range(comment.syntax().text_range()) - .map_or(SyntaxContextId::ROOT, |s| s.ctx), + ctxt: hygiene.span_for_range(comment.syntax().text_range()).ctx, }), }) .collect::>(); @@ -71,10 +65,10 @@ impl RawAttrs { pub fn from_attrs_owner( db: &dyn ExpandDatabase, - span_anchor: SpanAnchor, owner: InFile<&dyn ast::HasAttrs>, + hygiene: SpanMapRef<'_>, ) -> Self { - Self::new(db, span_anchor, owner.value, &db.span_map(owner.file_id)) + Self::new(db, owner.value, hygiene) } pub fn merge(&self, other: Self) -> Self { @@ -221,9 +215,8 @@ impl fmt::Display for AttrInput { impl Attr { fn from_src( db: &dyn ExpandDatabase, - span_anchor: SpanAnchor, ast: ast::Meta, - hygiene: &SpanMap, + hygiene: SpanMapRef<'_>, id: AttrId, ) -> Option { let path = Interned::new(ModPath::from_src(db, ast.path()?, hygiene)?); @@ -234,31 +227,20 @@ impl Attr { }; Some(Interned::new(AttrInput::Literal(value))) } else if let Some(tt) = ast.token_tree() { - // FIXME: We could also allocate ids for attributes and use the attribute itself as an anchor - let offset = - db.ast_id_map(span_anchor.file_id).get_raw(span_anchor.ast_id).text_range().start(); - let tree = syntax_node_to_token_tree(tt.syntax(), span_anchor, offset, hygiene); + let tree = syntax_node_to_token_tree(tt.syntax(), hygiene); Some(Interned::new(AttrInput::TokenTree(Box::new(tree)))) } else { None }; - Some(Attr { - id, - path, - input, - ctxt: hygiene - .span_for_range(ast.syntax().text_range()) - .map_or(SyntaxContextId::ROOT, |s| s.ctx), - }) + Some(Attr { id, path, input, ctxt: hygiene.span_for_range(ast.syntax().text_range()).ctx }) } fn from_tt(db: &dyn ExpandDatabase, tt: &tt::Subtree, id: AttrId) -> Option { // FIXME: Unecessary roundtrip tt -> ast -> tt - let (parse, _map) = mbe::token_tree_to_syntax_node(tt, mbe::TopEntryPoint::MetaItem); + let (parse, map) = mbe::token_tree_to_syntax_node(tt, mbe::TopEntryPoint::MetaItem); let ast = ast::Meta::cast(parse.syntax_node())?; - // FIXME: we discard spans here! - Self::from_src(db, SpanAnchor::DUMMY, ast, &SpanMap::default(), id) + Self::from_src(db, ast, SpanMapRef::ExpansionSpanMap(&map), id) } pub fn path(&self) -> &ModPath { @@ -331,7 +313,10 @@ impl Attr { return None; } let path = meta.path()?; - Some((ModPath::from_src(db, path, &span_map)?, call_site)) + Some(( + ModPath::from_src(db, path, SpanMapRef::ExpansionSpanMap(&span_map))?, + call_site, + )) }); Some(paths) diff --git a/crates/hir-expand/src/builtin_attr_macro.rs b/crates/hir-expand/src/builtin_attr_macro.rs index de8c0ac810bf..c16b881df826 100644 --- a/crates/hir-expand/src/builtin_attr_macro.rs +++ b/crates/hir-expand/src/builtin_attr_macro.rs @@ -79,9 +79,8 @@ fn dummy_attr_expand( /// /// As such, we expand `#[derive(Foo, bar::Bar)]` into /// ``` -/// #[Foo] -/// #[bar::Bar] -/// (); +/// #![Foo] +/// #![bar::Bar] /// ``` /// which allows fallback path resolution in hir::Semantics to properly identify our derives. /// Since we do not expand the attribute in nameres though, we keep the original item. @@ -124,12 +123,10 @@ pub fn pseudo_derive_attr_expansion( .split(|tt| matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', .. })))) { token_trees.push(mk_leaf('#')); + token_trees.push(mk_leaf('!')); token_trees.push(mk_leaf('[')); token_trees.extend(tt.iter().cloned()); token_trees.push(mk_leaf(']')); } - token_trees.push(mk_leaf('(')); - token_trees.push(mk_leaf(')')); - token_trees.push(mk_leaf(';')); ExpandResult::ok(tt::Subtree { delimiter: tt.delimiter, token_trees }) } diff --git a/crates/hir-expand/src/builtin_derive_macro.rs b/crates/hir-expand/src/builtin_derive_macro.rs index 16cce35c13d9..e9d137d99007 100644 --- a/crates/hir-expand/src/builtin_derive_macro.rs +++ b/crates/hir-expand/src/builtin_derive_macro.rs @@ -1,20 +1,19 @@ //! Builtin derives. use ::tt::Span; -use base_db::{CrateOrigin, LangCrateOrigin}; +use base_db::{span::SpanData, CrateOrigin, LangCrateOrigin}; use itertools::izip; use rustc_hash::FxHashSet; use stdx::never; use tracing::debug; use crate::{ + hygiene::span_with_def_site_ctxt, name::{AsName, Name}, - tt, SpanMap, -}; -use syntax::{ - ast::{self, AstNode, FieldList, HasAttrs, HasGenericParams, HasName, HasTypeBounds}, - TextSize, + span::SpanMapRef, + tt, }; +use syntax::ast::{self, AstNode, FieldList, HasAttrs, HasGenericParams, HasName, HasTypeBounds}; use crate::{db::ExpandDatabase, name, quote, ExpandError, ExpandResult, MacroCallId}; @@ -31,12 +30,15 @@ macro_rules! register_builtin { db: &dyn ExpandDatabase, id: MacroCallId, tt: &ast::Adt, - token_map: &SpanMap, + token_map: SpanMapRef<'_>, ) -> ExpandResult { let expander = match *self { $( BuiltinDeriveExpander::$trait => $expand, )* }; - expander(db, id, tt, token_map) + + let span = db.lookup_intern_macro_call(id).span(db); + let span = span_with_def_site_ctxt(db, span, id); + expander(db, id, span, tt, token_map) } fn find_by_name(name: &name::Name) -> Option { @@ -119,7 +121,7 @@ impl VariantShape { } } - fn from(tm: &SpanMap, value: Option) -> Result { + fn from(tm: SpanMapRef<'_>, value: Option) -> Result { let r = match value { None => VariantShape::Unit, Some(FieldList::RecordFieldList(it)) => VariantShape::Struct( @@ -191,7 +193,7 @@ struct BasicAdtInfo { associated_types: Vec, } -fn parse_adt(tm: &SpanMap, adt: &ast::Adt) -> Result { +fn parse_adt(tm: SpanMapRef<'_>, adt: &ast::Adt) -> Result { let (name, generic_param_list, shape) = match adt { ast::Adt::Struct(it) => ( it.name(), @@ -236,44 +238,21 @@ fn parse_adt(tm: &SpanMap, adt: &ast::Adt) -> Result match this { Some(it) => { param_type_set.insert(it.as_name()); - mbe::syntax_node_to_token_tree( - it.syntax(), - tm.span_for_range(it.syntax().first_token().unwrap().text_range()) - .unwrap() - .anchor, - TextSize::from(0), - tm, - ) + mbe::syntax_node_to_token_tree(it.syntax(), tm) } None => tt::Subtree::empty(), } }; let bounds = match ¶m { - ast::TypeOrConstParam::Type(it) => it.type_bound_list().map(|it| { - mbe::syntax_node_to_token_tree( - it.syntax(), - tm.span_for_range(it.syntax().first_token().unwrap().text_range()) - .unwrap() - .anchor, - TextSize::from(0), - tm, - ) - }), + ast::TypeOrConstParam::Type(it) => { + it.type_bound_list().map(|it| mbe::syntax_node_to_token_tree(it.syntax(), tm)) + } ast::TypeOrConstParam::Const(_) => None, }; let ty = if let ast::TypeOrConstParam::Const(param) = param { let ty = param .ty() - .map(|ty| { - mbe::syntax_node_to_token_tree( - ty.syntax(), - tm.span_for_range(ty.syntax().first_token().unwrap().text_range()) - .unwrap() - .anchor, - TextSize::from(0), - tm, - ) - }) + .map(|ty| mbe::syntax_node_to_token_tree(ty.syntax(), tm)) .unwrap_or_else(tt::Subtree::empty); Some(ty) } else { @@ -307,25 +286,21 @@ fn parse_adt(tm: &SpanMap, adt: &ast::Adt) -> Result let name = p.path()?.qualifier()?.as_single_name_ref()?.as_name(); param_type_set.contains(&name).then_some(p) }) - .map(|it| { - mbe::syntax_node_to_token_tree( - it.syntax(), - tm.span_for_range(it.syntax().first_token().unwrap().text_range()).unwrap().anchor, - TextSize::from(0), - tm, - ) - }) + .map(|it| mbe::syntax_node_to_token_tree(it.syntax(), tm)) .collect(); - let name_token = name_to_token(&tm, name)?; + let name_token = name_to_token(tm, name)?; Ok(BasicAdtInfo { name: name_token, shape, param_types, associated_types }) } -fn name_to_token(token_map: &SpanMap, name: Option) -> Result { +fn name_to_token( + token_map: SpanMapRef<'_>, + name: Option, +) -> Result { let name = name.ok_or_else(|| { debug!("parsed item has no name"); ExpandError::other("missing name") })?; - let span = token_map.span_for_range(name.syntax().text_range()).unwrap(); + let span = token_map.span_for_range(name.syntax().text_range()); let name_token = tt::Ident { span, text: name.text().into() }; Ok(name_token) } @@ -362,8 +337,10 @@ fn name_to_token(token_map: &SpanMap, name: Option) -> Result, trait_path: tt::Subtree, make_trait_body: impl FnOnce(&BasicAdtInfo) -> tt::Subtree, ) -> ExpandResult { @@ -423,21 +400,23 @@ fn find_builtin_crate(db: &dyn ExpandDatabase, id: MacroCallId) -> tt::TokenTree fn copy_expand( db: &dyn ExpandDatabase, id: MacroCallId, + span: SpanData, tt: &ast::Adt, - tm: &SpanMap, + tm: SpanMapRef<'_>, ) -> ExpandResult { let krate = find_builtin_crate(db, id); - expand_simple_derive(tt, tm, quote! { #krate::marker::Copy }, |_| quote! {}) + expand_simple_derive(span, tt, tm, quote! { #krate::marker::Copy }, |_| quote! {}) } fn clone_expand( db: &dyn ExpandDatabase, id: MacroCallId, + span: SpanData, tt: &ast::Adt, - tm: &SpanMap, + tm: SpanMapRef<'_>, ) -> ExpandResult { let krate = find_builtin_crate(db, id); - expand_simple_derive(tt, tm, quote! { #krate::clone::Clone }, |adt| { + expand_simple_derive(span, tt, tm, quote! { #krate::clone::Clone }, |adt| { if matches!(adt.shape, AdtShape::Union) { let star = tt::Punct { char: '*', spacing: ::tt::Spacing::Alone, span: tt::SpanData::DUMMY }; @@ -491,11 +470,12 @@ fn and_and() -> tt::Subtree { fn default_expand( db: &dyn ExpandDatabase, id: MacroCallId, + span: SpanData, tt: &ast::Adt, - tm: &SpanMap, + tm: SpanMapRef<'_>, ) -> ExpandResult { let krate = &find_builtin_crate(db, id); - expand_simple_derive(tt, tm, quote! { #krate::default::Default }, |adt| { + expand_simple_derive(span, tt, tm, quote! { #krate::default::Default }, |adt| { let body = match &adt.shape { AdtShape::Struct(fields) => { let name = &adt.name; @@ -531,11 +511,12 @@ fn default_expand( fn debug_expand( db: &dyn ExpandDatabase, id: MacroCallId, + span: SpanData, tt: &ast::Adt, - tm: &SpanMap, + tm: SpanMapRef<'_>, ) -> ExpandResult { let krate = &find_builtin_crate(db, id); - expand_simple_derive(tt, tm, quote! { #krate::fmt::Debug }, |adt| { + expand_simple_derive(span, tt, tm, quote! { #krate::fmt::Debug }, |adt| { let for_variant = |name: String, v: &VariantShape| match v { VariantShape::Struct(fields) => { let for_fields = fields.iter().map(|it| { @@ -609,11 +590,12 @@ fn debug_expand( fn hash_expand( db: &dyn ExpandDatabase, id: MacroCallId, + span: SpanData, tt: &ast::Adt, - tm: &SpanMap, + tm: SpanMapRef<'_>, ) -> ExpandResult { let krate = &find_builtin_crate(db, id); - expand_simple_derive(tt, tm, quote! { #krate::hash::Hash }, |adt| { + expand_simple_derive(span, tt, tm, quote! { #krate::hash::Hash }, |adt| { if matches!(adt.shape, AdtShape::Union) { // FIXME: Return expand error here return quote! {}; @@ -660,21 +642,23 @@ fn hash_expand( fn eq_expand( db: &dyn ExpandDatabase, id: MacroCallId, + span: SpanData, tt: &ast::Adt, - tm: &SpanMap, + tm: SpanMapRef<'_>, ) -> ExpandResult { let krate = find_builtin_crate(db, id); - expand_simple_derive(tt, tm, quote! { #krate::cmp::Eq }, |_| quote! {}) + expand_simple_derive(span, tt, tm, quote! { #krate::cmp::Eq }, |_| quote! {}) } fn partial_eq_expand( db: &dyn ExpandDatabase, id: MacroCallId, + span: SpanData, tt: &ast::Adt, - tm: &SpanMap, + tm: SpanMapRef<'_>, ) -> ExpandResult { let krate = find_builtin_crate(db, id); - expand_simple_derive(tt, tm, quote! { #krate::cmp::PartialEq }, |adt| { + expand_simple_derive(span, tt, tm, quote! { #krate::cmp::PartialEq }, |adt| { if matches!(adt.shape, AdtShape::Union) { // FIXME: Return expand error here return quote! {}; @@ -738,11 +722,12 @@ fn self_and_other_patterns( fn ord_expand( db: &dyn ExpandDatabase, id: MacroCallId, + span: SpanData, tt: &ast::Adt, - tm: &SpanMap, + tm: SpanMapRef<'_>, ) -> ExpandResult { let krate = &find_builtin_crate(db, id); - expand_simple_derive(tt, tm, quote! { #krate::cmp::Ord }, |adt| { + expand_simple_derive(span, tt, tm, quote! { #krate::cmp::Ord }, |adt| { fn compare( krate: &tt::TokenTree, left: tt::Subtree, @@ -800,11 +785,12 @@ fn ord_expand( fn partial_ord_expand( db: &dyn ExpandDatabase, id: MacroCallId, + span: SpanData, tt: &ast::Adt, - tm: &SpanMap, + tm: SpanMapRef<'_>, ) -> ExpandResult { let krate = &find_builtin_crate(db, id); - expand_simple_derive(tt, tm, quote! { #krate::cmp::PartialOrd }, |adt| { + expand_simple_derive(span, tt, tm, quote! { #krate::cmp::PartialOrd }, |adt| { fn compare( krate: &tt::TokenTree, left: tt::Subtree, diff --git a/crates/hir-expand/src/builtin_fn_macro.rs b/crates/hir-expand/src/builtin_fn_macro.rs index 2a541a36735b..9fb04a2f1b08 100644 --- a/crates/hir-expand/src/builtin_fn_macro.rs +++ b/crates/hir-expand/src/builtin_fn_macro.rs @@ -556,9 +556,10 @@ pub(crate) fn include_arg_to_tt( let path = parse_string(&arg.0)?; let file_id = relative_file(db, *arg_id, &path, false)?; + // why are we not going through a SyntaxNode here? let subtree = parse_to_token_tree( + SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, &db.file_text(file_id), - SpanAnchor { file_id: file_id.into(), ast_id: ROOT_ERASED_FILE_AST_ID }, ) .ok_or(mbe::ExpandError::ConversionError)?; Ok((triomphe::Arc::new(subtree), file_id)) diff --git a/crates/hir-expand/src/db.rs b/crates/hir-expand/src/db.rs index e176bef78b5a..393e391f0517 100644 --- a/crates/hir-expand/src/db.rs +++ b/crates/hir-expand/src/db.rs @@ -3,15 +3,15 @@ use ::tt::{SpanAnchor as _, SyntaxContext}; use base_db::{ salsa, - span::{SpanAnchor, SyntaxContextId, ROOT_ERASED_FILE_AST_ID}, - CrateId, Edition, SourceDatabase, + span::{SpanAnchor, SyntaxContextId}, + CrateId, Edition, FileId, SourceDatabase, }; use either::Either; use limit::Limit; -use mbe::{map_from_syntax_node, syntax_node_to_token_tree, ValueResult}; +use mbe::{syntax_node_to_token_tree, ValueResult}; use syntax::{ ast::{self, HasAttrs, HasDocComments}, - AstNode, Parse, SyntaxError, SyntaxNode, SyntaxToken, TextSize, T, + AstNode, Parse, SyntaxError, SyntaxNode, SyntaxToken, T, }; use triomphe::Arc; @@ -21,9 +21,10 @@ use crate::{ builtin_attr_macro::pseudo_derive_attr_expansion, builtin_fn_macro::EagerExpander, hygiene::{self, SyntaxContextData, Transparency}, + span::{RealSpanMap, SpanMap, SpanMapRef}, tt, AstId, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander, EagerCallInfo, - ExpandError, ExpandResult, ExpandTo, HirFileId, HirFileIdRepr, MacroCallId, MacroCallKind, - MacroCallLoc, MacroDefId, MacroDefKind, MacroFile, ProcMacroExpander, SpanMap, + ExpandError, ExpandResult, ExpandTo, ExpansionSpanMap, HirFileId, HirFileIdRepr, MacroCallId, + MacroCallKind, MacroCallLoc, MacroDefId, MacroDefKind, MacroFile, ProcMacroExpander, }; /// Total limit on the number of tokens produced by any macro invocation. @@ -102,10 +103,11 @@ pub trait ExpandDatabase: SourceDatabase { fn parse_macro_expansion( &self, macro_file: MacroFile, - ) -> ExpandResult<(Parse, Arc)>; - // FIXME: This always allocates one for non macro files which is wasteful. + ) -> ExpandResult<(Parse, Arc)>; #[salsa::transparent] - fn span_map(&self, file_id: HirFileId) -> Arc; + fn span_map(&self, file_id: HirFileId) -> SpanMap; + + fn real_span_map(&self, file_id: FileId) -> Arc; /// Macro ids. That's probably the tricksiest bit in rust-analyzer, and the /// reason why we use salsa at all. @@ -164,13 +166,20 @@ pub trait ExpandDatabase: SourceDatabase { ) -> ExpandResult>; } -fn span_map(db: &dyn ExpandDatabase, file_id: HirFileId) -> Arc { +#[inline] +pub fn span_map(db: &dyn ExpandDatabase, file_id: HirFileId) -> SpanMap { match file_id.repr() { - HirFileIdRepr::FileId(_) => Arc::new(Default::default()), - HirFileIdRepr::MacroFile(m) => db.parse_macro_expansion(m).value.1, + HirFileIdRepr::FileId(file_id) => SpanMap::RealSpanMap(db.real_span_map(file_id)), + HirFileIdRepr::MacroFile(m) => { + SpanMap::ExpansionSpanMap(db.parse_macro_expansion(m).value.1) + } } } +pub fn real_span_map(db: &dyn ExpandDatabase, file_id: FileId) -> Arc { + Arc::new(RealSpanMap::from_file(db, file_id)) +} + /// This expands the given macro call, but with different arguments. This is /// used for completion, where we want to see what 'would happen' if we insert a /// token. The `token_to_map` mapped down into the expansion, with the mapped @@ -181,17 +190,15 @@ pub fn expand_speculative( speculative_args: &SyntaxNode, token_to_map: SyntaxToken, ) -> Option<(SyntaxNode, SyntaxToken)> { + // FIXME spanmaps let loc = db.lookup_intern_macro_call(actual_macro_call); - let file_id = loc.kind.file_id(); // Build the subtree and token mapping for the speculative args let _censor = censor_for_macro_input(&loc, speculative_args); let mut tt = mbe::syntax_node_to_token_tree( speculative_args, // we don't leak these spans into any query so its fine to make them absolute - SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, - TextSize::new(0), - &Default::default(), + SpanMapRef::RealSpanMap(&RealSpanMap::empty(SpanAnchor::DUMMY.file_id)), ); let attr_arg = match loc.kind { @@ -211,9 +218,7 @@ pub fn expand_speculative( Some(token_tree) => { let mut tree = syntax_node_to_token_tree( token_tree.syntax(), - SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, - TextSize::new(0), - &Default::default(), + SpanMapRef::RealSpanMap(&RealSpanMap::empty(SpanAnchor::DUMMY.file_id)), ); tree.delimiter = tt::Delimiter::UNSPECIFIED; @@ -242,12 +247,7 @@ pub fn expand_speculative( db, actual_macro_call, &adt, - &map_from_syntax_node( - speculative_args, - // we don't leak these spans into any query so its fine to make them absolute - SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, - TextSize::new(0), - ), + SpanMapRef::RealSpanMap(&RealSpanMap::empty(SpanAnchor::DUMMY.file_id)), ) } MacroDefKind::Declarative(it) => { @@ -261,15 +261,13 @@ pub fn expand_speculative( }; let expand_to = macro_expand_to(db, actual_macro_call); - let (node, mut rev_tmap) = - token_tree_to_syntax_node(db, &speculative_expansion.value, expand_to); - rev_tmap.real_file = false; + let (node, rev_tmap) = token_tree_to_syntax_node(db, &speculative_expansion.value, expand_to); let syntax_node = node.syntax_node(); let token = rev_tmap .ranges_with_span(tt::SpanData { range: token_to_map.text_range(), - anchor: SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, + anchor: SpanAnchor::DUMMY, ctx: SyntaxContextId::DUMMY, }) .filter_map(|range| syntax_node.covering_element(range).into_token()) @@ -310,7 +308,7 @@ fn parse_or_expand_with_err( fn parse_macro_expansion( db: &dyn ExpandDatabase, macro_file: MacroFile, -) -> ExpandResult<(Parse, Arc)> { +) -> ExpandResult<(Parse, Arc)> { let _p = profile::span("parse_macro_expansion"); let mbe::ValueResult { value: tt, err } = db.macro_expand(macro_file.macro_call_id); @@ -319,8 +317,7 @@ fn parse_macro_expansion( tracing::debug!("expanded = {}", tt.as_debug_string()); tracing::debug!("kind = {:?}", expand_to); - let (parse, mut rev_token_map) = token_tree_to_syntax_node(db, &tt, expand_to); - rev_token_map.real_file = false; + let (parse, rev_token_map) = token_tree_to_syntax_node(db, &tt, expand_to); ExpandResult { value: (parse, Arc::new(rev_token_map)), err } } @@ -366,18 +363,21 @@ fn macro_arg( { ValueResult::ok(Some(Arc::new(arg.0.clone()))) } else { + //FIXME: clean this up, the ast id map lookup is done twice here let (parse, map) = match loc.kind.file_id().repr() { HirFileIdRepr::FileId(file_id) => { - (db.parse(file_id).to_syntax(), Arc::new(Default::default())) + let syntax = db.parse(file_id).to_syntax(); + + (syntax, SpanMap::RealSpanMap(db.real_span_map(file_id))) } HirFileIdRepr::MacroFile(macro_file) => { let (parse, map) = db.parse_macro_expansion(macro_file).value; - (parse, map) + (parse, SpanMap::ExpansionSpanMap(map)) } }; let root = parse.syntax_node(); - let (syntax, offset, ast_id) = match loc.kind { + let syntax = match loc.kind { MacroCallKind::FnLike { ast_id, .. } => { let node = &ast_id.to_ptr(db).to_node(&root); let offset = node.syntax().text_range().start(); @@ -386,7 +386,7 @@ fn macro_arg( if let Some(e) = mismatched_delimiters(&tt) { return ValueResult::only_err(e); } - (tt, offset, ast_id.value.erase()) + tt } None => { return ValueResult::only_err(Arc::new(Box::new([ @@ -396,15 +396,9 @@ fn macro_arg( } } MacroCallKind::Derive { ast_id, .. } => { - let syntax_node = ast_id.to_ptr(db).to_node(&root).syntax().clone(); - let offset = syntax_node.text_range().start(); - (syntax_node, offset, ast_id.value.erase()) - } - MacroCallKind::Attr { ast_id, .. } => { - let syntax_node = ast_id.to_ptr(db).to_node(&root).syntax().clone(); - let offset = syntax_node.text_range().start(); - (syntax_node, offset, ast_id.value.erase()) + ast_id.to_ptr(db).to_node(&root).syntax().clone() } + MacroCallKind::Attr { ast_id, .. } => ast_id.to_ptr(db).to_node(&root).syntax().clone(), }; let censor = censor_for_macro_input(&loc, &syntax); // let mut fixups = fixup::fixup_syntax(&node); @@ -416,13 +410,8 @@ fn macro_arg( // fixups.replace, // fixups.append, // ); - let mut tt = mbe::syntax_node_to_token_tree_censored( - &syntax, - SpanAnchor { file_id: loc.kind.file_id(), ast_id }, - offset, - &map, - censor, - ); + + let mut tt = mbe::syntax_node_to_token_tree_censored(&syntax, map.as_ref(), censor); if loc.def.is_proc_macro() { // proc macros expect their inputs without parentheses, MBEs expect it with them included @@ -492,18 +481,19 @@ fn decl_macro_expander( let is_2021 = db.crate_graph()[def_crate].edition >= Edition::Edition2021; let (root, map) = match id.file_id.repr() { HirFileIdRepr::FileId(file_id) => { - (db.parse(file_id).syntax_node(), Arc::new(Default::default())) + // FIXME: Arc + // FIXME: id.to_ptr duplicated, expensive + (db.parse(file_id).syntax_node(), SpanMap::RealSpanMap(db.real_span_map(file_id))) } HirFileIdRepr::MacroFile(macro_file) => { let (parse, map) = db.parse_macro_expansion(macro_file).value; - (parse.syntax_node(), map) + (parse.syntax_node(), SpanMap::ExpansionSpanMap(map)) } }; let transparency = |node| { // ... would be nice to have the item tree here - let attrs = - RawAttrs::new(db, SpanAnchor::DUMMY, node, &Default::default()).filter(db, def_crate); + let attrs = RawAttrs::new(db, node, map.as_ref()).filter(db, def_crate); match &*attrs .iter() .find(|it| { @@ -526,12 +516,7 @@ fn decl_macro_expander( ast::Macro::MacroRules(macro_rules) => ( match macro_rules.token_tree() { Some(arg) => { - let tt = mbe::syntax_node_to_token_tree( - arg.syntax(), - SpanAnchor { file_id: id.file_id, ast_id: id.value.erase() }, - macro_rules.syntax().text_range().start(), - &map, - ); + let tt = mbe::syntax_node_to_token_tree(arg.syntax(), map.as_ref()); let mac = mbe::DeclarativeMacro::parse_macro_rules(&tt, is_2021); mac } @@ -545,12 +530,7 @@ fn decl_macro_expander( ast::Macro::MacroDef(macro_def) => ( match macro_def.body() { Some(arg) => { - let tt = mbe::syntax_node_to_token_tree( - arg.syntax(), - SpanAnchor { file_id: id.file_id, ast_id: id.value.erase() }, - macro_def.syntax().text_range().start(), - &map, - ); + let tt = mbe::syntax_node_to_token_tree(arg.syntax(), map.as_ref()); let mac = mbe::DeclarativeMacro::parse_macro2(&tt, is_2021); mac } @@ -591,10 +571,16 @@ fn macro_expand( // FIXME: add firewall query for this? let hir_file_id = loc.kind.file_id(); let (root, map) = match hir_file_id.repr() { - HirFileIdRepr::FileId(file_id) => (db.parse(file_id).syntax_node(), None), + HirFileIdRepr::FileId(file_id) => { + // FIXME: query for span map + ( + db.parse(file_id).syntax_node(), + SpanMap::RealSpanMap(db.real_span_map(file_id)), + ) + } HirFileIdRepr::MacroFile(macro_file) => { let (parse, map) = db.parse_macro_expansion(macro_file).value; - (parse.syntax_node(), Some(map)) + (parse.syntax_node(), SpanMap::ExpansionSpanMap(map)) } }; let MacroCallKind::Derive { ast_id, .. } = loc.kind else { unreachable!() }; @@ -602,23 +588,7 @@ fn macro_expand( // FIXME: we might need to remove the spans from the input to the derive macro here let _censor = censor_for_macro_input(&loc, node.syntax()); - let _t; - expander.expand( - db, - macro_call_id, - &node, - match &map { - Some(map) => map, - None => { - _t = map_from_syntax_node( - node.syntax(), - SpanAnchor { file_id: hir_file_id, ast_id: ast_id.value.erase() }, - node.syntax().text_range().start(), - ); - &_t - } - }, - ) + expander.expand(db, macro_call_id, &node, map.as_ref()) } _ => { let ValueResult { value, err } = db.macro_arg(macro_call_id); @@ -732,7 +702,7 @@ fn token_tree_to_syntax_node( db: &dyn ExpandDatabase, tt: &tt::Subtree, expand_to: ExpandTo, -) -> (Parse, SpanMap) { +) -> (Parse, ExpansionSpanMap) { let entry_point = match expand_to { ExpandTo::Statements => mbe::TopEntryPoint::MacroStmts, ExpandTo::Items => mbe::TopEntryPoint::MacroItems, @@ -741,14 +711,14 @@ fn token_tree_to_syntax_node( ExpandTo::Expr => mbe::TopEntryPoint::Expr, }; let mut tm = mbe::token_tree_to_syntax_node(tt, entry_point); - // now what the hell is going on here + // FIXME: now what the hell is going on here tm.1.span_map.sort_by(|(_, a), (_, b)| { a.anchor.file_id.cmp(&b.anchor.file_id).then_with(|| { - let map = db.ast_id_map(a.anchor.file_id); - map.get_raw(a.anchor.ast_id) + let map = db.ast_id_map(a.anchor.file_id.into()); + map.get_erased(a.anchor.ast_id) .text_range() .start() - .cmp(&map.get_raw(b.anchor.ast_id).text_range().start()) + .cmp(&map.get_erased(b.anchor.ast_id).text_range().start()) }) }); tm diff --git a/crates/hir-expand/src/eager.rs b/crates/hir-expand/src/eager.rs index dc6507a92de5..bcb5383c3f9b 100644 --- a/crates/hir-expand/src/eager.rs +++ b/crates/hir-expand/src/eager.rs @@ -19,7 +19,7 @@ //! //! See the full discussion : use base_db::{ - span::{SpanAnchor, SyntaxContextId, ROOT_ERASED_FILE_AST_ID}, + span::{SpanAnchor, SyntaxContextId}, CrateId, }; use rustc_hash::FxHashMap; @@ -30,8 +30,9 @@ use crate::{ ast::{self, AstNode}, db::ExpandDatabase, mod_path::ModPath, - EagerCallInfo, ExpandError, ExpandResult, ExpandTo, InFile, MacroCallId, MacroCallKind, - MacroCallLoc, MacroDefId, MacroDefKind, SpanMap, + span::{RealSpanMap, SpanMapRef}, + EagerCallInfo, ExpandError, ExpandResult, ExpandTo, ExpansionSpanMap, InFile, MacroCallId, + MacroCallKind, MacroCallLoc, MacroDefId, MacroDefKind, }; pub fn expand_eager_macro_input( @@ -39,6 +40,7 @@ pub fn expand_eager_macro_input( krate: CrateId, macro_call: InFile, def: MacroDefId, + call_site: SyntaxContextId, resolver: &dyn Fn(ModPath) -> Option, ) -> ExpandResult> { let ast_map = db.ast_id_map(macro_call.file_id); @@ -55,18 +57,10 @@ pub fn expand_eager_macro_input( krate, eager: None, kind: MacroCallKind::FnLike { ast_id: call_id, expand_to: ExpandTo::Expr }, - // FIXME - call_site: SyntaxContextId::ROOT, + call_site, }); let ExpandResult { value: (arg_exp, arg_exp_map), err: parse_err } = db.parse_macro_expansion(arg_id.as_macro_file()); - // we need this map here as the expansion of the eager input fake file loses whitespace ... - // let mut ws_mapping = FxHashMap::default(); - // if let Some((tm)) = db.macro_arg(arg_id).value.as_deref() { - // ws_mapping.extend(tm.entries().filter_map(|(id, range)| { - // Some((arg_exp_map.first_range_by_token(id, syntax::SyntaxKind::TOMBSTONE)?, range)) - // })); - // } let ExpandResult { value: expanded_eager_input, err } = { eager_macro_recur( @@ -74,6 +68,7 @@ pub fn expand_eager_macro_input( &arg_exp_map, InFile::new(arg_id.as_file(), arg_exp.syntax_node()), krate, + call_site, resolver, ) }; @@ -83,44 +78,12 @@ pub fn expand_eager_macro_input( return ExpandResult { value: None, err }; }; + // FIXME: Spans! let mut subtree = mbe::syntax_node_to_token_tree( &expanded_eager_input, - // is this right? - SpanAnchor { file_id: arg_id.as_file(), ast_id: ROOT_ERASED_FILE_AST_ID }, - TextSize::new(0), - // FIXME: Spans! `eager_macro_recur` needs to fill out a span map for us - &Default::default(), + RealSpanMap::empty(::DUMMY.file_id), ); - // let og_tmap = if let Some(tt) = macro_call.value.token_tree() { - // let mut ids_used = FxHashSet::default(); - // let mut og_tmap = mbe::syntax_node_to_token_map(tt.syntax()); - // // The tokenmap and ids of subtree point into the expanded syntax node, but that is inaccessible from the outside - // // so we need to remap them to the original input of the eager macro. - // subtree.visit_ids(&mut |id| { - // // Note: we discard all token ids of braces and the like here, but that's not too bad and only a temporary fix - - // if let Some(range) = expanded_eager_input_token_map - // .first_range_by_token(id, syntax::SyntaxKind::TOMBSTONE) - // { - // // remap from expanded eager input to eager input expansion - // if let Some(og_range) = mapping.get(&range) { - // // remap from eager input expansion to original eager input - // if let Some(&og_range) = ws_mapping.get(og_range) { - // if let Some(og_token) = og_tmap.token_by_range(og_range) { - // ids_used.insert(og_token); - // return og_token; - // } - // } - // } - // } - // tt::TokenId::UNSPECIFIED - // }); - // og_tmap.filter(|id| ids_used.contains(&id)); - // og_tmap - // } else { - // Default::default() - // }; subtree.delimiter = crate::tt::Delimiter::UNSPECIFIED; let loc = MacroCallLoc { @@ -132,8 +95,7 @@ pub fn expand_eager_macro_input( error: err.clone(), })), kind: MacroCallKind::FnLike { ast_id: call_id, expand_to }, - // FIXME - call_site: SyntaxContextId::ROOT, + call_site, }; ExpandResult { value: Some(db.intern_macro_call(loc)), err } @@ -144,7 +106,8 @@ fn lazy_expand( def: &MacroDefId, macro_call: InFile, krate: CrateId, -) -> ExpandResult<(InFile>, Arc)> { + call_site: SyntaxContextId, +) -> ExpandResult<(InFile>, Arc)> { let ast_id = db.ast_id_map(macro_call.file_id).ast_id(¯o_call.value); let expand_to = ExpandTo::from_call_site(¯o_call.value); @@ -153,8 +116,8 @@ fn lazy_expand( db, krate, MacroCallKind::FnLike { ast_id, expand_to }, - // FIXME - SyntaxContextId::ROOT, + // FIXME: This is wrong + call_site, ); let macro_file = id.as_macro_file(); @@ -164,9 +127,10 @@ fn lazy_expand( fn eager_macro_recur( db: &dyn ExpandDatabase, - hygiene: &SpanMap, + hygiene: &ExpansionSpanMap, curr: InFile, krate: CrateId, + call_site: SyntaxContextId, macro_resolver: &dyn Fn(ModPath) -> Option, ) -> ExpandResult)>> { let original = curr.value.clone_for_update(); @@ -204,7 +168,10 @@ fn eager_macro_recur( continue; } }; - let def = match call.path().and_then(|path| ModPath::from_src(db, path, hygiene)) { + let def = match call + .path() + .and_then(|path| ModPath::from_src(db, path, SpanMapRef::ExpansionSpanMap(hygiene))) + { Some(path) => match macro_resolver(path.clone()) { Some(def) => def, None => { @@ -225,6 +192,8 @@ fn eager_macro_recur( krate, curr.with_value(call.clone()), def, + // FIXME: This call site is not quite right I think? We probably need to mark it? + call_site, macro_resolver, ); match value { @@ -260,7 +229,7 @@ fn eager_macro_recur( | MacroDefKind::BuiltInDerive(..) | MacroDefKind::ProcMacro(..) => { let ExpandResult { value: (parse, tm), err } = - lazy_expand(db, &def, curr.with_value(call.clone()), krate); + lazy_expand(db, &def, curr.with_value(call.clone()), krate, call_site); // replace macro inside let ExpandResult { value, err: error } = eager_macro_recur( @@ -269,6 +238,7 @@ fn eager_macro_recur( // FIXME: We discard parse errors here parse.as_ref().map(|it| it.syntax_node()), krate, + call_site, macro_resolver, ); let err = err.or(error); diff --git a/crates/hir-expand/src/files.rs b/crates/hir-expand/src/files.rs new file mode 100644 index 000000000000..45875d949817 --- /dev/null +++ b/crates/hir-expand/src/files.rs @@ -0,0 +1,293 @@ +use std::iter; + +use base_db::{ + span::{HirFileId, HirFileIdRepr, MacroFile, SyntaxContextId}, + FileRange, +}; +use either::Either; +use syntax::{AstNode, SyntaxNode, SyntaxToken, TextRange}; + +use crate::{db, ExpansionInfo, HirFileIdExt as _}; + +// FIXME: Make an InRealFile wrapper +/// `InFile` stores a value of `T` inside a particular file/syntax tree. +/// +/// Typical usages are: +/// +/// * `InFile` -- syntax node in a file +/// * `InFile` -- ast node in a file +/// * `InFile` -- offset in a file +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] +pub struct InFile { + pub file_id: HirFileId, + pub value: T, +} + +impl InFile { + pub fn new(file_id: HirFileId, value: T) -> InFile { + InFile { file_id, value } + } + + pub fn with_value(&self, value: U) -> InFile { + InFile::new(self.file_id, value) + } + + pub fn map U, U>(self, f: F) -> InFile { + InFile::new(self.file_id, f(self.value)) + } + + pub fn as_ref(&self) -> InFile<&T> { + self.with_value(&self.value) + } + + pub fn file_syntax(&self, db: &dyn db::ExpandDatabase) -> SyntaxNode { + db.parse_or_expand(self.file_id) + } +} + +impl InFile<&T> { + pub fn cloned(&self) -> InFile { + self.with_value(self.value.clone()) + } +} + +impl InFile> { + pub fn transpose(self) -> Option> { + let value = self.value?; + Some(InFile::new(self.file_id, value)) + } +} + +impl InFile> { + pub fn transpose(self) -> Either, InFile> { + match self.value { + Either::Left(l) => Either::Left(InFile::new(self.file_id, l)), + Either::Right(r) => Either::Right(InFile::new(self.file_id, r)), + } + } +} + +impl InFile<&SyntaxNode> { + pub fn ancestors_with_macros( + self, + db: &dyn db::ExpandDatabase, + ) -> impl Iterator> + Clone + '_ { + iter::successors(Some(self.cloned()), move |node| match node.value.parent() { + Some(parent) => Some(node.with_value(parent)), + None => node.file_id.call_node(db), + }) + } + + /// Skips the attributed item that caused the macro invocation we are climbing up + pub fn ancestors_with_macros_skip_attr_item( + self, + db: &dyn db::ExpandDatabase, + ) -> impl Iterator> + '_ { + let succ = move |node: &InFile| match node.value.parent() { + Some(parent) => Some(node.with_value(parent)), + None => { + let parent_node = node.file_id.call_node(db)?; + if node.file_id.is_attr_macro(db) { + // macro call was an attributed item, skip it + // FIXME: does this fail if this is a direct expansion of another macro? + parent_node.map(|node| node.parent()).transpose() + } else { + Some(parent_node) + } + } + }; + iter::successors(succ(&self.cloned()), succ) + } + + /// Falls back to the macro call range if the node cannot be mapped up fully. + /// + /// For attributes and derives, this will point back to the attribute only. + /// For the entire item use [`InFile::original_file_range_full`]. + pub fn original_file_range(self, db: &dyn db::ExpandDatabase) -> FileRange { + match self.file_id.repr() { + HirFileIdRepr::FileId(file_id) => FileRange { file_id, range: self.value.text_range() }, + HirFileIdRepr::MacroFile(mac_file) => { + if let Some((res, ctxt)) = + ExpansionInfo::new(db, mac_file).map_node_range_up(db, self.value.text_range()) + { + // FIXME: Figure out an API that makes proper use of ctx, this only exists to + // keep pre-token map rewrite behaviour. + if ctxt.is_root() { + return res; + } + } + // Fall back to whole macro call. + let loc = db.lookup_intern_macro_call(mac_file.macro_call_id); + loc.kind.original_call_range(db) + } + } + } + + /// Falls back to the macro call range if the node cannot be mapped up fully. + pub fn original_file_range_full(self, db: &dyn db::ExpandDatabase) -> FileRange { + match self.file_id.repr() { + HirFileIdRepr::FileId(file_id) => FileRange { file_id, range: self.value.text_range() }, + HirFileIdRepr::MacroFile(mac_file) => { + if let Some((res, ctxt)) = + ExpansionInfo::new(db, mac_file).map_node_range_up(db, self.value.text_range()) + { + // FIXME: Figure out an API that makes proper use of ctx, this only exists to + // keep pre-token map rewrite behaviour. + if ctxt.is_root() { + return res; + } + } + // Fall back to whole macro call. + let loc = db.lookup_intern_macro_call(mac_file.macro_call_id); + loc.kind.original_call_range_with_body(db) + } + } + } + + /// Attempts to map the syntax node back up its macro calls. + pub fn original_file_range_opt( + self, + db: &dyn db::ExpandDatabase, + ) -> Option<(FileRange, SyntaxContextId)> { + match self.file_id.repr() { + HirFileIdRepr::FileId(file_id) => { + Some((FileRange { file_id, range: self.value.text_range() }, SyntaxContextId::ROOT)) + } + HirFileIdRepr::MacroFile(mac_file) => { + ExpansionInfo::new(db, mac_file).map_node_range_up(db, self.value.text_range()) + } + } + } + + pub fn original_syntax_node(self, db: &dyn db::ExpandDatabase) -> Option> { + // This kind of upmapping can only be achieved in attribute expanded files, + // as we don't have node inputs otherwise and therefore can't find an `N` node in the input + let Some(file_id) = self.file_id.macro_file() else { + return Some(self.map(Clone::clone)); + }; + if !self.file_id.is_attr_macro(db) { + return None; + } + + let (FileRange { file_id, range }, ctx) = + ExpansionInfo::new(db, file_id).map_node_range_up(db, self.value.text_range())?; + + // FIXME: Figure out an API that makes proper use of ctx, this only exists to + // keep pre-token map rewrite behaviour. + if !ctx.is_root() { + return None; + } + + let anc = db.parse(file_id).syntax_node().covering_element(range); + let kind = self.value.kind(); + // FIXME: This heuristic is brittle and with the right macro may select completely unrelated nodes? + let value = anc.ancestors().find(|it| it.kind() == kind)?; + Some(InFile::new(file_id.into(), value)) + } +} + +impl InFile { + pub fn upmap_once( + self, + db: &dyn db::ExpandDatabase, + ) -> Option>> { + Some(self.file_id.expansion_info(db)?.map_range_up_once(db, self.value.text_range())) + } + + /// Falls back to the macro call range if the node cannot be mapped up fully. + pub fn original_file_range(self, db: &dyn db::ExpandDatabase) -> FileRange { + match self.file_id.repr() { + HirFileIdRepr::FileId(file_id) => FileRange { file_id, range: self.value.text_range() }, + HirFileIdRepr::MacroFile(mac_file) => { + if let Some(res) = self.original_file_range_opt(db) { + return res; + } + // Fall back to whole macro call. + let loc = db.lookup_intern_macro_call(mac_file.macro_call_id); + loc.kind.original_call_range(db) + } + } + } + + /// Attempts to map the syntax node back up its macro calls. + pub fn original_file_range_opt(self, db: &dyn db::ExpandDatabase) -> Option { + match self.file_id.repr() { + HirFileIdRepr::FileId(file_id) => { + Some(FileRange { file_id, range: self.value.text_range() }) + } + HirFileIdRepr::MacroFile(_) => { + let (range, ctxt) = ascend_range_up_macros(db, self.map(|it| it.text_range())); + + // FIXME: Figure out an API that makes proper use of ctx, this only exists to + // keep pre-token map rewrite behaviour. + if ctxt.is_root() { + Some(range) + } else { + None + } + } + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] +pub struct InMacroFile { + pub file_id: MacroFile, + pub value: T, +} + +impl From> for InFile { + fn from(macro_file: InMacroFile) -> Self { + InFile { file_id: macro_file.file_id.into(), value: macro_file.value } + } +} + +pub fn ascend_range_up_macros( + db: &dyn db::ExpandDatabase, + range: InFile, +) -> (FileRange, SyntaxContextId) { + match range.file_id.repr() { + HirFileIdRepr::FileId(file_id) => { + (FileRange { file_id, range: range.value }, SyntaxContextId::ROOT) + } + HirFileIdRepr::MacroFile(m) => { + ExpansionInfo::new(db, m).map_token_range_up(db, range.value) + } + } +} + +impl InFile { + pub fn descendants(self) -> impl Iterator> { + self.value.syntax().descendants().filter_map(T::cast).map(move |n| self.with_value(n)) + } + + // FIXME: this should return `Option>` + pub fn original_ast_node(self, db: &dyn db::ExpandDatabase) -> Option> { + // This kind of upmapping can only be achieved in attribute expanded files, + // as we don't have node inputs otherwise and therefore can't find an `N` node in the input + let Some(file_id) = self.file_id.macro_file() else { + return Some(self); + }; + if !self.file_id.is_attr_macro(db) { + return None; + } + + let (FileRange { file_id, range }, ctx) = ExpansionInfo::new(db, file_id) + .map_node_range_up(db, self.value.syntax().text_range())?; + + // FIXME: Figure out an API that makes proper use of ctx, this only exists to + // keep pre-token map rewrite behaviour. + if !ctx.is_root() { + return None; + } + + // FIXME: This heuristic is brittle and with the right macro may select completely unrelated nodes? + let anc = db.parse(file_id).syntax_node().covering_element(range); + let value = anc.ancestors().find_map(N::cast)?; + return Some(InFile::new(file_id.into(), value)); + } + + pub fn syntax(&self) -> InFile<&SyntaxNode> { + self.with_value(self.value.syntax()) + } +} diff --git a/crates/hir-expand/src/hygiene.rs b/crates/hir-expand/src/hygiene.rs index f83a9bf2d657..66d9f679d894 100644 --- a/crates/hir-expand/src/hygiene.rs +++ b/crates/hir-expand/src/hygiene.rs @@ -2,7 +2,9 @@ //! //! Specifically, `ast` + `Hygiene` allows you to create a `Name`. Note that, at //! this moment, this is horribly incomplete and handles only `$crate`. -use base_db::span::{MacroCallId, SyntaxContextId}; +use std::iter; + +use base_db::span::{MacroCallId, SpanData, SyntaxContextId}; use crate::db::ExpandDatabase; @@ -48,6 +50,39 @@ pub enum Transparency { Opaque, } +pub fn span_with_def_site_ctxt( + db: &dyn ExpandDatabase, + span: SpanData, + expn_id: MacroCallId, +) -> SpanData { + span_with_ctxt_from_mark(db, span, expn_id, Transparency::Opaque) +} + +pub fn span_with_call_site_ctxt( + db: &dyn ExpandDatabase, + span: SpanData, + expn_id: MacroCallId, +) -> SpanData { + span_with_ctxt_from_mark(db, span, expn_id, Transparency::Transparent) +} + +pub fn span_with_mixed_site_ctxt( + db: &dyn ExpandDatabase, + span: SpanData, + expn_id: MacroCallId, +) -> SpanData { + span_with_ctxt_from_mark(db, span, expn_id, Transparency::SemiTransparent) +} + +fn span_with_ctxt_from_mark( + db: &dyn ExpandDatabase, + span: SpanData, + expn_id: MacroCallId, + transparency: Transparency, +) -> SpanData { + SpanData { ctx: db.apply_mark(SyntaxContextId::ROOT, expn_id, transparency), ..span } +} + pub(super) fn apply_mark( db: &dyn ExpandDatabase, ctxt: SyntaxContextId, @@ -65,7 +100,7 @@ pub(super) fn apply_mark( call_site_ctxt.normalize_to_macro_rules(db) }; - if call_site_ctxt.is_root(db) { + if call_site_ctxt.is_root() { return apply_mark_internal(db, ctxt, Some(call_id), transparency); } @@ -131,7 +166,6 @@ fn apply_mark_internal( }) } pub trait SyntaxContextExt { - fn is_root(self, db: &dyn ExpandDatabase) -> bool; fn normalize_to_macro_rules(self, db: &dyn ExpandDatabase) -> Self; fn normalize_to_macros_2_0(self, db: &dyn ExpandDatabase) -> Self; fn parent_ctxt(self, db: &dyn ExpandDatabase) -> Self; @@ -148,9 +182,6 @@ fn handle_self_ref(p: SyntaxContextId, n: SyntaxContextId) -> SyntaxContextId { } impl SyntaxContextExt for SyntaxContextId { - fn is_root(self, db: &dyn ExpandDatabase) -> bool { - db.lookup_intern_syntax_context(self).outer_expn.is_none() - } fn normalize_to_macro_rules(self, db: &dyn ExpandDatabase) -> Self { handle_self_ref(self, db.lookup_intern_syntax_context(self).opaque_and_semitransparent) } @@ -164,20 +195,20 @@ impl SyntaxContextExt for SyntaxContextId { let data = db.lookup_intern_syntax_context(self); (data.outer_expn, data.outer_transparency) } - fn marks(mut self, db: &dyn ExpandDatabase) -> Vec<(Option, Transparency)> { - let mut marks = Vec::new(); - while self != SyntaxContextId::ROOT { - marks.push(self.outer_mark(db)); - self = self.parent_ctxt(db); - } + fn marks(self, db: &dyn ExpandDatabase) -> Vec<(Option, Transparency)> { + let mut marks = marks_rev(self, db).collect::>(); marks.reverse(); marks } } -// pub(super) fn with_ctxt_from_mark(db: &ExpandDatabase, file_id: HirFileId) { -// self.with_ctxt_from_mark(expn_id, Transparency::Transparent) -// } -// pub(super) fn with_call_site_ctxt(db: &ExpandDatabase, file_id: HirFileId) { -// self.with_ctxt_from_mark(expn_id, Transparency::Transparent) -// } +// FIXME: Make this a SyntaxContextExt method once we have RPIT +pub fn marks_rev( + ctxt: SyntaxContextId, + db: &dyn ExpandDatabase, +) -> impl Iterator, Transparency)> + '_ { + iter::successors(Some(ctxt), move |&mark| { + Some(mark.parent_ctxt(db)).filter(|&it| it != SyntaxContextId::ROOT) + }) + .map(|ctx| ctx.outer_mark(db)) +} diff --git a/crates/hir-expand/src/lib.rs b/crates/hir-expand/src/lib.rs index 6864f477ae8d..d2e5e7c3643f 100644 --- a/crates/hir-expand/src/lib.rs +++ b/crates/hir-expand/src/lib.rs @@ -18,25 +18,25 @@ pub mod quote; pub mod eager; pub mod mod_path; pub mod attrs; +pub mod span; +pub mod files; // mod fixup; use triomphe::Arc; -use std::{fmt, hash::Hash, iter}; +use std::{fmt, hash::Hash}; use base_db::{ - span::{HirFileIdRepr, SyntaxContextId}, + span::{HirFileIdRepr, SpanData, SyntaxContextId}, CrateId, FileId, FileRange, ProcMacroKind, }; use either::Either; use syntax::{ - algo::{self, skip_trivia_token}, ast::{self, AstNode, HasDocComments}, - AstPtr, Direction, SyntaxNode, SyntaxNodePtr, SyntaxToken, TextSize, + SyntaxNode, SyntaxToken, TextRange, TextSize, }; use crate::{ - ast_id_map::{AstIdNode, ErasedFileAstId, FileAstId}, attrs::AttrId, builtin_attr_macro::BuiltinAttrExpander, builtin_derive_macro::BuiltinDeriveExpander, @@ -44,12 +44,15 @@ use crate::{ db::TokenExpander, mod_path::ModPath, proc_macro::ProcMacroExpander, + span::ExpansionSpanMap, }; +pub use crate::ast_id_map::{AstId, ErasedAstId, ErasedFileAstId}; +pub use crate::files::{InFile, InMacroFile}; + pub use base_db::span::{HirFileId, MacroCallId, MacroFile}; pub use mbe::ValueResult; -pub type SpanMap = ::mbe::TokenMap; pub type DeclarativeMacro = ::mbe::DeclarativeMacro; pub mod tt { @@ -103,7 +106,7 @@ impl fmt::Display for ExpandError { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct MacroCallLoc { pub def: MacroDefId, - pub(crate) krate: CrateId, + pub krate: CrateId, /// Some if this is a macro call for an eager macro. Note that this is `None` /// for the eager input macro file. eager: Option>, @@ -247,8 +250,7 @@ impl HirFileIdExt for HirFileId { /// Return expansion information if it is a macro-expansion file fn expansion_info(self, db: &dyn db::ExpandDatabase) -> Option { - let macro_file = self.macro_file()?; - ExpansionInfo::new(db, macro_file) + Some(ExpansionInfo::new(db, self.macro_file()?)) } fn as_builtin_derive_attr_node( @@ -340,15 +342,14 @@ impl MacroDefId { } pub fn ast_id(&self) -> Either, AstId> { - let id = match self.kind { + match self.kind { MacroDefKind::ProcMacro(.., id) => return Either::Right(id), MacroDefKind::Declarative(id) | MacroDefKind::BuiltIn(_, id) | MacroDefKind::BuiltInAttr(_, id) | MacroDefKind::BuiltInDerive(_, id) - | MacroDefKind::BuiltInEager(_, id) => id, - }; - Either::Left(id) + | MacroDefKind::BuiltInEager(_, id) => Either::Left(id), + } } pub fn is_proc_macro(&self) -> bool { @@ -390,6 +391,18 @@ impl MacroDefId { } impl MacroCallLoc { + pub fn span(&self, db: &dyn db::ExpandDatabase) -> SpanData { + let ast_id = self.kind.erased_ast_id(); + let file_id = self.kind.file_id(); + let range = db.ast_id_map(file_id).get_erased(ast_id).text_range(); + match file_id.repr() { + HirFileIdRepr::FileId(file_id) => db.real_span_map(file_id).span_for_range(range), + HirFileIdRepr::MacroFile(m) => { + db.parse_macro_expansion(m).value.1.span_for_range(range) + } + } + } + pub fn to_node(&self, db: &dyn db::ExpandDatabase) -> InFile { match self.kind { MacroCallKind::FnLike { ast_id, .. } => { @@ -430,17 +443,15 @@ impl MacroCallLoc { match self.kind { MacroCallKind::FnLike { expand_to, .. } => expand_to, MacroCallKind::Derive { .. } => ExpandTo::Items, - MacroCallKind::Attr { .. } if self.def.is_attribute_derive() => ExpandTo::Statements, + MacroCallKind::Attr { .. } if self.def.is_attribute_derive() => ExpandTo::Items, MacroCallKind::Attr { .. } => { - // is this always correct? + // FIXME(stmt_expr_attributes) ExpandTo::Items } } } } -// FIXME: attribute indices do not account for nested `cfg_attr` - impl MacroCallKind { /// Returns the file containing the macro invocation. fn file_id(&self) -> HirFileId { @@ -451,6 +462,14 @@ impl MacroCallKind { } } + fn erased_ast_id(&self) -> ErasedFileAstId { + match *self { + MacroCallKind::FnLike { ast_id: InFile { value, .. }, .. } => value.erase(), + MacroCallKind::Derive { ast_id: InFile { value, .. }, .. } => value.erase(), + MacroCallKind::Attr { ast_id: InFile { value, .. }, .. } => value.erase(), + } + } + /// Returns the original file range that best describes the location of this macro call. /// /// Unlike `MacroCallKind::original_call_range`, this also spans the item of attributes and derives. @@ -518,34 +537,40 @@ impl MacroCallKind { FileRange { range, file_id } } - fn arg(&self, db: &dyn db::ExpandDatabase) -> Option> { + // FIXME: -> InFile it should be impossible for the token tree to be missing at + // this point! + fn arg(&self, db: &dyn db::ExpandDatabase) -> InFile> { match self { - MacroCallKind::FnLike { ast_id, .. } => ast_id - .to_in_file_node(db) - .map(|it| Some(it.token_tree()?.syntax().clone())) - .transpose(), + MacroCallKind::FnLike { ast_id, .. } => { + ast_id.to_in_file_node(db).map(|it| Some(it.token_tree()?.syntax().clone())) + } MacroCallKind::Derive { ast_id, .. } => { - Some(ast_id.to_in_file_node(db).syntax().cloned()) + ast_id.to_in_file_node(db).syntax().cloned().map(Some) } MacroCallKind::Attr { ast_id, .. } => { - Some(ast_id.to_in_file_node(db).syntax().cloned()) + ast_id.to_in_file_node(db).syntax().cloned().map(Some) } } } } /// ExpansionInfo mainly describes how to map text range between src and expanded macro +// FIXME: can be expensive to create, we should check the use sites and maybe replace them with +// simpler function calls if the map is only used once #[derive(Debug, Clone, PartialEq, Eq)] pub struct ExpansionInfo { - expanded: InMacroFile, + pub expanded: InMacroFile, /// The argument TokenTree or item for attributes - arg: InFile, + // FIXME: Can this ever be `None`? + arg: InFile>, /// The `macro_rules!` or attribute input. attr_input_or_mac_def: Option>, macro_def: TokenExpander, macro_arg: Arc, - exp_map: Arc, + exp_map: Arc, + /// [`None`] if the call is in a real file + arg_map: Option>, } impl ExpansionInfo { @@ -554,81 +579,133 @@ impl ExpansionInfo { } pub fn call_node(&self) -> Option> { - Some(self.arg.with_value(self.arg.value.parent()?)) + Some(self.arg.with_value(self.arg.value.as_ref()?.parent()?)) } - /// Map a token down from macro input into the macro expansion. - /// - /// The inner workings of this function differ slightly depending on the type of macro we are dealing with: - /// - declarative: - /// For declarative macros, we need to accommodate for the macro definition site(which acts as a second unchanging input) - /// , as tokens can mapped in and out of it. - /// To do this we shift all ids in the expansion by the maximum id of the definition site giving us an easy - /// way to map all the tokens. - /// - attribute: - /// Attributes have two different inputs, the input tokentree in the attribute node and the item - /// the attribute is annotating. Similarly as for declarative macros we need to do a shift here - /// as well. Currently this is done by shifting the attribute input by the maximum id of the item. - /// - function-like and derives: - /// Both of these only have one simple call site input so no special handling is required here. - pub fn map_token_down( - &self, - db: &dyn db::ExpandDatabase, - token: InFile<&SyntaxToken>, + /// Maps the passed in file range down into a macro expansion if it is the input to a macro call. + pub fn map_range_down<'a>( + &'a self, + db: &'a dyn db::ExpandDatabase, + FileRange { file_id, range: absolute_range }: FileRange, // FIXME: use this for range mapping, so that we can resolve inline format args _relative_token_offset: Option, - ) -> Option> + '_> { - assert_eq!(token.file_id, self.arg.file_id); + // FIXME: ret ty should be wrapped in InMacroFile + ) -> Option> + 'a> { + // search for all entries in the span map that have the given span and return the + // corresponding text ranges inside the expansion + // FIXME: Make this proper let span_map = &self.exp_map.span_map; let (start, end) = if span_map .first() - .map_or(false, |(_, span)| span.anchor.file_id == token.file_id) + .map_or(false, |(_, span)| span.anchor.file_id == file_id) { - (0, span_map.partition_point(|a| a.1.anchor.file_id == token.file_id)) + (0, span_map.partition_point(|a| a.1.anchor.file_id == file_id)) } else { - let start = span_map.partition_point(|a| a.1.anchor.file_id != token.file_id); - ( - start, - start + span_map[start..].partition_point(|a| a.1.anchor.file_id == token.file_id), - ) + let start = span_map.partition_point(|a| a.1.anchor.file_id != file_id); + (start, start + span_map[start..].partition_point(|a| a.1.anchor.file_id == file_id)) }; - let token_text_range = token.value.text_range(); - let ast_id_map = db.ast_id_map(token.file_id); let tokens = span_map[start..end] .iter() .filter_map(move |(range, span)| { - let offset = ast_id_map.get_raw(span.anchor.ast_id).text_range().start(); + // we need to resolve the relative ranges here to make sure that we are in fact + // considering differently anchored spans (this might occur with proc-macros) + let offset = db + .ast_id_map(span.anchor.file_id.into()) + .get_erased(span.anchor.ast_id) + .text_range() + .start(); let abs_range = span.range + offset; - token_text_range.eq(&abs_range).then_some(*range) + absolute_range.eq(&abs_range).then_some(*range) }) .flat_map(move |range| self.expanded.value.covering_element(range).into_token()); Some(tokens.map(move |token| InFile::new(self.expanded.file_id.into(), token))) } - /// Map a token up out of the expansion it resides in into the arguments of the macro call of the expansion. - pub fn map_token_up( + /// Maps up the text range out of the expansion hierarchy back into the original file its from. + pub fn map_token_range_up( + &self, + db: &dyn db::ExpandDatabase, + range: TextRange, + ) -> (FileRange, SyntaxContextId) { + debug_assert!(self.expanded.value.text_range().contains_range(range)); + let span = self.exp_map.span_for_range(range); + let anchor_offset = db + .ast_id_map(span.anchor.file_id.into()) + .get_erased(span.anchor.ast_id) + .text_range() + .start(); + (FileRange { file_id: span.anchor.file_id, range: span.range + anchor_offset }, span.ctx) + } + + /// Maps up the text range out of the expansion hierarchy back into the original file its from. + pub fn map_node_range_up( + &self, + db: &dyn db::ExpandDatabase, + range: TextRange, + ) -> Option<(FileRange, SyntaxContextId)> { + debug_assert!(self.expanded.value.text_range().contains_range(range)); + let mut spans = self.exp_map.spans_for_node_range(range); + let SpanData { range, anchor, ctx } = spans.next()?; + let mut start = range.start(); + let mut end = range.end(); + + for span in spans { + if span.anchor != anchor || span.ctx != ctx { + return None; + } + start = start.min(span.range.start()); + end = end.max(span.range.end()); + } + let anchor_offset = + db.ast_id_map(anchor.file_id.into()).get_erased(anchor.ast_id).text_range().start(); + Some(( + FileRange { + file_id: anchor.file_id, + range: TextRange::new(start, end) + anchor_offset, + }, + ctx, + )) + } + + /// Maps up the text range out of the expansion into is macro call. + pub fn map_range_up_once( &self, db: &dyn db::ExpandDatabase, - token: InFile<&SyntaxToken>, - ) -> Option> { - self.exp_map.span_for_range(token.value.text_range()).and_then(|span| { - let anchor = - db.ast_id_map(span.anchor.file_id).get_raw(span.anchor.ast_id).text_range().start(); - InFile::new( - span.anchor.file_id, - db.parse_or_expand(span.anchor.file_id) - .covering_element(span.range + anchor) - .into_token(), - ) - .transpose() - }) + token: TextRange, + ) -> InFile> { + debug_assert!(self.expanded.value.text_range().contains_range(token)); + let span = self.exp_map.span_for_range(token); + match &self.arg_map { + None => { + let file_id = span.anchor.file_id.into(); + let anchor_offset = + db.ast_id_map(file_id).get_erased(span.anchor.ast_id).text_range().start(); + InFile { file_id, value: smallvec::smallvec![span.range + anchor_offset] } + } + Some(arg_map) => { + let arg_range = self + .arg + .value + .as_ref() + .map_or_else(|| TextRange::empty(TextSize::from(0)), |it| it.text_range()); + InFile::new( + self.arg.file_id, + arg_map + .ranges_with_span(span) + .filter(|range| range.intersect(arg_range).is_some()) + .collect(), + ) + } + } } - fn new(db: &dyn db::ExpandDatabase, macro_file: MacroFile) -> Option { + pub fn new(db: &dyn db::ExpandDatabase, macro_file: MacroFile) -> ExpansionInfo { let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_file.macro_call_id); - let arg_tt = loc.kind.arg(db)?; + let arg_tt = loc.kind.arg(db); + let arg_map = + arg_tt.file_id.macro_file().map(|file| db.parse_macro_expansion(file).value.1); let macro_def = db.macro_expander(loc.def); let (parse, exp_map) = db.parse_macro_expansion(macro_file).value; @@ -662,331 +739,18 @@ impl ExpansionInfo { _ => None, }); - Some(ExpansionInfo { + ExpansionInfo { expanded, arg: arg_tt, attr_input_or_mac_def, macro_arg, macro_def, exp_map, - }) - } -} - -/// `AstId` points to an AST node in any file. -/// -/// It is stable across reparses, and can be used as salsa key/value. -pub type AstId = InFile>; - -impl AstId { - pub fn to_node(&self, db: &dyn db::ExpandDatabase) -> N { - self.to_ptr(db).to_node(&db.parse_or_expand(self.file_id)) - } - pub fn to_in_file_node(&self, db: &dyn db::ExpandDatabase) -> InFile { - InFile::new(self.file_id, self.to_ptr(db).to_node(&db.parse_or_expand(self.file_id))) - } - pub fn to_ptr(&self, db: &dyn db::ExpandDatabase) -> AstPtr { - db.ast_id_map(self.file_id).get(self.value) - } -} - -pub type ErasedAstId = InFile; - -impl ErasedAstId { - pub fn to_ptr(&self, db: &dyn db::ExpandDatabase) -> SyntaxNodePtr { - db.ast_id_map(self.file_id).get_raw(self.value) - } -} - -/// `InFile` stores a value of `T` inside a particular file/syntax tree. -/// -/// Typical usages are: -/// -/// * `InFile` -- syntax node in a file -/// * `InFile` -- ast node in a file -/// * `InFile` -- offset in a file -#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] -pub struct InFile { - pub file_id: HirFileId, - pub value: T, -} - -impl InFile { - pub fn new(file_id: HirFileId, value: T) -> InFile { - InFile { file_id, value } - } - - pub fn with_value(&self, value: U) -> InFile { - InFile::new(self.file_id, value) - } - - pub fn map U, U>(self, f: F) -> InFile { - InFile::new(self.file_id, f(self.value)) - } - - pub fn as_ref(&self) -> InFile<&T> { - self.with_value(&self.value) - } - - pub fn file_syntax(&self, db: &dyn db::ExpandDatabase) -> SyntaxNode { - db.parse_or_expand(self.file_id) - } -} - -impl InFile<&T> { - pub fn cloned(&self) -> InFile { - self.with_value(self.value.clone()) - } -} - -impl InFile> { - pub fn transpose(self) -> Option> { - let value = self.value?; - Some(InFile::new(self.file_id, value)) - } -} - -impl InFile> { - pub fn transpose(self) -> Either, InFile> { - match self.value { - Either::Left(l) => Either::Left(InFile::new(self.file_id, l)), - Either::Right(r) => Either::Right(InFile::new(self.file_id, r)), + arg_map, } } } -impl InFile<&SyntaxNode> { - pub fn ancestors_with_macros( - self, - db: &dyn db::ExpandDatabase, - ) -> impl Iterator> + Clone + '_ { - iter::successors(Some(self.cloned()), move |node| match node.value.parent() { - Some(parent) => Some(node.with_value(parent)), - None => node.file_id.call_node(db), - }) - } - - /// Skips the attributed item that caused the macro invocation we are climbing up - pub fn ancestors_with_macros_skip_attr_item( - self, - db: &dyn db::ExpandDatabase, - ) -> impl Iterator> + '_ { - let succ = move |node: &InFile| match node.value.parent() { - Some(parent) => Some(node.with_value(parent)), - None => { - let parent_node = node.file_id.call_node(db)?; - if node.file_id.is_attr_macro(db) { - // macro call was an attributed item, skip it - // FIXME: does this fail if this is a direct expansion of another macro? - parent_node.map(|node| node.parent()).transpose() - } else { - Some(parent_node) - } - } - }; - iter::successors(succ(&self.cloned()), succ) - } - - /// Falls back to the macro call range if the node cannot be mapped up fully. - /// - /// For attributes and derives, this will point back to the attribute only. - /// For the entire item use [`InFile::original_file_range_full`]. - pub fn original_file_range(self, db: &dyn db::ExpandDatabase) -> FileRange { - match self.file_id.repr() { - HirFileIdRepr::FileId(file_id) => FileRange { file_id, range: self.value.text_range() }, - HirFileIdRepr::MacroFile(mac_file) => { - if let Some(res) = self.original_file_range_opt(db) { - return res; - } - // Fall back to whole macro call. - let loc = db.lookup_intern_macro_call(mac_file.macro_call_id); - loc.kind.original_call_range(db) - } - } - } - - /// Falls back to the macro call range if the node cannot be mapped up fully. - pub fn original_file_range_full(self, db: &dyn db::ExpandDatabase) -> FileRange { - match self.file_id.repr() { - HirFileIdRepr::FileId(file_id) => FileRange { file_id, range: self.value.text_range() }, - HirFileIdRepr::MacroFile(mac_file) => { - if let Some(res) = self.original_file_range_opt(db) { - return res; - } - // Fall back to whole macro call. - let loc = db.lookup_intern_macro_call(mac_file.macro_call_id); - loc.kind.original_call_range_with_body(db) - } - } - } - - /// Attempts to map the syntax node back up its macro calls. - pub fn original_file_range_opt(self, db: &dyn db::ExpandDatabase) -> Option { - match ascend_node_border_tokens(db, self) { - Some(InFile { file_id, value: (first, last) }) => { - let original_file = file_id.original_file(db); - let range = first.text_range().cover(last.text_range()); - if file_id != original_file.into() { - tracing::error!("Failed mapping up more for {:?}", range); - return None; - } - Some(FileRange { file_id: original_file, range }) - } - _ if !self.file_id.is_macro() => Some(FileRange { - file_id: self.file_id.original_file(db), - range: self.value.text_range(), - }), - _ => None, - } - } - - pub fn original_syntax_node(self, db: &dyn db::ExpandDatabase) -> Option> { - // This kind of upmapping can only be achieved in attribute expanded files, - // as we don't have node inputs otherwise and therefore can't find an `N` node in the input - if !self.file_id.is_macro() { - return Some(self.map(Clone::clone)); - } else if !self.file_id.is_attr_macro(db) { - return None; - } - - if let Some(InFile { file_id, value: (first, last) }) = ascend_node_border_tokens(db, self) - { - if file_id.is_macro() { - let range = first.text_range().cover(last.text_range()); - tracing::error!("Failed mapping out of macro file for {:?}", range); - return None; - } - // FIXME: This heuristic is brittle and with the right macro may select completely unrelated nodes - let anc = algo::least_common_ancestor(&first.parent()?, &last.parent()?)?; - let kind = self.value.kind(); - let value = anc.ancestors().find(|it| it.kind() == kind)?; - return Some(InFile::new(file_id, value)); - } - None - } -} - -impl InFile { - pub fn upmap(self, db: &dyn db::ExpandDatabase) -> Option> { - let expansion = self.file_id.expansion_info(db)?; - expansion.map_token_up(db, self.as_ref()) - } - - /// Falls back to the macro call range if the node cannot be mapped up fully. - pub fn original_file_range(self, db: &dyn db::ExpandDatabase) -> FileRange { - match self.file_id.repr() { - HirFileIdRepr::FileId(file_id) => FileRange { file_id, range: self.value.text_range() }, - HirFileIdRepr::MacroFile(mac_file) => { - if let Some(res) = self.original_file_range_opt(db) { - return res; - } - // Fall back to whole macro call. - let loc = db.lookup_intern_macro_call(mac_file.macro_call_id); - loc.kind.original_call_range(db) - } - } - } - - /// Attempts to map the syntax node back up its macro calls. - pub fn original_file_range_opt(self, db: &dyn db::ExpandDatabase) -> Option { - match self.file_id.repr() { - HirFileIdRepr::FileId(file_id) => { - Some(FileRange { file_id, range: self.value.text_range() }) - } - HirFileIdRepr::MacroFile(_) => { - let expansion = self.file_id.expansion_info(db)?; - let InFile { file_id, value } = ascend_call_token(db, &expansion, self)?; - let original_file = file_id.original_file(db); - if file_id != original_file.into() { - return None; - } - Some(FileRange { file_id: original_file, range: value.text_range() }) - } - } - } -} - -#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] -pub struct InMacroFile { - pub file_id: MacroFile, - pub value: T, -} - -impl From> for InFile { - fn from(macro_file: InMacroFile) -> Self { - InFile { file_id: macro_file.file_id.into(), value: macro_file.value } - } -} - -// FIXME: Get rid of this -fn ascend_node_border_tokens( - db: &dyn db::ExpandDatabase, - InFile { file_id, value: node }: InFile<&SyntaxNode>, -) -> Option> { - let expansion = file_id.expansion_info(db)?; - - let first_token = |node: &SyntaxNode| skip_trivia_token(node.first_token()?, Direction::Next); - let last_token = |node: &SyntaxNode| skip_trivia_token(node.last_token()?, Direction::Prev); - - // FIXME: Once the token map rewrite is done, this shouldnt need to rely on syntax nodes and tokens anymore - let first = first_token(node)?; - let last = last_token(node)?; - let first = ascend_call_token(db, &expansion, InFile::new(file_id, first))?; - let last = ascend_call_token(db, &expansion, InFile::new(file_id, last))?; - (first.file_id == last.file_id).then(|| InFile::new(first.file_id, (first.value, last.value))) -} - -fn ascend_call_token( - db: &dyn db::ExpandDatabase, - expansion: &ExpansionInfo, - token: InFile, -) -> Option> { - let mut mapping = expansion.map_token_up(db, token.as_ref())?; - - loop { - match mapping.file_id.expansion_info(db) { - Some(info) => mapping = info.map_token_up(db, mapping.as_ref())?, - None => return Some(mapping), - } - } -} - -impl InFile { - pub fn descendants(self) -> impl Iterator> { - self.value.syntax().descendants().filter_map(T::cast).map(move |n| self.with_value(n)) - } - - // FIXME: this should return `Option>` - pub fn original_ast_node(self, db: &dyn db::ExpandDatabase) -> Option> { - // This kind of upmapping can only be achieved in attribute expanded files, - // as we don't have node inputs otherwise and therefore can't find an `N` node in the input - if !self.file_id.is_macro() { - return Some(self); - } else if !self.file_id.is_attr_macro(db) { - return None; - } - - if let Some(InFile { file_id, value: (first, last) }) = - ascend_node_border_tokens(db, self.syntax()) - { - if file_id.is_macro() { - let range = first.text_range().cover(last.text_range()); - tracing::error!("Failed mapping out of macro file for {:?}", range); - return None; - } - // FIXME: This heuristic is brittle and with the right macro may select completely unrelated nodes - let anc = algo::least_common_ancestor(&first.parent()?, &last.parent()?)?; - let value = anc.ancestors().find_map(N::cast)?; - return Some(InFile::new(file_id, value)); - } - None - } - - pub fn syntax(&self) -> InFile<&SyntaxNode> { - self.with_value(self.value.syntax()) - } -} - /// In Rust, macros expand token trees to token trees. When we want to turn a /// token tree into an AST node, we need to figure out what kind of AST node we /// want: something like `foo` can be a type, an expression, or a pattern. @@ -1051,9 +815,4 @@ impl ExpandTo { } } -#[derive(Debug)] -pub struct UnresolvedMacro { - pub path: ModPath, -} - intern::impl_internable!(ModPath, attrs::AttrInput); diff --git a/crates/hir-expand/src/mod_path.rs b/crates/hir-expand/src/mod_path.rs index 9396b08b2800..a518f7eb6a9f 100644 --- a/crates/hir-expand/src/mod_path.rs +++ b/crates/hir-expand/src/mod_path.rs @@ -7,9 +7,9 @@ use std::{ use crate::{ db::ExpandDatabase, - hygiene::{SyntaxContextExt, Transparency}, + hygiene::{marks_rev, SyntaxContextExt, Transparency}, name::{known, AsName, Name}, - SpanMap, + span::SpanMapRef, }; use base_db::{span::SyntaxContextId, CrateId}; use smallvec::SmallVec; @@ -47,7 +47,7 @@ impl ModPath { pub fn from_src( db: &dyn ExpandDatabase, path: ast::Path, - hygiene: &SpanMap, + hygiene: SpanMapRef<'_>, ) -> Option { convert_path(db, None, path, hygiene) } @@ -194,7 +194,7 @@ fn convert_path( db: &dyn ExpandDatabase, prefix: Option, path: ast::Path, - hygiene: &SpanMap, + hygiene: SpanMapRef<'_>, ) -> Option { let prefix = match path.qualifier() { Some(qual) => Some(convert_path(db, prefix, qual, hygiene)?), @@ -208,14 +208,14 @@ fn convert_path( if prefix.is_some() { return None; } - resolve_crate_root( - db, - hygiene - .span_for_range(name_ref.syntax().text_range()) - .map_or(SyntaxContextId::ROOT, |s| s.ctx), + ModPath::from_kind( + resolve_crate_root( + db, + hygiene.span_for_range(name_ref.syntax().text_range()).ctx, + ) + .map(PathKind::DollarCrate) + .unwrap_or(PathKind::Crate), ) - .map(PathKind::DollarCrate) - .map(ModPath::from_kind)? } else { let mut res = prefix.unwrap_or_else(|| { ModPath::from_kind( @@ -265,13 +265,12 @@ fn convert_path( // We follow what it did anyway :) if mod_path.segments.len() == 1 && mod_path.kind == PathKind::Plain { if let Some(_macro_call) = path.syntax().parent().and_then(ast::MacroCall::cast) { - let syn_ctx = hygiene - .span_for_range(segment.syntax().text_range()) - .map_or(SyntaxContextId::ROOT, |s| s.ctx); + let syn_ctx = hygiene.span_for_range(segment.syntax().text_range()).ctx; if let Some(macro_call_id) = db.lookup_intern_syntax_context(syn_ctx).outer_expn { if db.lookup_intern_macro_call(macro_call_id).def.local_inner { - if let Some(crate_root) = resolve_crate_root(db, syn_ctx) { - mod_path.kind = PathKind::DollarCrate(crate_root); + mod_path.kind = match resolve_crate_root(db, syn_ctx) { + Some(crate_root) => PathKind::DollarCrate(crate_root), + None => PathKind::Crate, } } } @@ -289,30 +288,19 @@ pub fn resolve_crate_root(db: &dyn ExpandDatabase, mut ctxt: SyntaxContextId) -> // definitions actually produced by `macro` and `macro` definitions produced by // `macro_rules!`, but at least such configurations are not stable yet. ctxt = ctxt.normalize_to_macro_rules(db); - let mut iter = ctxt.marks(db).into_iter().rev().peekable(); + let mut iter = marks_rev(ctxt, db).peekable(); let mut result_mark = None; // Find the last opaque mark from the end if it exists. - while let Some(&(mark, transparency)) = iter.peek() { - if transparency == Transparency::Opaque { - result_mark = Some(mark); - iter.next(); - } else { - break; - } + while let Some(&(mark, Transparency::Opaque)) = iter.peek() { + result_mark = Some(mark); + iter.next(); } // Then find the last semi-transparent mark from the end if it exists. - for (mark, transparency) in iter { - if transparency == Transparency::SemiTransparent { - result_mark = Some(mark); - } else { - break; - } + while let Some((mark, Transparency::SemiTransparent)) = iter.next() { + result_mark = Some(mark); } - match result_mark { - Some(Some(call)) => Some(db.lookup_intern_macro_call(call.into()).def.krate), - Some(None) | None => None, - } + result_mark.flatten().map(|call| db.lookup_intern_macro_call(call.into()).def.krate) } pub use crate::name as __name; diff --git a/crates/hir-expand/src/span.rs b/crates/hir-expand/src/span.rs new file mode 100644 index 000000000000..c16d761c1719 --- /dev/null +++ b/crates/hir-expand/src/span.rs @@ -0,0 +1,109 @@ +use base_db::{ + span::{ErasedFileAstId, SpanAnchor, SpanData, SyntaxContextId, ROOT_ERASED_FILE_AST_ID}, + FileId, +}; +use mbe::TokenMap; +use syntax::{ast::HasModuleItem, AstNode, TextRange, TextSize}; +use triomphe::Arc; + +use crate::db::ExpandDatabase; + +pub type ExpansionSpanMap = TokenMap; + +/// Spanmap for a macro file or a real file +#[derive(Clone, Debug)] +pub enum SpanMap { + /// Spanmap for a macro file + ExpansionSpanMap(Arc), + /// Spanmap for a real file + RealSpanMap(Arc), +} + +#[derive(Copy, Clone)] +pub enum SpanMapRef<'a> { + /// Spanmap for a macro file + ExpansionSpanMap(&'a ExpansionSpanMap), + /// Spanmap for a real file + RealSpanMap(&'a RealSpanMap), +} + +impl mbe::SpanMapper for SpanMap { + fn span_for(&self, range: TextRange) -> SpanData { + self.span_for_range(range) + } +} +impl mbe::SpanMapper for SpanMapRef<'_> { + fn span_for(&self, range: TextRange) -> SpanData { + self.span_for_range(range) + } +} +impl mbe::SpanMapper for RealSpanMap { + fn span_for(&self, range: TextRange) -> SpanData { + self.span_for_range(range) + } +} + +impl SpanMap { + pub fn span_for_range(&self, range: TextRange) -> SpanData { + match self { + Self::ExpansionSpanMap(span_map) => span_map.span_for_range(range), + Self::RealSpanMap(span_map) => span_map.span_for_range(range), + } + } + + pub fn as_ref(&self) -> SpanMapRef<'_> { + match self { + Self::ExpansionSpanMap(span_map) => SpanMapRef::ExpansionSpanMap(span_map), + Self::RealSpanMap(span_map) => SpanMapRef::RealSpanMap(span_map), + } + } +} + +impl SpanMapRef<'_> { + pub fn span_for_range(self, range: TextRange) -> SpanData { + match self { + Self::ExpansionSpanMap(span_map) => span_map.span_for_range(range), + Self::RealSpanMap(span_map) => span_map.span_for_range(range), + } + } +} + +#[derive(PartialEq, Eq, Hash, Debug)] +pub struct RealSpanMap { + file_id: FileId, + /// Invariant: Sorted vec over TextSize + // FIXME: SortedVec<(TextSize, ErasedFileAstId)>? + pairs: Box<[(TextSize, ErasedFileAstId)]>, +} + +impl RealSpanMap { + pub fn empty(file_id: FileId) -> Self { + RealSpanMap { file_id, pairs: Box::from([(TextSize::new(0), ROOT_ERASED_FILE_AST_ID)]) } + } + + pub fn from_file(db: &dyn ExpandDatabase, file_id: FileId) -> Self { + let mut pairs = vec![(TextSize::new(0), ROOT_ERASED_FILE_AST_ID)]; + let ast_id_map = db.ast_id_map(file_id.into()); + pairs.extend( + db.parse(file_id) + .tree() + .items() + .map(|item| (item.syntax().text_range().start(), ast_id_map.ast_id(&item).erase())), + ); + RealSpanMap { file_id, pairs: pairs.into_boxed_slice() } + } + + pub fn span_for_range(&self, range: TextRange) -> SpanData { + let start = range.start(); + let idx = self + .pairs + .binary_search_by(|&(it, _)| it.cmp(&start).then(std::cmp::Ordering::Less)) + .unwrap_err(); + let (offset, ast_id) = self.pairs[idx - 1]; + SpanData { + range: range - offset, + anchor: SpanAnchor { file_id: self.file_id, ast_id }, + ctx: SyntaxContextId::ROOT, + } + } +} diff --git a/crates/hir/src/attrs.rs b/crates/hir/src/attrs.rs index bb02620208bf..42d1515e5931 100644 --- a/crates/hir/src/attrs.rs +++ b/crates/hir/src/attrs.rs @@ -1,5 +1,6 @@ //! Attributes & documentation for hir types. +use base_db::FileId; use hir_def::{ attr::AttrsWithOwner, item_scope::ItemInNs, @@ -8,7 +9,10 @@ use hir_def::{ resolver::{HasResolver, Resolver, TypeNs}, AssocItemId, AttrDefId, ModuleDefId, }; -use hir_expand::name::Name; +use hir_expand::{ + name::Name, + span::{RealSpanMap, SpanMapRef}, +}; use hir_ty::db::HirDatabase; use syntax::{ast, AstNode}; @@ -234,7 +238,11 @@ fn modpath_from_str(db: &dyn HirDatabase, link: &str) -> Option { if ast_path.syntax().text() != link { return None; } - ModPath::from_src(db.upcast(), ast_path, &Default::default()) + ModPath::from_src( + db.upcast(), + ast_path, + SpanMapRef::RealSpanMap(&RealSpanMap::empty(FileId(0))), + ) }; let full = try_get_modpath(link); diff --git a/crates/hir/src/lib.rs b/crates/hir/src/lib.rs index b224b4b3a97c..443c19af82d3 100644 --- a/crates/hir/src/lib.rs +++ b/crates/hir/src/lib.rs @@ -59,7 +59,7 @@ use hir_def::{ Lookup, MacroExpander, MacroId, ModuleId, StaticId, StructId, TraitAliasId, TraitId, TypeAliasId, TypeOrConstParamId, TypeParamId, UnionId, }; -use hir_expand::{name::name, MacroCallKind}; +use hir_expand::{name::name, InMacroFile, MacroCallKind}; use hir_ty::{ all_super_traits, autoderef, check_orphan_rules, consteval::{try_const_usize, unknown_const_as_generic, ConstEvalError, ConstExt}, @@ -3483,11 +3483,41 @@ impl Impl { self.id.lookup(db.upcast()).container.into() } - pub fn as_builtin_derive(self, db: &dyn HirDatabase) -> Option> { + pub fn as_builtin_derive_attr(self, db: &dyn HirDatabase) -> Option> { let src = self.source(db)?; src.file_id.as_builtin_derive_attr_node(db.upcast()) } + pub fn as_builtin_derive_path(self, db: &dyn HirDatabase) -> Option> { + let src = self.source(db)?; + + let macro_file = src.file_id.macro_file()?; + let loc = db.lookup_intern_macro_call(macro_file.macro_call_id); + let (derive_attr, derive_index) = match loc.kind { + MacroCallKind::Derive { ast_id, derive_attr_index, derive_index } => { + let module_id = self.id.lookup(db.upcast()).container; + ( + db.crate_def_map(module_id.krate())[module_id.local_id] + .scope + .derive_macro_invoc(ast_id, derive_attr_index)?, + derive_index, + ) + } + _ => return None, + }; + let file_id = MacroFile { macro_call_id: derive_attr }; + let path = db + .parse_macro_expansion(file_id) + .value + .0 + .syntax_node() + .children() + .nth(derive_index as usize) + .and_then(::cast) + .and_then(|it| it.path())?; + Some(InMacroFile { file_id, value: path }) + } + pub fn check_orphan_rules(self, db: &dyn HirDatabase) -> bool { check_orphan_rules(db, self.id) } diff --git a/crates/hir/src/semantics.rs b/crates/hir/src/semantics.rs index 825d68cdbaed..137cffa7af15 100644 --- a/crates/hir/src/semantics.rs +++ b/crates/hir/src/semantics.rs @@ -19,6 +19,7 @@ use hir_expand::{db::ExpandDatabase, name::AsName, ExpansionInfo, HirFileIdExt, use itertools::Itertools; use rustc_hash::{FxHashMap, FxHashSet}; use smallvec::{smallvec, SmallVec}; +use stdx::TupleExt; use syntax::{ algo::skip_trivia_token, ast::{self, HasAttrs as _, HasGenericParams, HasLoopBody}, @@ -529,11 +530,11 @@ impl<'db> SemanticsImpl<'db> { token: SyntaxToken, // FIXME: We might want this to be Option to be able to opt out of subrange // mapping, specifically for node downmapping - offset: TextSize, + _offset: TextSize, f: &mut dyn FnMut(InFile) -> bool, ) { + // FIXME: Clean this up let _p = profile::span("descend_into_macros"); - let relative_token_offset = token.text_range().start().checked_sub(offset); let parent = match token.parent() { Some(it) => it, None => return, @@ -543,13 +544,35 @@ impl<'db> SemanticsImpl<'db> { None => return, }; let def_map = sa.resolver.def_map(); + let absolute_range = match sa.file_id.repr() { + base_db::span::HirFileIdRepr::FileId(file_id) => { + FileRange { file_id, range: token.text_range() } + } + base_db::span::HirFileIdRepr::MacroFile(m) => { + let span = + self.db.parse_macro_expansion(m).value.1.span_for_range(token.text_range()); + let range = span.range + + self + .db + .ast_id_map(span.anchor.file_id.into()) + .get_erased(span.anchor.ast_id) + .text_range() + .start(); + FileRange { file_id: span.anchor.file_id, range } + } + }; + + // fetch span information of token in real file, then use that look through expansions of + // calls the token is in and afterwards recursively with the same span. + // what about things where spans change? Due to being joined etc, that is we don't find the + // exact span anymore? let mut stack: SmallVec<[_; 4]> = smallvec![InFile::new(sa.file_id, token)]; let mut cache = self.expansion_info_cache.borrow_mut(); let mut mcache = self.macro_call_cache.borrow_mut(); let mut process_expansion_for_token = - |stack: &mut SmallVec<_>, macro_file, token: InFile<&_>| { + |stack: &mut SmallVec<_>, macro_file, _token: InFile<&_>| { let expansion_info = cache .entry(macro_file) .or_insert_with(|| macro_file.expansion_info(self.db.upcast())) @@ -560,11 +583,8 @@ impl<'db> SemanticsImpl<'db> { self.cache(value, file_id); } - let mapped_tokens = expansion_info.map_token_down( - self.db.upcast(), - token, - relative_token_offset, - )?; + let mapped_tokens = + expansion_info.map_range_down(self.db.upcast(), absolute_range, None)?; let len = stack.len(); // requeue the tokens we got from mapping our current token down @@ -728,6 +748,8 @@ impl<'db> SemanticsImpl<'db> { pub fn original_range_opt(&self, node: &SyntaxNode) -> Option { let node = self.find_file(node); node.original_file_range_opt(self.db.upcast()) + .filter(|(_, ctx)| ctx.is_root()) + .map(TupleExt::head) } /// Attempts to map the node out of macro expanded files. diff --git a/crates/hir/src/symbols.rs b/crates/hir/src/symbols.rs index ca7874c3683c..6ea926cc22d8 100644 --- a/crates/hir/src/symbols.rs +++ b/crates/hir/src/symbols.rs @@ -7,7 +7,7 @@ use hir_def::{ AdtId, AssocItemId, DefWithBodyId, HasModule, ImplId, Lookup, MacroId, ModuleDefId, ModuleId, TraitId, }; -use hir_expand::{HirFileId, InFile}; +use hir_expand::{files::ascend_range_up_macros, HirFileId, InFile}; use hir_ty::db::HirDatabase; use syntax::{ast::HasName, AstNode, SmolStr, SyntaxNode, SyntaxNodePtr}; @@ -50,13 +50,9 @@ impl DeclarationLocation { node.as_ref().original_file_range(db.upcast()) } - pub fn original_name_range(&self, db: &dyn HirDatabase) -> Option { - if let Some(file_id) = self.hir_file_id.file_id() { - // fast path to prevent parsing - return Some(FileRange { file_id, range: self.name_ptr.text_range() }); - } - let node = resolve_node(db, self.hir_file_id, &self.name_ptr); - node.as_ref().original_file_range_opt(db.upcast()) + pub fn original_name_range(&self, db: &dyn HirDatabase) -> FileRange { + let mapping = InFile::new(self.hir_file_id, self.name_ptr.text_range()); + ascend_range_up_macros(db.upcast(), mapping).0 } } diff --git a/crates/ide-completion/src/tests/proc_macros.rs b/crates/ide-completion/src/tests/proc_macros.rs index 2d6234e310c6..e55d1f315fed 100644 --- a/crates/ide-completion/src/tests/proc_macros.rs +++ b/crates/ide-completion/src/tests/proc_macros.rs @@ -9,6 +9,7 @@ fn check(ra_fixture: &str, expect: Expect) { } #[test] +#[ignore] // todo fn complete_dot_in_attr() { check( r#" @@ -40,6 +41,7 @@ fn main() { } #[test] +#[ignore] // TODO fn complete_dot_in_attr2() { check( r#" diff --git a/crates/ide-db/src/rename.rs b/crates/ide-db/src/rename.rs index cc9038fdfa2f..9f101468a350 100644 --- a/crates/ide-db/src/rename.rs +++ b/crates/ide-db/src/rename.rs @@ -22,10 +22,10 @@ //! Our current behavior is ¯\_(ツ)_/¯. use std::fmt; -use base_db::{AnchoredPathBuf, FileId, FileRange}; +use base_db::{span::SyntaxContextId, AnchoredPathBuf, FileId, FileRange}; use either::Either; use hir::{FieldSource, HasSource, HirFileIdExt, InFile, ModuleSource, Semantics}; -use stdx::never; +use stdx::{never, TupleExt}; use syntax::{ ast::{self, HasName}, AstNode, SyntaxKind, TextRange, T, @@ -103,6 +103,7 @@ impl Definition { /// renamed and extern crate names will report its range, though a rename will introduce /// an alias instead. pub fn range_for_rename(self, sema: &Semantics<'_, RootDatabase>) -> Option { + let syn_ctx_is_root = |(range, ctx): (_, SyntaxContextId)| ctx.is_root().then(|| range); let res = match self { Definition::Macro(mac) => { let src = mac.source(sema.db)?; @@ -110,14 +111,18 @@ impl Definition { Either::Left(it) => it.name()?, Either::Right(it) => it.name()?, }; - src.with_value(name.syntax()).original_file_range_opt(sema.db) + src.with_value(name.syntax()) + .original_file_range_opt(sema.db) + .and_then(syn_ctx_is_root) } Definition::Field(field) => { let src = field.source(sema.db)?; match &src.value { FieldSource::Named(record_field) => { let name = record_field.name()?; - src.with_value(name.syntax()).original_file_range_opt(sema.db) + src.with_value(name.syntax()) + .original_file_range_opt(sema.db) + .and_then(syn_ctx_is_root) } FieldSource::Pos(_) => None, } @@ -125,25 +130,31 @@ impl Definition { Definition::Module(module) => { let src = module.declaration_source(sema.db)?; let name = src.value.name()?; - src.with_value(name.syntax()).original_file_range_opt(sema.db) + src.with_value(name.syntax()) + .original_file_range_opt(sema.db) + .and_then(syn_ctx_is_root) } - Definition::Function(it) => name_range(it, sema), + Definition::Function(it) => name_range(it, sema).and_then(syn_ctx_is_root), Definition::Adt(adt) => match adt { - hir::Adt::Struct(it) => name_range(it, sema), - hir::Adt::Union(it) => name_range(it, sema), - hir::Adt::Enum(it) => name_range(it, sema), + hir::Adt::Struct(it) => name_range(it, sema).and_then(syn_ctx_is_root), + hir::Adt::Union(it) => name_range(it, sema).and_then(syn_ctx_is_root), + hir::Adt::Enum(it) => name_range(it, sema).and_then(syn_ctx_is_root), }, - Definition::Variant(it) => name_range(it, sema), - Definition::Const(it) => name_range(it, sema), - Definition::Static(it) => name_range(it, sema), - Definition::Trait(it) => name_range(it, sema), - Definition::TraitAlias(it) => name_range(it, sema), - Definition::TypeAlias(it) => name_range(it, sema), - Definition::Local(it) => name_range(it.primary_source(sema.db), sema), + Definition::Variant(it) => name_range(it, sema).and_then(syn_ctx_is_root), + Definition::Const(it) => name_range(it, sema).and_then(syn_ctx_is_root), + Definition::Static(it) => name_range(it, sema).and_then(syn_ctx_is_root), + Definition::Trait(it) => name_range(it, sema).and_then(syn_ctx_is_root), + Definition::TraitAlias(it) => name_range(it, sema).and_then(syn_ctx_is_root), + Definition::TypeAlias(it) => name_range(it, sema).and_then(syn_ctx_is_root), + Definition::Local(it) => { + name_range(it.primary_source(sema.db), sema).and_then(syn_ctx_is_root) + } Definition::GenericParam(generic_param) => match generic_param { hir::GenericParam::LifetimeParam(lifetime_param) => { let src = lifetime_param.source(sema.db)?; - src.with_value(src.value.lifetime()?.syntax()).original_file_range_opt(sema.db) + src.with_value(src.value.lifetime()?.syntax()) + .original_file_range_opt(sema.db) + .and_then(syn_ctx_is_root) } _ => { let x = match generic_param { @@ -156,22 +167,30 @@ impl Definition { Either::Left(x) => x.name()?, Either::Right(_) => return None, }; - src.with_value(name.syntax()).original_file_range_opt(sema.db) + src.with_value(name.syntax()) + .original_file_range_opt(sema.db) + .and_then(syn_ctx_is_root) } }, Definition::Label(label) => { let src = label.source(sema.db); let lifetime = src.value.lifetime()?; - src.with_value(lifetime.syntax()).original_file_range_opt(sema.db) + src.with_value(lifetime.syntax()) + .original_file_range_opt(sema.db) + .and_then(syn_ctx_is_root) } Definition::ExternCrateDecl(it) => { let src = it.source(sema.db)?; if let Some(rename) = src.value.rename() { let name = rename.name()?; - src.with_value(name.syntax()).original_file_range_opt(sema.db) + src.with_value(name.syntax()) + .original_file_range_opt(sema.db) + .and_then(syn_ctx_is_root) } else { let name = src.value.name_ref()?; - src.with_value(name.syntax()).original_file_range_opt(sema.db) + src.with_value(name.syntax()) + .original_file_range_opt(sema.db) + .and_then(syn_ctx_is_root) } } Definition::BuiltinType(_) => return None, @@ -183,7 +202,10 @@ impl Definition { }; return res; - fn name_range(def: D, sema: &Semantics<'_, RootDatabase>) -> Option + fn name_range( + def: D, + sema: &Semantics<'_, RootDatabase>, + ) -> Option<(FileRange, SyntaxContextId)> where D: HasSource, D::Ast: ast::HasName, @@ -256,8 +278,10 @@ fn rename_mod( let file_id = src.file_id.original_file(sema.db); match src.value.name() { Some(name) => { - if let Some(file_range) = - src.with_value(name.syntax()).original_file_range_opt(sema.db) + if let Some(file_range) = src + .with_value(name.syntax()) + .original_file_range_opt(sema.db) + .map(TupleExt::head) { source_change.insert_source_edit( file_id, @@ -493,7 +517,12 @@ fn source_edit_from_def( for source in local.sources(sema.db) { let source = match source.source.clone().original_ast_node(sema.db) { Some(source) => source, - None => match source.source.syntax().original_file_range_opt(sema.db) { + None => match source + .source + .syntax() + .original_file_range_opt(sema.db) + .map(TupleExt::head) + { Some(FileRange { file_id: file_id2, range }) => { file_id = Some(file_id2); edit.replace(range, new_name.to_owned()); diff --git a/crates/ide-diagnostics/src/handlers/macro_error.rs b/crates/ide-diagnostics/src/handlers/macro_error.rs index 7ca0a0eab2b6..2993950be04a 100644 --- a/crates/ide-diagnostics/src/handlers/macro_error.rs +++ b/crates/ide-diagnostics/src/handlers/macro_error.rs @@ -60,9 +60,6 @@ macro_rules! compile_error { () => {} } #[test] fn eager_macro_concat() { - // FIXME: this is incorrectly handling `$crate`, resulting in a wrong diagnostic. - // See: https://github.com/rust-lang/rust-analyzer/issues/10300 - check_diagnostics( r#" //- /lib.rs crate:lib deps:core @@ -80,7 +77,6 @@ macro_rules! m { fn f() { m!(); - //^^^^ error: unresolved macro $crate::private::concat } //- /core.rs crate:core diff --git a/crates/ide-diagnostics/src/handlers/typed_hole.rs b/crates/ide-diagnostics/src/handlers/typed_hole.rs index 4e215a89d793..ea5c7564d343 100644 --- a/crates/ide-diagnostics/src/handlers/typed_hole.rs +++ b/crates/ide-diagnostics/src/handlers/typed_hole.rs @@ -33,7 +33,7 @@ pub(crate) fn typed_hole(ctx: &DiagnosticsContext<'_>, d: &hir::TypedHole) -> Di fn fixes(ctx: &DiagnosticsContext<'_>, d: &hir::TypedHole) -> Option> { let db = ctx.sema.db; let root = db.parse_or_expand(d.expr.file_id); - let original_range = + let (original_range, _) = d.expr.as_ref().map(|it| it.to_node(&root)).syntax().original_file_range_opt(db)?; let scope = ctx.sema.scope(d.expr.value.to_node(&root).syntax())?; let mut assists = vec![]; diff --git a/crates/ide-diagnostics/src/handlers/unresolved_module.rs b/crates/ide-diagnostics/src/handlers/unresolved_module.rs index bbbd21741e5d..234ca2a7671b 100644 --- a/crates/ide-diagnostics/src/handlers/unresolved_module.rs +++ b/crates/ide-diagnostics/src/handlers/unresolved_module.rs @@ -152,9 +152,7 @@ mod baz {} main_node: Some( InFile { file_id: FileId( - FileId( - 0, - ), + 0, ), value: MODULE@0..8 MOD_KW@0..3 "mod" diff --git a/crates/ide/src/expand_macro.rs b/crates/ide/src/expand_macro.rs index f7c6a0139e0c..cca3e68d5024 100644 --- a/crates/ide/src/expand_macro.rs +++ b/crates/ide/src/expand_macro.rs @@ -1,4 +1,4 @@ -use hir::{HirFileIdExt, Semantics}; +use hir::{HirFileIdExt, InFile, Semantics}; use ide_db::{ base_db::FileId, helpers::pick_best_token, syntax_helpers::insert_whitespace_into_node::insert_ws_into, RootDatabase, @@ -49,7 +49,9 @@ pub(crate) fn expand_macro(db: &RootDatabase, position: FilePosition) -> Option< let name = descended.parent_ancestors().filter_map(ast::Path::cast).last()?.to_string(); // up map out of the #[derive] expansion - let token = hir::InFile::new(hir_file, descended).upmap(db)?.value; + let InFile { file_id, value: tokens } = + hir::InFile::new(hir_file, descended).upmap_once(db)?; + let token = sema.parse_or_expand(file_id).covering_element(tokens[0]).into_token()?; let attr = token.parent_ancestors().find_map(ast::Attr::cast)?; let expansions = sema.expand_derive_macro(&attr)?; let idx = attr @@ -338,8 +340,8 @@ fn main() { expect![[r#" match_ast! { - if let Some(it) = ast::TraitDef::cast(container.clone()){} - else if let Some(it) = ast::ImplDef::cast(container.clone()){} + if let Some(it) = ast::TraitDef::cast((container).clone()){} + else if let Some(it) = ast::ImplDef::cast((container).clone()){} else { { continue diff --git a/crates/ide/src/goto_definition.rs b/crates/ide/src/goto_definition.rs index 59e8578cf154..816f1bebee42 100644 --- a/crates/ide/src/goto_definition.rs +++ b/crates/ide/src/goto_definition.rs @@ -401,11 +401,11 @@ fn bar() { //- /lib.rs macro_rules! define_fn { () => (fn foo() {}) - //^^^ + } define_fn!(); - +//^^^^^^^^^^^^^ fn bar() { $0foo(); } diff --git a/crates/ide/src/goto_implementation.rs b/crates/ide/src/goto_implementation.rs index 544c6b42317e..3593c5c7dd9c 100644 --- a/crates/ide/src/goto_implementation.rs +++ b/crates/ide/src/goto_implementation.rs @@ -249,7 +249,7 @@ impl T for &Foo {} r#" //- minicore: copy, derive #[derive(Copy)] -//^^^^^^^^^^^^^^^ + //^^^^ struct Foo$0; "#, ); diff --git a/crates/ide/src/hover.rs b/crates/ide/src/hover.rs index e0b64fe7988e..fc0c9ef17fa1 100644 --- a/crates/ide/src/hover.rs +++ b/crates/ide/src/hover.rs @@ -166,6 +166,7 @@ fn hover_simple( } else { sema.descend_into_macros_with_same_text(original_token.clone(), offset) }; + dbg!(&descended); let descended = || descended.iter(); let result = descended() diff --git a/crates/ide/src/inlay_hints/closure_captures.rs b/crates/ide/src/inlay_hints/closure_captures.rs index d691303c18b8..2f8b959516df 100644 --- a/crates/ide/src/inlay_hints/closure_captures.rs +++ b/crates/ide/src/inlay_hints/closure_captures.rs @@ -2,6 +2,7 @@ //! //! Tests live in [`bind_pat`][super::bind_pat] module. use ide_db::{base_db::FileId, famous_defs::FamousDefs}; +use stdx::TupleExt; use syntax::ast::{self, AstNode}; use text_edit::{TextRange, TextSize}; @@ -73,7 +74,9 @@ pub(super) fn hints( capture.display_place(sema.db) ), None, - source.name().and_then(|name| name.syntax().original_file_range_opt(sema.db)), + source.name().and_then(|name| { + name.syntax().original_file_range_opt(sema.db).map(TupleExt::head) + }), ); acc.push(InlayHint { needs_resolve: label.needs_resolve(), diff --git a/crates/ide/src/navigation_target.rs b/crates/ide/src/navigation_target.rs index 32f211c6b289..1e4d0e8cdc6e 100644 --- a/crates/ide/src/navigation_target.rs +++ b/crates/ide/src/navigation_target.rs @@ -4,8 +4,8 @@ use std::fmt; use either::Either; use hir::{ - symbols::FileSymbol, AssocItem, FieldSource, HasContainer, HasSource, HirDisplay, HirFileId, - InFile, LocalSource, ModuleSource, + db::ExpandDatabase, symbols::FileSymbol, AssocItem, FieldSource, HasContainer, HasSource, + HirDisplay, HirFileId, InFile, LocalSource, ModuleSource, }; use ide_db::{ base_db::{FileId, FileRange}, @@ -40,6 +40,8 @@ pub struct NavigationTarget { /// comments, and `focus_range` is the range of the identifier. /// /// Clients should place the cursor on this range when navigating to this target. + /// + /// This range must be contained within [`Self::full_range`]. pub focus_range: Option, pub name: SmolStr, pub kind: Option, @@ -166,13 +168,9 @@ impl NavigationTarget { impl TryToNav for FileSymbol { fn try_to_nav(&self, db: &RootDatabase) -> Option { let full_range = self.loc.original_range(db); - let focus_range = self.loc.original_name_range(db).and_then(|it| { - if it.file_id == full_range.file_id { - Some(it.range) - } else { - None - } - }); + let focus_range = self.loc.original_name_range(db); + let focus_range = + if focus_range.file_id == full_range.file_id { Some(focus_range.range) } else { None }; Some(NavigationTarget { file_id: full_range.file_id, @@ -363,11 +361,11 @@ impl ToNav for hir::Module { impl TryToNav for hir::Impl { fn try_to_nav(&self, db: &RootDatabase) -> Option { let InFile { file_id, value } = self.source(db)?; - let derive_attr = self.as_builtin_derive(db); + let derive_path = self.as_builtin_derive_path(db); - let (focus, syntax) = match &derive_attr { - Some(attr) => (None, attr.value.syntax()), - None => (value.self_ty(), value.syntax()), + let (file_id, focus, syntax) = match &derive_path { + Some(attr) => (attr.file_id.into(), None, attr.value.syntax()), + None => (file_id, value.self_ty(), value.syntax()), }; let (file_id, full_range, focus_range) = orig_range_with_focus(db, file_id, syntax, focus); @@ -628,19 +626,30 @@ impl TryToNav for hir::ConstParam { } } +/// Returns the original range of the syntax node, and the range of the name mapped out of macro expansions +/// Additionally verifies that the name span is in bounds and related to the original range. fn orig_range_with_focus( db: &RootDatabase, hir_file: HirFileId, value: &SyntaxNode, name: Option, ) -> (FileId, TextRange, Option) { - let FileRange { file_id, range: full_range } = - InFile::new(hir_file, value).original_file_range(db); + let FileRange { file_id, range } = + match InFile::new(hir_file, value).original_file_range_opt(db) { + Some((range, ctxt)) if ctxt.is_root() => range, + _ => db + .lookup_intern_macro_call(hir_file.macro_file().unwrap().macro_call_id) + .kind + .original_call_range(db), + }; let focus_range = name .and_then(|it| InFile::new(hir_file, it.syntax()).original_file_range_opt(db)) - .and_then(|range| if range.file_id == file_id { Some(range.range) } else { None }); + .filter(|(frange, ctxt)| { + ctxt.is_root() && frange.file_id == file_id && frange.range.contains_range(frange.range) + }) + .map(|(frange, _ctxt)| frange.range); - (file_id, full_range, focus_range) + (file_id, range, focus_range) } #[cfg(test)] diff --git a/crates/ide/src/static_index.rs b/crates/ide/src/static_index.rs index 7ee50f7a67f8..cb1236836b08 100644 --- a/crates/ide/src/static_index.rs +++ b/crates/ide/src/static_index.rs @@ -242,6 +242,7 @@ mod tests { } } + #[track_caller] fn check_definitions(ra_fixture: &str) { let (analysis, ranges) = fixture::annotations_without_marker(ra_fixture); let s = StaticIndex::compute(&analysis); diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html index 64e614cecd20..aa79cd9b829f 100644 --- a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html +++ b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html @@ -127,7 +127,7 @@ println!("Hello {:+}!", 5); println!("{:#x}!", 27); println!("Hello {:05}!", 5); - println!("Hello {:05}!", -5); + println!("Hello {:05}!", -5); println!("{:#010x}!", 27); println!("Hello {0} is {1:.5}", "x", 0.01); println!("Hello {1} is {2:.0$}", 5, "x", 0.01); diff --git a/crates/mbe/src/benchmark.rs b/crates/mbe/src/benchmark.rs index eb0773570b32..e7fbb918897d 100644 --- a/crates/mbe/src/benchmark.rs +++ b/crates/mbe/src/benchmark.rs @@ -10,12 +10,12 @@ use tt::{Span, SpanAnchor, SyntaxContext}; use crate::{ parser::{MetaVarKind, Op, RepeatKind, Separator}, - syntax_node_to_token_tree, DeclarativeMacro, + syntax_node_to_token_tree, DeclarativeMacro, SpanMapper, }; type SpanData = tt::SpanData; -#[derive(PartialEq, Eq, Clone, Copy, Debug)] +#[derive(PartialEq, Eq, Clone, Copy, Debug, Hash)] struct DummyFile; impl SpanAnchor for DummyFile { const DUMMY: Self = DummyFile; @@ -27,6 +27,14 @@ impl SyntaxContext for DummyCtx { const DUMMY: Self = DummyCtx; } +struct NoOpMap; + +impl SpanMapper for NoOpMap { + fn span_for(&self, range: syntax::TextRange) -> SpanData { + SpanData { range, anchor: DummyFile, ctx: DummyCtx } + } +} + #[test] fn benchmark_parse_macro_rules() { if skip_slow_tests() { @@ -79,12 +87,7 @@ fn macro_rules_fixtures_tt() -> FxHashMap> { .filter_map(ast::MacroRules::cast) .map(|rule| { let id = rule.name().unwrap().to_string(); - let def_tt = syntax_node_to_token_tree( - rule.token_tree().unwrap().syntax(), - DummyFile, - 0.into(), - &Default::default(), - ); + let def_tt = syntax_node_to_token_tree(rule.token_tree().unwrap().syntax(), NoOpMap); (id, def_tt) }) .collect() diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index 482d0157b2db..0b8461200e63 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -30,12 +30,12 @@ use crate::{ // FIXME: we probably should re-think `token_tree_to_syntax_node` interfaces pub use ::parser::TopEntryPoint; -pub use tt::{Delimiter, DelimiterKind, Punct}; +pub use tt::{Delimiter, DelimiterKind, Punct, SyntaxContext}; pub use crate::{ syntax_bridge::{ map_from_syntax_node, parse_exprs_with_sep, parse_to_token_tree, syntax_node_to_token_tree, - syntax_node_to_token_tree_censored, token_tree_to_syntax_node, + syntax_node_to_token_tree_censored, token_tree_to_syntax_node, SpanMapper, }, token_map::TokenMap, }; diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index f47123336b44..36c63b365d2c 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -17,43 +17,52 @@ use crate::{to_parser_input::to_parser_input, tt_iter::TtIter, TokenMap}; #[cfg(test)] mod tests; +pub trait SpanMapper { + fn span_for(&self, range: TextRange) -> S; +} + +impl SpanMapper for TokenMap { + fn span_for(&self, range: TextRange) -> S { + self.span_for_range(range) + } +} + +impl> SpanMapper for &SM { + fn span_for(&self, range: TextRange) -> S { + SM::span_for(self, range) + } +} + /// Convert the syntax node to a `TokenTree` (what macro /// will consume). -/// `anchor` and `anchor_offset` are used to convert the node's spans -/// to relative spans, relative to the passed anchor. -/// `map` is used to resolve the converted spans accordingly. /// TODO: Flesh out the doc comment more thoroughly -pub fn syntax_node_to_token_tree( +pub fn syntax_node_to_token_tree( node: &SyntaxNode, - anchor: Anchor, - anchor_offset: TextSize, - map: &TokenMap>, + map: SpanMap, ) -> tt::Subtree> where SpanData: Span, Anchor: Copy, Ctx: SyntaxContext, + SpanMap: SpanMapper>, { - assert!(anchor_offset <= node.text_range().start()); - let mut c = Converter::new(node, anchor_offset, vec![], map); - convert_tokens(&mut c, anchor) + let mut c = Converter::new(node, vec![], map); + convert_tokens(&mut c) } -pub fn syntax_node_to_token_tree_censored( +pub fn syntax_node_to_token_tree_censored( node: &SyntaxNode, - anchor: Anchor, - anchor_offset: TextSize, - map: &TokenMap>, + map: SpanMap, censored: Vec, ) -> tt::Subtree> where + SpanMap: SpanMapper>, SpanData: Span, Anchor: Copy, Ctx: SyntaxContext, { - assert!(anchor_offset <= node.text_range().start()); - let mut c = Converter::new(node, anchor_offset, censored, map); - convert_tokens(&mut c, anchor) + let mut c = Converter::new(node, censored, map); + convert_tokens(&mut c) } // The following items are what `rustc` macro can be parsed into : @@ -113,20 +122,21 @@ where SpanData: Span, Ctx: SyntaxContext, { - let mut map = TokenMap::default(); + let mut map = TokenMap::empty(); node.descendants_with_tokens().filter_map(NodeOrToken::into_token).for_each(|t| { map.insert( t.text_range(), SpanData { range: t.text_range() - anchor_offset, anchor, ctx: Ctx::DUMMY }, ); }); + map.finish(); map } /// Convert a string to a `TokenTree` pub fn parse_to_token_tree( - text: &str, anchor: Anchor, + text: &str, ) -> Option>> where SpanData: Span, @@ -137,8 +147,8 @@ where if lexed.errors().next().is_some() { return None; } - let mut conv = RawConverter { lexed, pos: 0, _offset: TextSize::default() }; - Some(convert_tokens(&mut conv, anchor)) + let mut conv = RawConverter { lexed, pos: 0, anchor }; + Some(convert_tokens(&mut conv)) } /// Split token tree with separate expr: $($e:expr)SEP* @@ -175,10 +185,7 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec( - conv: &mut C, - anchor: Anchor, -) -> tt::Subtree> +fn convert_tokens(conv: &mut C) -> tt::Subtree> where C: TokenConverter, Ctx: SyntaxContext, @@ -188,16 +195,15 @@ where let entry = tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: vec![] }; let mut stack = NonEmptyVec::new(entry); - while let Some((token, rel_range, abs_range)) = conv.bump() { + while let Some((token, abs_range)) = conv.bump() { let tt::Subtree { delimiter, token_trees: result } = stack.last_mut(); - let mk_dummy_span = || SpanData { range: rel_range, anchor, ctx: Ctx::DUMMY }; let kind = token.kind(conv); let tt = match kind { // Desugar doc comments into doc attributes COMMENT => { - let span = conv.span_for(abs_range).unwrap_or_else(mk_dummy_span); + let span = conv.span_for(abs_range); if let Some(tokens) = conv.convert_doc_comment(&token, span) { result.extend(tokens); } @@ -215,8 +221,7 @@ where // and end the subtree here if matches!(expected, Some(expected) if expected == kind) { if let Some(mut subtree) = stack.pop() { - subtree.delimiter.close = - conv.span_for(abs_range).unwrap_or_else(mk_dummy_span); + subtree.delimiter.close = conv.span_for(abs_range); stack.last_mut().token_trees.push(subtree.into()); } continue; @@ -231,11 +236,12 @@ where // Start a new subtree if let Some(kind) = delim { + let open = conv.span_for(abs_range); stack.push(tt::Subtree { delimiter: tt::Delimiter { - open: conv.span_for(abs_range).unwrap_or_else(mk_dummy_span), + open, // will be overwritten on subtree close above - close: mk_dummy_span(), + close: open, kind, }, token_trees: vec![], @@ -250,21 +256,12 @@ where let Some(char) = token.to_char(conv) else { panic!("Token from lexer must be single char: token = {token:#?}") }; - tt::Leaf::from(tt::Punct { - char, - spacing, - span: conv.span_for(abs_range).unwrap_or_else(mk_dummy_span), - }) - .into() + tt::Leaf::from(tt::Punct { char, spacing, span: conv.span_for(abs_range) }).into() } _ => { macro_rules! make_leaf { ($i:ident) => { - tt::$i { - span: conv.span_for(abs_range).unwrap_or_else(mk_dummy_span), - text: token.to_text(conv), - } - .into() + tt::$i { span: conv.span_for(abs_range), text: token.to_text(conv) }.into() }; } let leaf: tt::Leaf<_> = match kind { @@ -273,32 +270,21 @@ where UNDERSCORE => make_leaf!(Ident), k if k.is_keyword() => make_leaf!(Ident), k if k.is_literal() => make_leaf!(Literal), - // FIXME: Check whether span splitting works as intended LIFETIME_IDENT => { - let char_unit = TextSize::of('\''); - let r = TextRange::at(rel_range.start(), char_unit); let apostrophe = tt::Leaf::from(tt::Punct { char: '\'', spacing: tt::Spacing::Joint, - span: conv.span_for(abs_range).unwrap_or(SpanData { - range: r, - anchor, - ctx: Ctx::DUMMY, - }), + span: conv + .span_for(TextRange::at(abs_range.start(), TextSize::of('\''))), }); result.push(apostrophe.into()); - let r = TextRange::at( - rel_range.start() + char_unit, - rel_range.len() - char_unit, - ); let ident = tt::Leaf::from(tt::Ident { text: SmolStr::new(&token.to_text(conv)[1..]), - span: conv.span_for(abs_range).unwrap_or(SpanData { - range: r, - anchor, - ctx: Ctx::DUMMY, - }), + span: conv.span_for(TextRange::at( + abs_range.start() + TextSize::of('\''), + abs_range.end(), + )), }); result.push(ident.into()); continue; @@ -433,10 +419,10 @@ fn convert_doc_comment( } /// A raw token (straight from lexer) converter -struct RawConverter<'a> { +struct RawConverter<'a, Anchor> { lexed: parser::LexedStr<'a>, pos: usize, - _offset: TextSize, + anchor: Anchor, } trait SrcToken: std::fmt::Debug { @@ -456,28 +442,28 @@ trait TokenConverter: Sized { span: SpanData, ) -> Option>>>; - fn bump(&mut self) -> Option<(Self::Token, TextRange, TextRange)>; + fn bump(&mut self) -> Option<(Self::Token, TextRange)>; fn peek(&self) -> Option; - fn span_for(&self, range: TextRange) -> Option>; + fn span_for(&self, range: TextRange) -> SpanData; } -impl SrcToken> for usize { - fn kind(&self, ctx: &RawConverter<'_>) -> SyntaxKind { +impl SrcToken> for usize { + fn kind(&self, ctx: &RawConverter<'_, Anchor>) -> SyntaxKind { ctx.lexed.kind(*self) } - fn to_char(&self, ctx: &RawConverter<'_>) -> Option { + fn to_char(&self, ctx: &RawConverter<'_, Anchor>) -> Option { ctx.lexed.text(*self).chars().next() } - fn to_text(&self, ctx: &RawConverter<'_>) -> SmolStr { + fn to_text(&self, ctx: &RawConverter<'_, Anchor>) -> SmolStr { ctx.lexed.text(*self).into() } } -impl TokenConverter for RawConverter<'_> +impl TokenConverter for RawConverter<'_, Anchor> where SpanData: Span, { @@ -492,7 +478,7 @@ where convert_doc_comment(&doc_comment(text), span) } - fn bump(&mut self) -> Option<(Self::Token, TextRange, TextRange)> { + fn bump(&mut self) -> Option<(Self::Token, TextRange)> { if self.pos == self.lexed.len() { return None; } @@ -501,7 +487,7 @@ where let range = self.lexed.text_range(token); let range = TextRange::new(range.start.try_into().ok()?, range.end.try_into().ok()?); - Some((token, range, range)) + Some((token, range)) } fn peek(&self) -> Option { @@ -511,41 +497,27 @@ where Some(self.pos) } - fn span_for(&self, _: TextRange) -> Option> { - None + fn span_for(&self, range: TextRange) -> SpanData { + SpanData { range, anchor: self.anchor, ctx: Ctx::DUMMY } } } -struct Converter<'a, Anchor, Ctx> { +struct Converter { current: Option, preorder: PreorderWithTokens, range: TextRange, punct_offset: Option<(SyntaxToken, TextSize)>, /// Used to make the emitted text ranges in the spans relative to the span anchor. - offset: TextSize, - map: &'a TokenMap>, + map: SpanMap, censored: Vec, } -impl<'a, Anchor, Ctx> Converter<'a, Anchor, Ctx> { - fn new( - node: &SyntaxNode, - anchor_offset: TextSize, - censored: Vec, - map: &'a TokenMap>, - ) -> Self { +impl Converter { + fn new(node: &SyntaxNode, censored: Vec, map: SpanMap) -> Self { let range = node.text_range(); let mut preorder = node.preorder_with_tokens(); let first = Self::next_token(&mut preorder, &censored); - Converter { - current: first, - preorder, - range, - punct_offset: None, - offset: anchor_offset, - censored, - map, - } + Converter { current: first, preorder, range, punct_offset: None, censored, map } } fn next_token(preorder: &mut PreorderWithTokens, censor: &[SyntaxNode]) -> Option { @@ -577,29 +549,30 @@ impl SynToken { } } -impl SrcToken> for SynToken { - fn kind(&self, ctx: &Converter<'_, Anchor, Ctx>) -> SyntaxKind { +impl SrcToken> for SynToken { + fn kind(&self, ctx: &Converter) -> SyntaxKind { match self { SynToken::Ordinary(token) => token.kind(), SynToken::Punct(..) => SyntaxKind::from_char(self.to_char(ctx).unwrap()).unwrap(), } } - fn to_char(&self, _ctx: &Converter<'_, Anchor, Ctx>) -> Option { + fn to_char(&self, _ctx: &Converter) -> Option { match self { SynToken::Ordinary(_) => None, SynToken::Punct(it, i) => it.text().chars().nth(*i), } } - fn to_text(&self, _ctx: &Converter<'_, Anchor, Ctx>) -> SmolStr { + fn to_text(&self, _ctx: &Converter) -> SmolStr { match self { SynToken::Ordinary(token) | SynToken::Punct(token, _) => token.text().into(), } } } -impl TokenConverter for Converter<'_, Anchor, Ctx> +impl TokenConverter for Converter where SpanData: Span, + SpanMap: SpanMapper>, { type Token = SynToken; fn convert_doc_comment( @@ -610,18 +583,14 @@ where convert_doc_comment(token.token(), span) } - fn bump(&mut self) -> Option<(Self::Token, TextRange, TextRange)> { + fn bump(&mut self) -> Option<(Self::Token, TextRange)> { if let Some((punct, offset)) = self.punct_offset.clone() { if usize::from(offset) + 1 < punct.text().len() { let offset = offset + TextSize::of('.'); let range = punct.text_range(); self.punct_offset = Some((punct.clone(), offset)); let range = TextRange::at(range.start() + offset, TextSize::of('.')); - return Some(( - SynToken::Punct(punct, u32::from(offset) as usize), - range - self.offset, - range, - )); + return Some((SynToken::Punct(punct, u32::from(offset) as usize), range)); } } @@ -634,11 +603,11 @@ where self.punct_offset = Some((curr.clone(), 0.into())); let range = curr.text_range(); let range = TextRange::at(range.start(), TextSize::of('.')); - (SynToken::Punct(curr, 0 as usize), range - self.offset, range) + (SynToken::Punct(curr, 0 as usize), range) } else { self.punct_offset = None; let range = curr.text_range(); - (SynToken::Ordinary(curr), range - self.offset, range) + (SynToken::Ordinary(curr), range) }; Some(token) @@ -665,12 +634,15 @@ where Some(token) } - fn span_for(&self, range: TextRange) -> Option> { - self.map.span_for_range(range) + fn span_for(&self, range: TextRange) -> SpanData { + self.map.span_for(range) } } -struct TtTreeSink<'a, Anchor, Ctx> { +struct TtTreeSink<'a, Anchor, Ctx> +where + SpanData: Span, +{ buf: String, cursor: Cursor<'a, SpanData>, text_pos: TextSize, @@ -688,12 +660,12 @@ where cursor, text_pos: 0.into(), inner: SyntaxTreeBuilder::default(), - token_map: TokenMap::default(), + token_map: TokenMap::empty(), } } fn finish(mut self) -> (Parse, TokenMap>) { - self.token_map.shrink_to_fit(); + self.token_map.finish(); (self.inner.finish(), self.token_map) } } @@ -825,6 +797,7 @@ where self.inner.token(kind, self.buf.as_str()); self.buf.clear(); + // FIXME: Emitting whitespace for this is really just a hack, we should get rid of it. // Add whitespace between adjoint puncts let next = last.bump(); if let ( @@ -839,6 +812,7 @@ where // need to add whitespace either. if curr.spacing == tt::Spacing::Alone && curr.char != ';' && next.char != '\'' { self.inner.token(WHITESPACE, " "); + self.token_map.insert(TextRange::at(self.text_pos, TextSize::of(' ')), curr.span); self.text_pos += TextSize::of(' '); } } diff --git a/crates/mbe/src/syntax_bridge/tests.rs b/crates/mbe/src/syntax_bridge/tests.rs index 0275e5397c9b..2e21977f681b 100644 --- a/crates/mbe/src/syntax_bridge/tests.rs +++ b/crates/mbe/src/syntax_bridge/tests.rs @@ -7,12 +7,14 @@ use tt::{ Leaf, Punct, Spacing, SpanAnchor, SyntaxContext, }; +use crate::SpanMapper; + use super::syntax_node_to_token_tree; fn check_punct_spacing(fixture: &str) { type SpanData = tt::SpanData; - #[derive(PartialEq, Eq, Clone, Copy, Debug)] + #[derive(PartialEq, Eq, Clone, Copy, Debug, Hash)] struct DummyFile; impl SpanAnchor for DummyFile { const DUMMY: Self = DummyFile; @@ -24,9 +26,16 @@ fn check_punct_spacing(fixture: &str) { const DUMMY: Self = DummyCtx; } + struct NoOpMap; + + impl SpanMapper for NoOpMap { + fn span_for(&self, range: syntax::TextRange) -> SpanData { + SpanData { range, anchor: DummyFile, ctx: DummyCtx } + } + } + let source_file = ast::SourceFile::parse(fixture).ok().unwrap(); - let subtree = - syntax_node_to_token_tree(source_file.syntax(), DummyFile, 0.into(), &Default::default()); + let subtree = syntax_node_to_token_tree(source_file.syntax(), NoOpMap); let mut annotations: HashMap<_, _> = extract_annotations(fixture) .into_iter() .map(|(range, annotation)| { diff --git a/crates/mbe/src/token_map.rs b/crates/mbe/src/token_map.rs index dfbf54410b18..978ee268c5cc 100644 --- a/crates/mbe/src/token_map.rs +++ b/crates/mbe/src/token_map.rs @@ -2,7 +2,7 @@ use std::hash::Hash; -use stdx::never; +use stdx::itertools::Itertools; use syntax::TextRange; use tt::Span; @@ -15,23 +15,29 @@ use tt::Span; /// Maps absolute text ranges for the corresponding file to the relevant span data. #[derive(Debug, PartialEq, Eq, Clone, Hash)] // FIXME: Rename to SpanMap -pub struct TokenMap { +pub struct TokenMap { // FIXME: This needs to be sorted by (FileId, AstId) // Then we can do a binary search on the file id, // then a bin search on the ast id pub span_map: Vec<(TextRange, S)>, // span_map2: rustc_hash::FxHashMap, - pub real_file: bool, } -impl Default for TokenMap { - fn default() -> Self { - Self { span_map: Vec::new(), real_file: true } +impl TokenMap { + pub fn empty() -> Self { + Self { span_map: Vec::new() } } -} -impl TokenMap { - pub(crate) fn shrink_to_fit(&mut self) { + pub fn finish(&mut self) { + debug_assert_eq!( + self.span_map + .iter() + .sorted_by_key(|it| (it.0.start(), it.0.end())) + .tuple_windows() + .find(|(range, next)| range.0.end() != next.0.start()), + None, + "span map has holes!" + ); self.span_map.shrink_to_fit(); } @@ -40,6 +46,8 @@ impl TokenMap { } pub fn ranges_with_span(&self, span: S) -> impl Iterator + '_ { + // FIXME: linear search + // FIXME: Disregards resolving spans to get more matches! See ExpansionInfo::map_token_down self.span_map.iter().filter_map( move |(range, s)| { if s == &span { @@ -51,20 +59,31 @@ impl TokenMap { ) } - // FIXME: Should be infallible - pub fn span_for_range(&self, range: TextRange) -> Option { + // FIXME: We need APIs for fetching the span of a token as well as for a whole node. The node + // one *is* fallible though. + // Token span fetching technically only needs an offset really, as the entire file span is + // populated, where node fetching is more like fetching the spans at all source positions, and + // then we need to verify that all those positions have the same context, if not we fail! But + // how do we handle them having different span ranges? + + pub fn span_for_range(&self, range: TextRange) -> S { // TODO FIXME: make this proper self.span_map .iter() - .filter_map(|(r, s)| Some((r, s, r.intersect(range)?))) + .filter_map(|(r, s)| Some((r, s, r.intersect(range).filter(|it| !it.is_empty())?))) .max_by_key(|(_, _, intersection)| intersection.len()) - .map(|(_, &s, _)| s) - .or_else(|| { - if !self.real_file { - never!("no span for range {:?} in {:#?}", range, self.span_map); - } - None - }) + .map_or_else( + || panic!("no span for range {:?} in {:#?}", range, self.span_map), + |(_, &s, _)| s, + ) + } + + pub fn spans_for_node_range(&self, range: TextRange) -> impl Iterator + '_ { + // TODO FIXME: make this proper + self.span_map + .iter() + .filter(move |(r, _)| r.intersect(range).filter(|it| !it.is_empty()).is_some()) + .map(|&(_, s)| s) } // pub fn ranges_by_token( diff --git a/crates/proc-macro-srv/src/lib.rs b/crates/proc-macro-srv/src/lib.rs index 84bd15efb8be..00042480e302 100644 --- a/crates/proc-macro-srv/src/lib.rs +++ b/crates/proc-macro-srv/src/lib.rs @@ -10,7 +10,7 @@ //! * By **copying** the whole rustc `lib_proc_macro` code, we are able to build this with `stable` //! rustc rather than `unstable`. (Although in general ABI compatibility is still an issue)… -#![cfg(feature = "sysroot-abi")] +#![cfg(any(feature = "sysroot-abi", rust_analyzer))] #![feature(proc_macro_internals, proc_macro_diagnostic, proc_macro_span)] #![warn(rust_2018_idioms, unused_lifetimes, semicolon_in_expressions_from_macros)] #![allow(unreachable_pub)] diff --git a/crates/rust-analyzer/src/cargo_target_spec.rs b/crates/rust-analyzer/src/cargo_target_spec.rs index 4742dc1dfa79..d679bfcb014c 100644 --- a/crates/rust-analyzer/src/cargo_target_spec.rs +++ b/crates/rust-analyzer/src/cargo_target_spec.rs @@ -209,24 +209,26 @@ mod tests { use super::*; use cfg::CfgExpr; - use hir::HirFileId; - use ide_db::base_db::span::{SpanAnchor, SyntaxContextId, ROOT_ERASED_FILE_AST_ID}; - use mbe::syntax_node_to_token_tree; + use hir_def::tt::{self, Span}; + use mbe::{syntax_node_to_token_tree, SpanMapper}; use syntax::{ ast::{self, AstNode}, - SmolStr, TextSize, + SmolStr, }; + struct NoOpMap; + + impl SpanMapper for NoOpMap { + fn span_for(&self, _: syntax::TextRange) -> tt::SpanData { + tt::SpanData::DUMMY + } + } + fn check(cfg: &str, expected_features: &[&str]) { let cfg_expr = { let source_file = ast::SourceFile::parse(cfg).ok().unwrap(); let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - let tt = syntax_node_to_token_tree::<_, SyntaxContextId>( - tt.syntax(), - SpanAnchor { file_id: HirFileId::from(0), ast_id: ROOT_ERASED_FILE_AST_ID }, - TextSize::new(0), - &Default::default(), - ); + let tt = syntax_node_to_token_tree(tt.syntax(), &NoOpMap); CfgExpr::parse(&tt) }; diff --git a/crates/stdx/Cargo.toml b/crates/stdx/Cargo.toml index 35986799fdd4..ea6c11ac0d56 100644 --- a/crates/stdx/Cargo.toml +++ b/crates/stdx/Cargo.toml @@ -17,6 +17,7 @@ always-assert = { version = "0.1.2", features = ["log"] } jod-thread = "0.1.2" libc.workspace = true crossbeam-channel = "0.5.5" +itertools.workspace = true # Think twice before adding anything here [target.'cfg(windows)'.dependencies] diff --git a/crates/stdx/src/lib.rs b/crates/stdx/src/lib.rs index f26c7f6dfc2d..af7846d49c64 100644 --- a/crates/stdx/src/lib.rs +++ b/crates/stdx/src/lib.rs @@ -15,6 +15,7 @@ pub mod thread; pub mod anymap; pub use always_assert::{always, never}; +pub use itertools; #[inline(always)] pub fn is_ci() -> bool { @@ -40,6 +41,24 @@ Uncomment `default = [ "backtrace" ]` in `crates/stdx/Cargo.toml`. ); } +pub trait TupleExt { + type Head; + type Tail; + fn head(self) -> Self::Head; + fn tail(self) -> Self::Tail; +} + +impl TupleExt for (T, U) { + type Head = T; + type Tail = U; + fn head(self) -> Self::Head { + self.0 + } + fn tail(self) -> Self::Tail { + self.1 + } +} + pub fn to_lower_snake_case(s: &str) -> String { to_snake_case(s, char::to_lowercase) } diff --git a/crates/tt/src/lib.rs b/crates/tt/src/lib.rs index 73e75a051bd6..1374ae1a6fc6 100644 --- a/crates/tt/src/lib.rs +++ b/crates/tt/src/lib.rs @@ -45,20 +45,32 @@ pub struct SpanData { } impl Span for SpanData { + type Anchor = Anchor; const DUMMY: Self = SpanData { range: TextRange::empty(TextSize::new(0)), anchor: Anchor::DUMMY, ctx: Ctx::DUMMY, }; + fn anchor(self) -> Self::Anchor { + self.anchor + } + fn mk(anchor: Self::Anchor, range: TextRange) -> Self { + SpanData { anchor, range, ctx: Ctx::DUMMY } + } } -pub trait SpanAnchor: std::fmt::Debug + Copy + Sized + Eq { +pub trait SpanAnchor: + std::fmt::Debug + Copy + Sized + Eq + Copy + fmt::Debug + std::hash::Hash +{ const DUMMY: Self; } // FIXME: Get rid of this trait? pub trait Span: std::fmt::Debug + Copy + Sized + Eq { const DUMMY: Self; + type Anchor: Copy + fmt::Debug + Eq + std::hash::Hash; + fn anchor(self) -> Self::Anchor; + fn mk(anchor: Self::Anchor, range: TextRange) -> Self; } pub trait SyntaxContext: std::fmt::Debug + Copy + Sized + Eq { From c43078f99d93af2bdc8f799ae77e10127aba39ca Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 25 Nov 2023 14:39:55 +0100 Subject: [PATCH 09/24] Re-implement InFile wrappers as type aliases over generic InFileWrapper --- crates/base-db/src/span.rs | 20 +-- .../hir-def/src/macro_expansion_tests/mod.rs | 4 +- crates/hir-expand/src/db.rs | 8 +- crates/hir-expand/src/files.rs | 155 +++++++++++------- crates/hir-expand/src/lib.rs | 10 +- crates/hir/src/lib.rs | 6 +- crates/hir/src/semantics.rs | 12 +- crates/hir/src/symbols.rs | 5 +- .../src/handlers/generate_enum_variant.rs | 6 +- crates/ide-db/src/rename.rs | 2 +- crates/ide/src/annotations.rs | 6 +- crates/ide/src/status.rs | 10 +- crates/proc-macro-api/src/msg/flat.rs | 4 +- 13 files changed, 146 insertions(+), 102 deletions(-) diff --git a/crates/base-db/src/span.rs b/crates/base-db/src/span.rs index a78f558759b0..e183e9b19965 100644 --- a/crates/base-db/src/span.rs +++ b/crates/base-db/src/span.rs @@ -102,7 +102,7 @@ impl fmt::Debug for HirFileId { } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct MacroFile { +pub struct MacroFileId { pub macro_call_id: MacroCallId, } /// `MacroCallId` identifies a particular macro invocation, like @@ -113,18 +113,18 @@ crate::impl_intern_key!(MacroCallId); impl MacroCallId { pub fn as_file(self) -> HirFileId { - MacroFile { macro_call_id: self }.into() + MacroFileId { macro_call_id: self }.into() } - pub fn as_macro_file(self) -> MacroFile { - MacroFile { macro_call_id: self } + pub fn as_macro_file(self) -> MacroFileId { + MacroFileId { macro_call_id: self } } } #[derive(Clone, Copy, PartialEq, Eq, Hash)] pub enum HirFileIdRepr { FileId(FileId), - MacroFile(MacroFile), + MacroFile(MacroFileId), } impl fmt::Debug for HirFileIdRepr { @@ -145,8 +145,8 @@ impl From for HirFileId { } } -impl From for HirFileId { - fn from(MacroFile { macro_call_id: MacroCallId(id) }: MacroFile) -> Self { +impl From for HirFileId { + fn from(MacroFileId { macro_call_id: MacroCallId(id) }: MacroFileId) -> Self { let id = id.as_u32(); assert!(id < Self::MAX_FILE_ID); HirFileId(id | Self::MACRO_FILE_TAG_MASK) @@ -163,10 +163,10 @@ impl HirFileId { } #[inline] - pub fn macro_file(self) -> Option { + pub fn macro_file(self) -> Option { match self.0 & Self::MACRO_FILE_TAG_MASK { 0 => None, - _ => Some(MacroFile { + _ => Some(MacroFileId { macro_call_id: MacroCallId(InternId::from(self.0 ^ Self::MACRO_FILE_TAG_MASK)), }), } @@ -184,7 +184,7 @@ impl HirFileId { pub fn repr(self) -> HirFileIdRepr { match self.0 & Self::MACRO_FILE_TAG_MASK { 0 => HirFileIdRepr::FileId(FileId(self.0)), - _ => HirFileIdRepr::MacroFile(MacroFile { + _ => HirFileIdRepr::MacroFile(MacroFileId { macro_call_id: MacroCallId(InternId::from(self.0 ^ Self::MACRO_FILE_TAG_MASK)), }), } diff --git a/crates/hir-def/src/macro_expansion_tests/mod.rs b/crates/hir-def/src/macro_expansion_tests/mod.rs index 27ec63d171b0..355b82a5f42c 100644 --- a/crates/hir-def/src/macro_expansion_tests/mod.rs +++ b/crates/hir-def/src/macro_expansion_tests/mod.rs @@ -18,7 +18,7 @@ use std::{iter, ops::Range, sync}; use base_db::{fixture::WithFixture, ProcMacro, SourceDatabase}; use expect_test::Expect; -use hir_expand::{db::ExpandDatabase, span::SpanMapRef, HirFileIdExt, InFile, MacroFile}; +use hir_expand::{db::ExpandDatabase, span::SpanMapRef, HirFileIdExt, InFile, MacroFileId}; use stdx::format_to; use syntax::{ ast::{self, edit::IndentLevel}, @@ -94,7 +94,7 @@ pub fn identity_when_valid(_attr: TokenStream, item: TokenStream) -> TokenStream }) .unwrap(); let macro_call_id = res.value.unwrap(); - let macro_file = MacroFile { macro_call_id }; + let macro_file = MacroFileId { macro_call_id }; let mut expansion_result = db.parse_macro_expansion(macro_file); expansion_result.err = expansion_result.err.or(res.err); expansions.push((macro_call.value.clone(), expansion_result)); diff --git a/crates/hir-expand/src/db.rs b/crates/hir-expand/src/db.rs index 393e391f0517..e31034884c5e 100644 --- a/crates/hir-expand/src/db.rs +++ b/crates/hir-expand/src/db.rs @@ -24,7 +24,7 @@ use crate::{ span::{RealSpanMap, SpanMap, SpanMapRef}, tt, AstId, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander, EagerCallInfo, ExpandError, ExpandResult, ExpandTo, ExpansionSpanMap, HirFileId, HirFileIdRepr, MacroCallId, - MacroCallKind, MacroCallLoc, MacroDefId, MacroDefKind, MacroFile, ProcMacroExpander, + MacroCallKind, MacroCallLoc, MacroDefId, MacroDefKind, MacroFileId, ProcMacroExpander, }; /// Total limit on the number of tokens produced by any macro invocation. @@ -102,7 +102,7 @@ pub trait ExpandDatabase: SourceDatabase { // This query is LRU cached fn parse_macro_expansion( &self, - macro_file: MacroFile, + macro_file: MacroFileId, ) -> ExpandResult<(Parse, Arc)>; #[salsa::transparent] fn span_map(&self, file_id: HirFileId) -> SpanMap; @@ -307,7 +307,7 @@ fn parse_or_expand_with_err( fn parse_macro_expansion( db: &dyn ExpandDatabase, - macro_file: MacroFile, + macro_file: MacroFileId, ) -> ExpandResult<(Parse, Arc)> { let _p = profile::span("parse_macro_expansion"); let mbe::ValueResult { value: tt, err } = db.macro_expand(macro_file.macro_call_id); @@ -326,7 +326,7 @@ fn parse_macro_expansion_error( db: &dyn ExpandDatabase, macro_call_id: MacroCallId, ) -> ExpandResult> { - db.parse_macro_expansion(MacroFile { macro_call_id }) + db.parse_macro_expansion(MacroFileId { macro_call_id }) .map(|it| it.0.errors().to_vec().into_boxed_slice()) } diff --git a/crates/hir-expand/src/files.rs b/crates/hir-expand/src/files.rs index 45875d949817..57a7fa5ec398 100644 --- a/crates/hir-expand/src/files.rs +++ b/crates/hir-expand/src/files.rs @@ -1,15 +1,14 @@ use std::iter; use base_db::{ - span::{HirFileId, HirFileIdRepr, MacroFile, SyntaxContextId}, - FileRange, + span::{HirFileId, HirFileIdRepr, MacroFileId, SyntaxContextId}, + FileId, FileRange, }; use either::Either; use syntax::{AstNode, SyntaxNode, SyntaxToken, TextRange}; use crate::{db, ExpansionInfo, HirFileIdExt as _}; -// FIXME: Make an InRealFile wrapper /// `InFile` stores a value of `T` inside a particular file/syntax tree. /// /// Typical usages are: @@ -18,55 +17,91 @@ use crate::{db, ExpansionInfo, HirFileIdExt as _}; /// * `InFile` -- ast node in a file /// * `InFile` -- offset in a file #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] -pub struct InFile { - pub file_id: HirFileId, +pub struct InFileWrapper { + pub file_id: FileKind, pub value: T, } +pub type InFile = InFileWrapper; +pub type InMacroFile = InFileWrapper; +pub type InRealFile = InFileWrapper; -impl InFile { - pub fn new(file_id: HirFileId, value: T) -> InFile { - InFile { file_id, value } +impl InFileWrapper { + pub fn new(file_id: FileKind, value: T) -> Self { + Self { file_id, value } } - pub fn with_value(&self, value: U) -> InFile { - InFile::new(self.file_id, value) + pub fn map U, U>(self, f: F) -> InFileWrapper { + InFileWrapper::new(self.file_id, f(self.value)) } +} - pub fn map U, U>(self, f: F) -> InFile { - InFile::new(self.file_id, f(self.value)) +impl InFileWrapper { + pub fn with_value(&self, value: U) -> InFileWrapper { + InFileWrapper::new(self.file_id, value) } - pub fn as_ref(&self) -> InFile<&T> { + pub fn as_ref(&self) -> InFileWrapper { self.with_value(&self.value) } +} - pub fn file_syntax(&self, db: &dyn db::ExpandDatabase) -> SyntaxNode { - db.parse_or_expand(self.file_id) +impl InFileWrapper { + pub fn cloned(&self) -> InFileWrapper { + self.with_value(self.value.clone()) } } -impl InFile<&T> { - pub fn cloned(&self) -> InFile { - self.with_value(self.value.clone()) +impl From> for InFile { + fn from(InMacroFile { file_id, value }: InMacroFile) -> Self { + InFile { file_id: file_id.into(), value } } } -impl InFile> { - pub fn transpose(self) -> Option> { - let value = self.value?; - Some(InFile::new(self.file_id, value)) +impl From> for InFile { + fn from(InRealFile { file_id, value }: InRealFile) -> Self { + InFile { file_id: file_id.into(), value } } } -impl InFile> { - pub fn transpose(self) -> Either, InFile> { +// region:transpose impls + +impl InFileWrapper> { + pub fn transpose(self) -> Option> { + Some(InFileWrapper::new(self.file_id, self.value?)) + } +} + +impl InFileWrapper> { + pub fn transpose(self) -> Either, InFileWrapper> { match self.value { - Either::Left(l) => Either::Left(InFile::new(self.file_id, l)), - Either::Right(r) => Either::Right(InFile::new(self.file_id, r)), + Either::Left(l) => Either::Left(InFileWrapper::new(self.file_id, l)), + Either::Right(r) => Either::Right(InFileWrapper::new(self.file_id, r)), } } } +// endregion:transpose impls + +// region:specific impls + +impl InFile { + pub fn file_syntax(&self, db: &dyn db::ExpandDatabase) -> SyntaxNode { + db.parse_or_expand(self.file_id) + } +} + +impl InRealFile { + pub fn file_syntax(&self, db: &dyn db::ExpandDatabase) -> SyntaxNode { + db.parse(self.file_id).syntax_node() + } +} + +impl InMacroFile { + pub fn file_syntax(&self, db: &dyn db::ExpandDatabase) -> SyntaxNode { + db.parse_macro_expansion(self.file_id).value.0.syntax_node() + } +} + impl InFile<&SyntaxNode> { pub fn ancestors_with_macros( self, @@ -159,11 +194,17 @@ impl InFile<&SyntaxNode> { } } - pub fn original_syntax_node(self, db: &dyn db::ExpandDatabase) -> Option> { + pub fn original_syntax_node( + self, + db: &dyn db::ExpandDatabase, + ) -> Option> { // This kind of upmapping can only be achieved in attribute expanded files, // as we don't have node inputs otherwise and therefore can't find an `N` node in the input - let Some(file_id) = self.file_id.macro_file() else { - return Some(self.map(Clone::clone)); + let file_id = match self.file_id.repr() { + HirFileIdRepr::FileId(file_id) => { + return Some(InRealFile { file_id, value: self.value.clone() }) + } + HirFileIdRepr::MacroFile(m) => m, }; if !self.file_id.is_attr_macro(db) { return None; @@ -182,7 +223,7 @@ impl InFile<&SyntaxNode> { let kind = self.value.kind(); // FIXME: This heuristic is brittle and with the right macro may select completely unrelated nodes? let value = anc.ancestors().find(|it| it.kind() == kind)?; - Some(InFile::new(file_id.into(), value)) + Some(InRealFile::new(file_id, value)) } } @@ -230,29 +271,11 @@ impl InFile { } } -#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] -pub struct InMacroFile { - pub file_id: MacroFile, - pub value: T, -} - -impl From> for InFile { - fn from(macro_file: InMacroFile) -> Self { - InFile { file_id: macro_file.file_id.into(), value: macro_file.value } - } -} - -pub fn ascend_range_up_macros( - db: &dyn db::ExpandDatabase, - range: InFile, -) -> (FileRange, SyntaxContextId) { - match range.file_id.repr() { - HirFileIdRepr::FileId(file_id) => { - (FileRange { file_id, range: range.value }, SyntaxContextId::ROOT) - } - HirFileIdRepr::MacroFile(m) => { - ExpansionInfo::new(db, m).map_token_range_up(db, range.value) - } +impl InFile { + /// Attempts to map the syntax node back up its macro calls. + pub fn original_file_range(self, db: &dyn db::ExpandDatabase) -> FileRange { + let (range, _ctxt) = ascend_range_up_macros(db, self); + range } } @@ -261,12 +284,14 @@ impl InFile { self.value.syntax().descendants().filter_map(T::cast).map(move |n| self.with_value(n)) } - // FIXME: this should return `Option>` - pub fn original_ast_node(self, db: &dyn db::ExpandDatabase) -> Option> { + pub fn original_ast_node(self, db: &dyn db::ExpandDatabase) -> Option> { // This kind of upmapping can only be achieved in attribute expanded files, // as we don't have node inputs otherwise and therefore can't find an `N` node in the input - let Some(file_id) = self.file_id.macro_file() else { - return Some(self); + let file_id = match self.file_id.repr() { + HirFileIdRepr::FileId(file_id) => { + return Some(InRealFile { file_id, value: self.value }) + } + HirFileIdRepr::MacroFile(m) => m, }; if !self.file_id.is_attr_macro(db) { return None; @@ -284,10 +309,24 @@ impl InFile { // FIXME: This heuristic is brittle and with the right macro may select completely unrelated nodes? let anc = db.parse(file_id).syntax_node().covering_element(range); let value = anc.ancestors().find_map(N::cast)?; - return Some(InFile::new(file_id.into(), value)); + Some(InRealFile::new(file_id, value)) } pub fn syntax(&self) -> InFile<&SyntaxNode> { self.with_value(self.value.syntax()) } } + +fn ascend_range_up_macros( + db: &dyn db::ExpandDatabase, + range: InFile, +) -> (FileRange, SyntaxContextId) { + match range.file_id.repr() { + HirFileIdRepr::FileId(file_id) => { + (FileRange { file_id, range: range.value }, SyntaxContextId::ROOT) + } + HirFileIdRepr::MacroFile(m) => { + ExpansionInfo::new(db, m).map_token_range_up(db, range.value) + } + } +} diff --git a/crates/hir-expand/src/lib.rs b/crates/hir-expand/src/lib.rs index d2e5e7c3643f..13eb90b18a8e 100644 --- a/crates/hir-expand/src/lib.rs +++ b/crates/hir-expand/src/lib.rs @@ -48,9 +48,9 @@ use crate::{ }; pub use crate::ast_id_map::{AstId, ErasedAstId, ErasedFileAstId}; -pub use crate::files::{InFile, InMacroFile}; +pub use crate::files::{InFile, InMacroFile, InRealFile}; -pub use base_db::span::{HirFileId, MacroCallId, MacroFile}; +pub use base_db::span::{HirFileId, MacroCallId, MacroFileId}; pub use mbe::ValueResult; pub type DeclarativeMacro = ::mbe::DeclarativeMacro; @@ -206,7 +206,7 @@ impl HirFileIdExt for HirFileId { loop { match file_id.repr() { HirFileIdRepr::FileId(id) => break id, - HirFileIdRepr::MacroFile(MacroFile { macro_call_id }) => { + HirFileIdRepr::MacroFile(MacroFileId { macro_call_id }) => { let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_call_id); let is_include_expansion = loc.def.is_include() && loc.eager.is_some(); file_id = match is_include_expansion.then(|| db.include_expand(macro_call_id)) { @@ -241,7 +241,7 @@ impl HirFileIdExt for HirFileId { loop { match call.file_id.repr() { HirFileIdRepr::FileId(file_id) => break Some((file_id, call.value)), - HirFileIdRepr::MacroFile(MacroFile { macro_call_id }) => { + HirFileIdRepr::MacroFile(MacroFileId { macro_call_id }) => { call = db.lookup_intern_macro_call(macro_call_id).to_node(db); } } @@ -700,7 +700,7 @@ impl ExpansionInfo { } } - pub fn new(db: &dyn db::ExpandDatabase, macro_file: MacroFile) -> ExpansionInfo { + pub fn new(db: &dyn db::ExpandDatabase, macro_file: MacroFileId) -> ExpansionInfo { let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_file.macro_call_id); let arg_tt = loc.kind.arg(db); diff --git a/crates/hir/src/lib.rs b/crates/hir/src/lib.rs index 443c19af82d3..aacc3b08cb79 100644 --- a/crates/hir/src/lib.rs +++ b/crates/hir/src/lib.rs @@ -59,7 +59,7 @@ use hir_def::{ Lookup, MacroExpander, MacroId, ModuleId, StaticId, StructId, TraitAliasId, TraitId, TypeAliasId, TypeOrConstParamId, TypeParamId, UnionId, }; -use hir_expand::{name::name, InMacroFile, MacroCallKind}; +use hir_expand::{name::name, MacroCallKind}; use hir_ty::{ all_super_traits, autoderef, check_orphan_rules, consteval::{try_const_usize, unknown_const_as_generic, ConstEvalError, ConstExt}, @@ -124,7 +124,7 @@ pub use { hir_expand::{ attrs::{Attr, AttrId}, name::{known, Name}, - tt, ExpandResult, HirFileId, HirFileIdExt, InFile, MacroFile, + tt, ExpandResult, HirFileId, HirFileIdExt, InFile, InMacroFile, InRealFile, MacroFileId, }, hir_ty::{ display::{ClosureStyle, HirDisplay, HirDisplayError, HirWrite}, @@ -3505,7 +3505,7 @@ impl Impl { } _ => return None, }; - let file_id = MacroFile { macro_call_id: derive_attr }; + let file_id = MacroFileId { macro_call_id: derive_attr }; let path = db .parse_macro_expansion(file_id) .value diff --git a/crates/hir/src/semantics.rs b/crates/hir/src/semantics.rs index 137cffa7af15..e124e14a5422 100644 --- a/crates/hir/src/semantics.rs +++ b/crates/hir/src/semantics.rs @@ -15,7 +15,9 @@ use hir_def::{ type_ref::Mutability, AsMacroCall, DefWithBodyId, FieldId, FunctionId, MacroId, TraitId, VariantId, }; -use hir_expand::{db::ExpandDatabase, name::AsName, ExpansionInfo, HirFileIdExt, MacroCallId}; +use hir_expand::{ + db::ExpandDatabase, files::InRealFile, name::AsName, ExpansionInfo, HirFileIdExt, MacroCallId, +}; use itertools::Itertools; use rustc_hash::{FxHashMap, FxHashSet}; use smallvec::{smallvec, SmallVec}; @@ -756,8 +758,8 @@ impl<'db> SemanticsImpl<'db> { /// This only work for attribute expansions, as other ones do not have nodes as input. pub fn original_ast_node(&self, node: N) -> Option { self.wrap_node_infile(node).original_ast_node(self.db.upcast()).map( - |InFile { file_id, value }| { - self.cache(find_root(value.syntax()), file_id); + |InRealFile { file_id, value }| { + self.cache(find_root(value.syntax()), file_id.into()); value }, ) @@ -768,8 +770,8 @@ impl<'db> SemanticsImpl<'db> { pub fn original_syntax_node(&self, node: &SyntaxNode) -> Option { let InFile { file_id, .. } = self.find_file(node); InFile::new(file_id, node).original_syntax_node(self.db.upcast()).map( - |InFile { file_id, value }| { - self.cache(find_root(&value), file_id); + |InRealFile { file_id, value }| { + self.cache(find_root(&value), file_id.into()); value }, ) diff --git a/crates/hir/src/symbols.rs b/crates/hir/src/symbols.rs index 6ea926cc22d8..a392070fd88e 100644 --- a/crates/hir/src/symbols.rs +++ b/crates/hir/src/symbols.rs @@ -7,7 +7,7 @@ use hir_def::{ AdtId, AssocItemId, DefWithBodyId, HasModule, ImplId, Lookup, MacroId, ModuleDefId, ModuleId, TraitId, }; -use hir_expand::{files::ascend_range_up_macros, HirFileId, InFile}; +use hir_expand::{HirFileId, InFile}; use hir_ty::db::HirDatabase; use syntax::{ast::HasName, AstNode, SmolStr, SyntaxNode, SyntaxNodePtr}; @@ -51,8 +51,7 @@ impl DeclarationLocation { } pub fn original_name_range(&self, db: &dyn HirDatabase) -> FileRange { - let mapping = InFile::new(self.hir_file_id, self.name_ptr.text_range()); - ascend_range_up_macros(db.upcast(), mapping).0 + InFile::new(self.hir_file_id, self.name_ptr.text_range()).original_file_range(db.upcast()) } } diff --git a/crates/ide-assists/src/handlers/generate_enum_variant.rs b/crates/ide-assists/src/handlers/generate_enum_variant.rs index be7a5e6c8bcc..1a1e992e28a4 100644 --- a/crates/ide-assists/src/handlers/generate_enum_variant.rs +++ b/crates/ide-assists/src/handlers/generate_enum_variant.rs @@ -1,4 +1,4 @@ -use hir::{HasSource, HirDisplay, HirFileIdExt, InFile}; +use hir::{HasSource, HirDisplay, InRealFile}; use ide_db::assists::{AssistId, AssistKind}; use syntax::{ ast::{self, make, HasArgList}, @@ -114,14 +114,14 @@ fn add_variant_to_accumulator( parent: PathParent, ) -> Option<()> { let db = ctx.db(); - let InFile { file_id, value: enum_node } = adt.source(db)?.original_ast_node(db)?; + let InRealFile { file_id, value: enum_node } = adt.source(db)?.original_ast_node(db)?; acc.add( AssistId("generate_enum_variant", AssistKind::Generate), "Generate variant", target, |builder| { - builder.edit_file(file_id.original_file(db)); + builder.edit_file(file_id); let node = builder.make_mut(enum_node); let variant = make_variant(ctx, name_ref, parent); node.variant_list().map(|it| it.add_variant(variant.clone_for_update())); diff --git a/crates/ide-db/src/rename.rs b/crates/ide-db/src/rename.rs index 9f101468a350..676d286b8dcc 100644 --- a/crates/ide-db/src/rename.rs +++ b/crates/ide-db/src/rename.rs @@ -533,7 +533,7 @@ fn source_edit_from_def( } }, }; - file_id = source.file_id.file_id(); + file_id = Some(source.file_id); if let Either::Left(pat) = source.value { let name_range = pat.name().unwrap().syntax().text_range(); // special cases required for renaming fields/locals in Record patterns diff --git a/crates/ide/src/annotations.rs b/crates/ide/src/annotations.rs index fb79b5dc211a..d7f82b4af3e1 100644 --- a/crates/ide/src/annotations.rs +++ b/crates/ide/src/annotations.rs @@ -1,4 +1,4 @@ -use hir::{HasSource, InFile, Semantics}; +use hir::{HasSource, InFile, InRealFile, Semantics}; use ide_db::{ base_db::{FileId, FilePosition, FileRange}, defs::Definition, @@ -149,8 +149,8 @@ pub(crate) fn annotations( node: InFile, source_file_id: FileId, ) -> Option<(TextRange, Option)> { - if let Some(InFile { file_id, value }) = node.original_ast_node(db) { - if file_id == source_file_id.into() { + if let Some(InRealFile { file_id, value }) = node.original_ast_node(db) { + if file_id == source_file_id { return Some(( value.syntax().text_range(), value.name().map(|name| name.syntax().text_range()), diff --git a/crates/ide/src/status.rs b/crates/ide/src/status.rs index c9ee460a1c26..e7f97ebe6f7b 100644 --- a/crates/ide/src/status.rs +++ b/crates/ide/src/status.rs @@ -2,7 +2,7 @@ use std::{fmt, marker::PhantomData}; use hir::{ db::{AstIdMapQuery, AttrsQuery, BlockDefMapQuery, ParseMacroExpansionQuery}, - Attr, Attrs, ExpandResult, MacroFile, Module, + Attr, Attrs, ExpandResult, MacroFileId, Module, }; use ide_db::{ base_db::{ @@ -199,8 +199,12 @@ impl StatCollect> for SyntaxTreeStats { } } -impl StatCollect, M)>> for SyntaxTreeStats { - fn collect_entry(&mut self, _: MacroFile, value: Option, M)>>) { +impl StatCollect, M)>> for SyntaxTreeStats { + fn collect_entry( + &mut self, + _: MacroFileId, + value: Option, M)>>, + ) { self.total += 1; self.retained += value.is_some() as usize; } diff --git a/crates/proc-macro-api/src/msg/flat.rs b/crates/proc-macro-api/src/msg/flat.rs index 22d9f3952cc2..f29aac529568 100644 --- a/crates/proc-macro-api/src/msg/flat.rs +++ b/crates/proc-macro-api/src/msg/flat.rs @@ -88,7 +88,7 @@ pub struct FlatTree { } #[derive(Serialize, Deserialize, Debug)] -pub struct SpanMap { +struct SpanMap { #[serde(skip_serializing)] serialize: bool, span_size: u32, @@ -122,7 +122,7 @@ impl SpanMap { } impl SpanMap { - pub fn do_serialize(&self) -> bool { + fn do_serialize(&self) -> bool { self.serialize } } From 6208960c4893ef43ea432a1eb28c5a28f754b049 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 25 Nov 2023 15:10:31 +0100 Subject: [PATCH 10/24] Deduplicate dummy test span maps --- crates/base-db/src/input.rs | 7 +- crates/base-db/src/lib.rs | 13 ++- crates/base-db/src/span.rs | 6 +- crates/cfg/src/tests.rs | 30 ++----- crates/mbe/src/benchmark.rs | 80 +++++++++---------- crates/mbe/src/lib.rs | 2 + crates/mbe/src/syntax_bridge.rs | 28 +++++++ crates/mbe/src/syntax_bridge/tests.rs | 32 +------- crates/proc-macro-srv-cli/src/main.rs | 2 + crates/proc-macro-srv/src/lib.rs | 1 + crates/rust-analyzer/src/cargo_target_spec.rs | 13 +-- crates/tt/src/lib.rs | 24 +----- 12 files changed, 99 insertions(+), 139 deletions(-) diff --git a/crates/base-db/src/input.rs b/crates/base-db/src/input.rs index 2fa5c25c91e5..0b04a91f626b 100644 --- a/crates/base-db/src/input.rs +++ b/crates/base-db/src/input.rs @@ -243,6 +243,9 @@ impl CrateDisplayName { } } +// FIXME: These should not be defined in here? Why does base db know about proc-macros +// ProcMacroKind is used in [`fixture`], but that module probably shouldn't be in this crate either. + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] pub struct ProcMacroId(pub u32); @@ -324,7 +327,9 @@ pub struct CrateData { pub dependencies: Vec, pub origin: CrateOrigin, pub is_proc_macro: bool, - // FIXME: These things should not be per crate! These are more per workspace crate graph level things + // FIXME: These things should not be per crate! These are more per workspace crate graph level + // things. This info does need to be somewhat present though as to prevent deduplication from + // happening across different workspaces with different layouts. pub target_layout: TargetLayoutLoadResult, pub channel: Option, } diff --git a/crates/base-db/src/lib.rs b/crates/base-db/src/lib.rs index 6dc1629c3be9..38a2641230b7 100644 --- a/crates/base-db/src/lib.rs +++ b/crates/base-db/src/lib.rs @@ -4,6 +4,7 @@ mod input; mod change; +// FIXME: Is this purely a test util mod? Consider #[cfg(test)] gating it. pub mod fixture; pub mod span; @@ -13,14 +14,13 @@ use rustc_hash::FxHashSet; use syntax::{ast, Parse, SourceFile, TextRange, TextSize}; use triomphe::Arc; -pub use crate::input::DependencyKind; pub use crate::{ change::Change, input::{ CrateData, CrateDisplayName, CrateGraph, CrateId, CrateName, CrateOrigin, Dependency, - Edition, Env, LangCrateOrigin, ProcMacro, ProcMacroExpander, ProcMacroExpansionError, - ProcMacroId, ProcMacroKind, ProcMacroLoadResult, ProcMacroPaths, ProcMacros, - ReleaseChannel, SourceRoot, SourceRootId, TargetLayoutLoadResult, + DependencyKind, Edition, Env, LangCrateOrigin, ProcMacro, ProcMacroExpander, + ProcMacroExpansionError, ProcMacroId, ProcMacroKind, ProcMacroLoadResult, ProcMacroPaths, + ProcMacros, ReleaseChannel, SourceRoot, SourceRootId, TargetLayoutLoadResult, }, }; pub use salsa::{self, Cancelled}; @@ -69,8 +69,7 @@ pub trait FileLoader { /// model. Everything else in rust-analyzer is derived from these queries. #[salsa::query_group(SourceDatabaseStorage)] pub trait SourceDatabase: FileLoader + std::fmt::Debug { - // Parses the file into the syntax tree. - #[salsa::invoke(parse_query)] + /// Parses the file into the syntax tree. fn parse(&self, file_id: FileId) -> Parse; /// The crate graph. @@ -82,7 +81,7 @@ pub trait SourceDatabase: FileLoader + std::fmt::Debug { fn proc_macros(&self) -> Arc; } -fn parse_query(db: &dyn SourceDatabase, file_id: FileId) -> Parse { +fn parse(db: &dyn SourceDatabase, file_id: FileId) -> Parse { let _p = profile::span("parse_query").detail(|| format!("{file_id:?}")); let text = db.file_text(file_id); SourceFile::parse(&text) diff --git a/crates/base-db/src/span.rs b/crates/base-db/src/span.rs index e183e9b19965..370c732813ac 100644 --- a/crates/base-db/src/span.rs +++ b/crates/base-db/src/span.rs @@ -1,3 +1,5 @@ +/// File and span related types. +// FIXME: This should probably be moved into its own crate. use std::fmt; use salsa::InternId; @@ -29,10 +31,10 @@ impl SyntaxContext for SyntaxContextId { } // inherent trait impls please tyvm impl SyntaxContextId { - // FIXME: This is very much UB, salsa exposes no way to create an InternId in a const context + // TODO: This is very much UB, salsa exposes no way to create an InternId in a const context // currently (which kind of makes sense but we need it here!) pub const ROOT: Self = SyntaxContextId(unsafe { core::mem::transmute(1) }); - // FIXME: This is very much UB, salsa exposes no way to create an InternId in a const context + // TODO: This is very much UB, salsa exposes no way to create an InternId in a const context // currently (which kind of makes sense but we need it here!) pub const SELF_REF: Self = SyntaxContextId(unsafe { core::mem::transmute(!0u32) }); diff --git a/crates/cfg/src/tests.rs b/crates/cfg/src/tests.rs index 61cdbded0b99..c7ac1af934a0 100644 --- a/crates/cfg/src/tests.rs +++ b/crates/cfg/src/tests.rs @@ -1,34 +1,14 @@ use arbitrary::{Arbitrary, Unstructured}; use expect_test::{expect, Expect}; -use mbe::{syntax_node_to_token_tree, SpanMapper}; +use mbe::{syntax_node_to_token_tree, DummyTestSpanMap}; use syntax::{ast, AstNode}; -use tt::{SpanAnchor, SyntaxContext}; use crate::{CfgAtom, CfgExpr, CfgOptions, DnfExpr}; -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -struct DummyFile; -impl SpanAnchor for DummyFile { - const DUMMY: Self = DummyFile; -} -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -struct DummyCtx; -impl SyntaxContext for DummyCtx { - const DUMMY: Self = DummyCtx; -} - -struct NoOpMap; - -impl SpanMapper> for NoOpMap { - fn span_for(&self, range: syntax::TextRange) -> tt::SpanData { - tt::SpanData { range, anchor: DummyFile, ctx: DummyCtx } - } -} - fn assert_parse_result(input: &str, expected: CfgExpr) { let source_file = ast::SourceFile::parse(input).ok().unwrap(); let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - let tt = syntax_node_to_token_tree(tt.syntax(), NoOpMap); + let tt = syntax_node_to_token_tree(tt.syntax(), DummyTestSpanMap); let cfg = CfgExpr::parse(&tt); assert_eq!(cfg, expected); } @@ -36,7 +16,7 @@ fn assert_parse_result(input: &str, expected: CfgExpr) { fn check_dnf(input: &str, expect: Expect) { let source_file = ast::SourceFile::parse(input).ok().unwrap(); let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - let tt = syntax_node_to_token_tree(tt.syntax(), NoOpMap); + let tt = syntax_node_to_token_tree(tt.syntax(), DummyTestSpanMap); let cfg = CfgExpr::parse(&tt); let actual = format!("#![cfg({})]", DnfExpr::new(cfg)); expect.assert_eq(&actual); @@ -45,7 +25,7 @@ fn check_dnf(input: &str, expect: Expect) { fn check_why_inactive(input: &str, opts: &CfgOptions, expect: Expect) { let source_file = ast::SourceFile::parse(input).ok().unwrap(); let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - let tt = syntax_node_to_token_tree(tt.syntax(), NoOpMap); + let tt = syntax_node_to_token_tree(tt.syntax(), DummyTestSpanMap); let cfg = CfgExpr::parse(&tt); let dnf = DnfExpr::new(cfg); let why_inactive = dnf.why_inactive(opts).unwrap().to_string(); @@ -56,7 +36,7 @@ fn check_why_inactive(input: &str, opts: &CfgOptions, expect: Expect) { fn check_enable_hints(input: &str, opts: &CfgOptions, expected_hints: &[&str]) { let source_file = ast::SourceFile::parse(input).ok().unwrap(); let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - let tt = syntax_node_to_token_tree(tt.syntax(), NoOpMap); + let tt = syntax_node_to_token_tree(tt.syntax(), DummyTestSpanMap); let cfg = CfgExpr::parse(&tt); let dnf = DnfExpr::new(cfg); let hints = dnf.compute_enable_hints(opts).map(|diff| diff.to_string()).collect::>(); diff --git a/crates/mbe/src/benchmark.rs b/crates/mbe/src/benchmark.rs index e7fbb918897d..271efe1a92e9 100644 --- a/crates/mbe/src/benchmark.rs +++ b/crates/mbe/src/benchmark.rs @@ -6,35 +6,13 @@ use syntax::{ AstNode, SmolStr, }; use test_utils::{bench, bench_fixture, skip_slow_tests}; -use tt::{Span, SpanAnchor, SyntaxContext}; +use tt::Span; use crate::{ parser::{MetaVarKind, Op, RepeatKind, Separator}, - syntax_node_to_token_tree, DeclarativeMacro, SpanMapper, + syntax_node_to_token_tree, DeclarativeMacro, DummyTestSpanData, DummyTestSpanMap, }; -type SpanData = tt::SpanData; - -#[derive(PartialEq, Eq, Clone, Copy, Debug, Hash)] -struct DummyFile; -impl SpanAnchor for DummyFile { - const DUMMY: Self = DummyFile; -} - -#[derive(PartialEq, Eq, Clone, Copy, Debug)] -struct DummyCtx; -impl SyntaxContext for DummyCtx { - const DUMMY: Self = DummyCtx; -} - -struct NoOpMap; - -impl SpanMapper for NoOpMap { - fn span_for(&self, range: syntax::TextRange) -> SpanData { - SpanData { range, anchor: DummyFile, ctx: DummyCtx } - } -} - #[test] fn benchmark_parse_macro_rules() { if skip_slow_tests() { @@ -70,14 +48,14 @@ fn benchmark_expand_macro_rules() { assert_eq!(hash, 69413); } -fn macro_rules_fixtures() -> FxHashMap> { +fn macro_rules_fixtures() -> FxHashMap> { macro_rules_fixtures_tt() .into_iter() .map(|(id, tt)| (id, DeclarativeMacro::parse_macro_rules(&tt, true))) .collect() } -fn macro_rules_fixtures_tt() -> FxHashMap> { +fn macro_rules_fixtures_tt() -> FxHashMap> { let fixture = bench_fixture::numerous_macro_rules(); let source_file = ast::SourceFile::parse(&fixture).ok().unwrap(); @@ -87,7 +65,8 @@ fn macro_rules_fixtures_tt() -> FxHashMap> { .filter_map(ast::MacroRules::cast) .map(|rule| { let id = rule.name().unwrap().to_string(); - let def_tt = syntax_node_to_token_tree(rule.token_tree().unwrap().syntax(), NoOpMap); + let def_tt = + syntax_node_to_token_tree(rule.token_tree().unwrap().syntax(), DummyTestSpanMap); (id, def_tt) }) .collect() @@ -95,8 +74,8 @@ fn macro_rules_fixtures_tt() -> FxHashMap> { /// Generate random invocation fixtures from rules fn invocation_fixtures( - rules: &FxHashMap>, -) -> Vec<(String, tt::Subtree)> { + rules: &FxHashMap>, +) -> Vec<(String, tt::Subtree)> { let mut seed = 123456789; let mut res = Vec::new(); @@ -118,8 +97,8 @@ fn invocation_fixtures( loop { let mut subtree = tt::Subtree { delimiter: tt::Delimiter { - open: SpanData::DUMMY, - close: SpanData::DUMMY, + open: DummyTestSpanData::DUMMY, + close: DummyTestSpanData::DUMMY, kind: tt::DelimiterKind::Invisible, }, token_trees: vec![], @@ -141,7 +120,11 @@ fn invocation_fixtures( } return res; - fn collect_from_op(op: &Op, parent: &mut tt::Subtree, seed: &mut usize) { + fn collect_from_op( + op: &Op, + parent: &mut tt::Subtree, + seed: &mut usize, + ) { return match op { Op::Var { kind, .. } => match kind.as_ref() { Some(MetaVarKind::Ident) => parent.token_trees.push(make_ident("foo")), @@ -227,22 +210,35 @@ fn invocation_fixtures( *seed = usize::wrapping_add(usize::wrapping_mul(*seed, a), c); *seed } - fn make_ident(ident: &str) -> tt::TokenTree { - tt::Leaf::Ident(tt::Ident { span: SpanData::DUMMY, text: SmolStr::new(ident) }).into() - } - fn make_punct(char: char) -> tt::TokenTree { - tt::Leaf::Punct(tt::Punct { span: SpanData::DUMMY, char, spacing: tt::Spacing::Alone }) + fn make_ident(ident: &str) -> tt::TokenTree { + tt::Leaf::Ident(tt::Ident { span: DummyTestSpanData::DUMMY, text: SmolStr::new(ident) }) .into() } - fn make_literal(lit: &str) -> tt::TokenTree { - tt::Leaf::Literal(tt::Literal { span: SpanData::DUMMY, text: SmolStr::new(lit) }).into() + fn make_punct(char: char) -> tt::TokenTree { + tt::Leaf::Punct(tt::Punct { + span: DummyTestSpanData::DUMMY, + char, + spacing: tt::Spacing::Alone, + }) + .into() + } + fn make_literal(lit: &str) -> tt::TokenTree { + tt::Leaf::Literal(tt::Literal { + span: DummyTestSpanData::DUMMY, + text: SmolStr::new(lit), + }) + .into() } fn make_subtree( kind: tt::DelimiterKind, - token_trees: Option>>, - ) -> tt::TokenTree { + token_trees: Option>>, + ) -> tt::TokenTree { tt::Subtree { - delimiter: tt::Delimiter { open: SpanData::DUMMY, close: SpanData::DUMMY, kind }, + delimiter: tt::Delimiter { + open: DummyTestSpanData::DUMMY, + close: DummyTestSpanData::DUMMY, + kind, + }, token_trees: token_trees.unwrap_or_default(), } .into() diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index 0b8461200e63..1b13b39f0177 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -40,6 +40,8 @@ pub use crate::{ token_map::TokenMap, }; +pub use crate::syntax_bridge::dummy_test_span_utils::*; + #[derive(Debug, PartialEq, Eq, Clone)] pub enum ParseError { UnexpectedToken(Box), diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 36c63b365d2c..688ccb23252c 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -33,6 +33,34 @@ impl> SpanMapper for &SM { } } +pub(crate) mod dummy_test_span_utils { + use super::*; + + pub type DummyTestSpanData = tt::SpanData; + + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] + pub struct DummyTestSpanAnchor; + impl tt::SpanAnchor for DummyTestSpanAnchor { + const DUMMY: Self = DummyTestSpanAnchor; + } + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + pub struct DummyTestSyntaxContext; + impl SyntaxContext for DummyTestSyntaxContext { + const DUMMY: Self = DummyTestSyntaxContext; + } + + pub struct DummyTestSpanMap; + + impl SpanMapper> for DummyTestSpanMap { + fn span_for( + &self, + range: syntax::TextRange, + ) -> tt::SpanData { + tt::SpanData { range, anchor: DummyTestSpanAnchor, ctx: DummyTestSyntaxContext } + } + } +} + /// Convert the syntax node to a `TokenTree` (what macro /// will consume). /// TODO: Flesh out the doc comment more thoroughly diff --git a/crates/mbe/src/syntax_bridge/tests.rs b/crates/mbe/src/syntax_bridge/tests.rs index 2e21977f681b..bd8187a148a5 100644 --- a/crates/mbe/src/syntax_bridge/tests.rs +++ b/crates/mbe/src/syntax_bridge/tests.rs @@ -4,38 +4,14 @@ use syntax::{ast, AstNode}; use test_utils::extract_annotations; use tt::{ buffer::{TokenBuffer, TokenTreeRef}, - Leaf, Punct, Spacing, SpanAnchor, SyntaxContext, + Leaf, Punct, Spacing, }; -use crate::SpanMapper; - -use super::syntax_node_to_token_tree; +use crate::{syntax_node_to_token_tree, DummyTestSpanData, DummyTestSpanMap}; fn check_punct_spacing(fixture: &str) { - type SpanData = tt::SpanData; - - #[derive(PartialEq, Eq, Clone, Copy, Debug, Hash)] - struct DummyFile; - impl SpanAnchor for DummyFile { - const DUMMY: Self = DummyFile; - } - - #[derive(PartialEq, Eq, Clone, Copy, Debug)] - struct DummyCtx; - impl SyntaxContext for DummyCtx { - const DUMMY: Self = DummyCtx; - } - - struct NoOpMap; - - impl SpanMapper for NoOpMap { - fn span_for(&self, range: syntax::TextRange) -> SpanData { - SpanData { range, anchor: DummyFile, ctx: DummyCtx } - } - } - let source_file = ast::SourceFile::parse(fixture).ok().unwrap(); - let subtree = syntax_node_to_token_tree(source_file.syntax(), NoOpMap); + let subtree = syntax_node_to_token_tree(source_file.syntax(), DummyTestSpanMap); let mut annotations: HashMap<_, _> = extract_annotations(fixture) .into_iter() .map(|(range, annotation)| { @@ -53,7 +29,7 @@ fn check_punct_spacing(fixture: &str) { while !cursor.eof() { while let Some(token_tree) = cursor.token_tree() { if let TokenTreeRef::Leaf( - Leaf::Punct(Punct { spacing, span: SpanData { range, .. }, .. }), + Leaf::Punct(Punct { spacing, span: DummyTestSpanData { range, .. }, .. }), _, ) = token_tree { diff --git a/crates/proc-macro-srv-cli/src/main.rs b/crates/proc-macro-srv-cli/src/main.rs index bece1951872c..ea65c33604f8 100644 --- a/crates/proc-macro-srv-cli/src/main.rs +++ b/crates/proc-macro-srv-cli/src/main.rs @@ -18,11 +18,13 @@ fn main() -> std::io::Result<()> { run() } +#[cfg(not(FALSE))] #[cfg(not(feature = "sysroot-abi"))] fn run() -> io::Result<()> { panic!("proc-macro-srv-cli requires the `sysroot-abi` feature to be enabled"); } +#[cfg(FALSE)] #[cfg(feature = "sysroot-abi")] fn run() -> io::Result<()> { use proc_macro_api::msg::{self, Message}; diff --git a/crates/proc-macro-srv/src/lib.rs b/crates/proc-macro-srv/src/lib.rs index 00042480e302..bd0d1b79fa1f 100644 --- a/crates/proc-macro-srv/src/lib.rs +++ b/crates/proc-macro-srv/src/lib.rs @@ -10,6 +10,7 @@ //! * By **copying** the whole rustc `lib_proc_macro` code, we are able to build this with `stable` //! rustc rather than `unstable`. (Although in general ABI compatibility is still an issue)… +#![cfg(FALSE)] // TODO #![cfg(any(feature = "sysroot-abi", rust_analyzer))] #![feature(proc_macro_internals, proc_macro_diagnostic, proc_macro_span)] #![warn(rust_2018_idioms, unused_lifetimes, semicolon_in_expressions_from_macros)] diff --git a/crates/rust-analyzer/src/cargo_target_spec.rs b/crates/rust-analyzer/src/cargo_target_spec.rs index d679bfcb014c..728bade0d0a5 100644 --- a/crates/rust-analyzer/src/cargo_target_spec.rs +++ b/crates/rust-analyzer/src/cargo_target_spec.rs @@ -209,26 +209,17 @@ mod tests { use super::*; use cfg::CfgExpr; - use hir_def::tt::{self, Span}; - use mbe::{syntax_node_to_token_tree, SpanMapper}; + use mbe::{syntax_node_to_token_tree, DummyTestSpanMap}; use syntax::{ ast::{self, AstNode}, SmolStr, }; - struct NoOpMap; - - impl SpanMapper for NoOpMap { - fn span_for(&self, _: syntax::TextRange) -> tt::SpanData { - tt::SpanData::DUMMY - } - } - fn check(cfg: &str, expected_features: &[&str]) { let cfg_expr = { let source_file = ast::SourceFile::parse(cfg).ok().unwrap(); let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - let tt = syntax_node_to_token_tree(tt.syntax(), &NoOpMap); + let tt = syntax_node_to_token_tree(tt.syntax(), &DummyTestSpanMap); CfgExpr::parse(&tt) }; diff --git a/crates/tt/src/lib.rs b/crates/tt/src/lib.rs index 1374ae1a6fc6..7977d97797ad 100644 --- a/crates/tt/src/lib.rs +++ b/crates/tt/src/lib.rs @@ -7,31 +7,9 @@ use std::fmt; use stdx::impl_from; -use text_size::{TextRange, TextSize}; pub use smol_str::SmolStr; - -/// Represents identity of the token. -/// -/// For hygiene purposes, we need to track which expanded tokens originated from -/// which source tokens. We do it by assigning an distinct identity to each -/// source token and making sure that identities are preserved during macro -/// expansion. -#[derive(Clone, Copy, PartialEq, Eq, Hash)] -pub struct TokenId(pub u32); - -impl fmt::Debug for TokenId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } -} - -impl TokenId { - pub const UNSPECIFIED: TokenId = TokenId(!0); - pub const fn unspecified() -> TokenId { - Self::UNSPECIFIED - } -} +pub use text_size::{TextRange, TextSize}; #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] pub struct SpanData { From ab8f12e169b72a39a6d2e68b569c8e4c29259ffc Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 25 Nov 2023 15:37:40 +0100 Subject: [PATCH 11/24] Rename hygiene vars and fields to span_map --- crates/hir-def/src/attr.rs | 53 +++++++++++++-------------- crates/hir-def/src/expander.rs | 14 +++---- crates/hir-def/src/item_tree.rs | 8 ++-- crates/hir-def/src/item_tree/lower.rs | 13 ++++--- crates/hir-def/src/lower.rs | 18 +++++---- crates/hir-def/src/path/lower.rs | 6 +-- crates/hir-def/src/visibility.rs | 14 +++---- crates/hir-expand/src/attrs.rs | 18 ++++----- crates/hir-expand/src/eager.rs | 4 +- crates/hir-expand/src/mod_path.rs | 12 +++--- crates/hir-ty/src/display.rs | 2 +- crates/hir/src/semantics.rs | 4 +- crates/hir/src/source_analyzer.rs | 2 +- 13 files changed, 85 insertions(+), 83 deletions(-) diff --git a/crates/hir-def/src/attr.rs b/crates/hir-def/src/attr.rs index 45dd981dceec..942b28fc1450 100644 --- a/crates/hir-def/src/attr.rs +++ b/crates/hir-def/src/attr.rs @@ -415,35 +415,32 @@ impl AttrsWithOwner { AttrDefId::StaticId(it) => attrs_from_item_tree_assoc(db, it), AttrDefId::FunctionId(it) => attrs_from_item_tree_assoc(db, it), AttrDefId::TypeAliasId(it) => attrs_from_item_tree_assoc(db, it), - AttrDefId::GenericParamId(it) => { - // FIXME: we could probably just make these relative to the params? - match it { - GenericParamId::ConstParamId(it) => { - let src = it.parent().child_source(db); - RawAttrs::from_attrs_owner( - db.upcast(), - src.with_value(&src.value[it.local_id()]), - db.span_map(src.file_id).as_ref(), - ) - } - GenericParamId::TypeParamId(it) => { - let src = it.parent().child_source(db); - RawAttrs::from_attrs_owner( - db.upcast(), - src.with_value(&src.value[it.local_id()]), - db.span_map(src.file_id).as_ref(), - ) - } - GenericParamId::LifetimeParamId(it) => { - let src = it.parent.child_source(db); - RawAttrs::from_attrs_owner( - db.upcast(), - src.with_value(&src.value[it.local_id]), - db.span_map(src.file_id).as_ref(), - ) - } + AttrDefId::GenericParamId(it) => match it { + GenericParamId::ConstParamId(it) => { + let src = it.parent().child_source(db); + RawAttrs::from_attrs_owner( + db.upcast(), + src.with_value(&src.value[it.local_id()]), + db.span_map(src.file_id).as_ref(), + ) } - } + GenericParamId::TypeParamId(it) => { + let src = it.parent().child_source(db); + RawAttrs::from_attrs_owner( + db.upcast(), + src.with_value(&src.value[it.local_id()]), + db.span_map(src.file_id).as_ref(), + ) + } + GenericParamId::LifetimeParamId(it) => { + let src = it.parent.child_source(db); + RawAttrs::from_attrs_owner( + db.upcast(), + src.with_value(&src.value[it.local_id]), + db.span_map(src.file_id).as_ref(), + ) + } + }, AttrDefId::ExternBlockId(it) => attrs_from_item_tree_loc(db, it), AttrDefId::ExternCrateId(it) => attrs_from_item_tree_loc(db, it), AttrDefId::UseId(it) => attrs_from_item_tree_loc(db, it), diff --git a/crates/hir-def/src/expander.rs b/crates/hir-def/src/expander.rs index d8ee61a3dca3..964f07072316 100644 --- a/crates/hir-def/src/expander.rs +++ b/crates/hir-def/src/expander.rs @@ -18,7 +18,7 @@ use crate::{ #[derive(Debug)] pub struct Expander { cfg_options: CfgOptions, - hygiene: SpanMap, + span_map: SpanMap, krate: CrateId, pub(crate) current_file_id: HirFileId, pub(crate) module: ModuleId, @@ -41,7 +41,7 @@ impl Expander { recursion_depth: 0, recursion_limit, cfg_options: db.crate_graph()[module.krate].cfg_options.clone(), - hygiene: db.span_map(current_file_id), + span_map: db.span_map(current_file_id), krate: module.krate, } } @@ -95,7 +95,7 @@ impl Expander { } pub fn exit(&mut self, db: &dyn DefDatabase, mut mark: Mark) { - self.hygiene = db.span_map(mark.file_id); + self.span_map = db.span_map(mark.file_id); self.current_file_id = mark.file_id; if self.recursion_depth == u32::MAX { // Recursion limit has been reached somewhere in the macro expansion tree. Reset the @@ -110,7 +110,7 @@ impl Expander { } pub fn ctx<'a>(&self, db: &'a dyn DefDatabase) -> LowerCtx<'a> { - LowerCtx::new(db, self.hygiene.clone(), self.current_file_id) + LowerCtx::new(db, self.span_map.clone(), self.current_file_id) } pub(crate) fn to_source(&self, value: T) -> InFile { @@ -118,7 +118,7 @@ impl Expander { } pub(crate) fn parse_attrs(&self, db: &dyn DefDatabase, owner: &dyn ast::HasAttrs) -> Attrs { - Attrs::filter(db, self.krate, RawAttrs::new(db.upcast(), owner, self.hygiene.as_ref())) + Attrs::filter(db, self.krate, RawAttrs::new(db.upcast(), owner, self.span_map.as_ref())) } pub(crate) fn cfg_options(&self) -> &CfgOptions { @@ -130,7 +130,7 @@ impl Expander { } pub(crate) fn parse_path(&mut self, db: &dyn DefDatabase, path: ast::Path) -> Option { - let ctx = LowerCtx::new(db, self.hygiene.clone(), self.current_file_id); + let ctx = LowerCtx::new(db, self.span_map.clone(), self.current_file_id); Path::from_src(path, &ctx) } @@ -174,7 +174,7 @@ impl Expander { let parse = value.cast::()?; self.recursion_depth += 1; - self.hygiene = db.span_map(file_id); + self.span_map = db.span_map(file_id); let old_file_id = std::mem::replace(&mut self.current_file_id, file_id); let mark = Mark { file_id: old_file_id, diff --git a/crates/hir-def/src/item_tree.rs b/crates/hir-def/src/item_tree.rs index 9c61deb423cb..97b62941842d 100644 --- a/crates/hir-def/src/item_tree.rs +++ b/crates/hir-def/src/item_tree.rs @@ -776,8 +776,8 @@ impl Use { // Note: The AST unwraps are fine, since if they fail we should have never obtained `index`. let ast = InFile::new(file_id, self.ast_id).to_node(db.upcast()); let ast_use_tree = ast.use_tree().expect("missing `use_tree`"); - let hygiene = db.span_map(file_id); - let (_, source_map) = lower::lower_use_tree(db, hygiene.as_ref(), ast_use_tree) + let span_map = db.span_map(file_id); + let (_, source_map) = lower::lower_use_tree(db, span_map.as_ref(), ast_use_tree) .expect("failed to lower use tree"); source_map[index].clone() } @@ -791,8 +791,8 @@ impl Use { // Note: The AST unwraps are fine, since if they fail we should have never obtained `index`. let ast = InFile::new(file_id, self.ast_id).to_node(db.upcast()); let ast_use_tree = ast.use_tree().expect("missing `use_tree`"); - let hygiene = db.span_map(file_id); - lower::lower_use_tree(db, hygiene.as_ref(), ast_use_tree) + let span_map = db.span_map(file_id); + lower::lower_use_tree(db, span_map.as_ref(), ast_use_tree) .expect("failed to lower use tree") .1 } diff --git a/crates/hir-def/src/item_tree/lower.rs b/crates/hir-def/src/item_tree/lower.rs index 933a59be1534..92a2a9462327 100644 --- a/crates/hir-def/src/item_tree/lower.rs +++ b/crates/hir-def/src/item_tree/lower.rs @@ -657,7 +657,8 @@ impl<'a> Ctx<'a> { } fn lower_visibility(&mut self, item: &dyn ast::HasVisibility) -> RawVisibilityId { - let vis = RawVisibility::from_ast_with_hygiene(self.db, item.visibility(), self.span_map()); + let vis = + RawVisibility::from_ast_with_span_map(self.db, item.visibility(), self.span_map()); self.data().vis.alloc(vis) } @@ -735,7 +736,7 @@ fn lower_abi(abi: ast::Abi) -> Interned { struct UseTreeLowering<'a> { db: &'a dyn DefDatabase, - hygiene: SpanMapRef<'a>, + span_map: SpanMapRef<'a>, mapping: Arena, } @@ -748,7 +749,7 @@ impl UseTreeLowering<'_> { // E.g. `use something::{inner}` (prefix is `None`, path is `something`) // or `use something::{path::{inner::{innerer}}}` (prefix is `something::path`, path is `inner`) Some(path) => { - match ModPath::from_src(self.db.upcast(), path, self.hygiene) { + match ModPath::from_src(self.db.upcast(), path, self.span_map) { Some(it) => Some(it), None => return None, // FIXME: report errors somewhere } @@ -767,7 +768,7 @@ impl UseTreeLowering<'_> { } else { let is_glob = tree.star_token().is_some(); let path = match tree.path() { - Some(path) => Some(ModPath::from_src(self.db.upcast(), path, self.hygiene)?), + Some(path) => Some(ModPath::from_src(self.db.upcast(), path, self.span_map)?), None => None, }; let alias = tree.rename().map(|a| { @@ -803,10 +804,10 @@ impl UseTreeLowering<'_> { pub(crate) fn lower_use_tree( db: &dyn DefDatabase, - hygiene: SpanMapRef<'_>, + span_map: SpanMapRef<'_>, tree: ast::UseTree, ) -> Option<(UseTree, Arena)> { - let mut lowering = UseTreeLowering { db, hygiene, mapping: Arena::new() }; + let mut lowering = UseTreeLowering { db, span_map, mapping: Arena::new() }; let tree = lowering.lower_use_tree(tree)?; Some((tree, lowering.mapping)) } diff --git a/crates/hir-def/src/lower.rs b/crates/hir-def/src/lower.rs index a5c22898245d..6e7fb0845407 100644 --- a/crates/hir-def/src/lower.rs +++ b/crates/hir-def/src/lower.rs @@ -13,26 +13,30 @@ use crate::{db::DefDatabase, path::Path}; pub struct LowerCtx<'a> { pub db: &'a dyn DefDatabase, - hygiene: SpanMap, + span_map: SpanMap, // FIXME: This optimization is probably pointless, ast id map should pretty much always exist anyways. ast_id_map: Option<(HirFileId, OnceCell>)>, } impl<'a> LowerCtx<'a> { - pub fn new(db: &'a dyn DefDatabase, hygiene: SpanMap, file_id: HirFileId) -> Self { - LowerCtx { db, hygiene, ast_id_map: Some((file_id, OnceCell::new())) } + pub fn new(db: &'a dyn DefDatabase, span_map: SpanMap, file_id: HirFileId) -> Self { + LowerCtx { db, span_map, ast_id_map: Some((file_id, OnceCell::new())) } } pub fn with_file_id(db: &'a dyn DefDatabase, file_id: HirFileId) -> Self { - LowerCtx { db, hygiene: db.span_map(file_id), ast_id_map: Some((file_id, OnceCell::new())) } + LowerCtx { + db, + span_map: db.span_map(file_id), + ast_id_map: Some((file_id, OnceCell::new())), + } } - pub fn with_hygiene(db: &'a dyn DefDatabase, hygiene: SpanMap) -> Self { - LowerCtx { db, hygiene, ast_id_map: None } + pub fn with_span_map(db: &'a dyn DefDatabase, span_map: SpanMap) -> Self { + LowerCtx { db, span_map, ast_id_map: None } } pub(crate) fn span_map(&self) -> SpanMapRef<'_> { - self.hygiene.as_ref() + self.span_map.as_ref() } pub(crate) fn lower_path(&self, ast: ast::Path) -> Option { diff --git a/crates/hir-def/src/path/lower.rs b/crates/hir-def/src/path/lower.rs index 9b499c5619ef..9ab4bdaa4ac8 100644 --- a/crates/hir-def/src/path/lower.rs +++ b/crates/hir-def/src/path/lower.rs @@ -26,7 +26,7 @@ pub(super) fn lower_path(mut path: ast::Path, ctx: &LowerCtx<'_>) -> Option) -> Option) -> Option>, ) -> RawVisibility { - Self::from_ast_with_hygiene(db, node.value, db.span_map(node.file_id).as_ref()) + Self::from_ast_with_span_map(db, node.value, db.span_map(node.file_id).as_ref()) } - pub(crate) fn from_ast_with_hygiene( + pub(crate) fn from_ast_with_span_map( db: &dyn DefDatabase, node: Option, - hygiene: SpanMapRef<'_>, + span_map: SpanMapRef<'_>, ) -> RawVisibility { - Self::from_ast_with_hygiene_and_default(db, node, RawVisibility::private(), hygiene) + Self::from_ast_with_span_map_and_default(db, node, RawVisibility::private(), span_map) } - pub(crate) fn from_ast_with_hygiene_and_default( + pub(crate) fn from_ast_with_span_map_and_default( db: &dyn DefDatabase, node: Option, default: RawVisibility, - hygiene: SpanMapRef<'_>, + span_map: SpanMapRef<'_>, ) -> RawVisibility { let node = match node { None => return default, @@ -57,7 +57,7 @@ impl RawVisibility { }; match node.kind() { ast::VisibilityKind::In(path) => { - let path = ModPath::from_src(db.upcast(), path, hygiene); + let path = ModPath::from_src(db.upcast(), path, span_map); let path = match path { None => return RawVisibility::private(), Some(path) => path, diff --git a/crates/hir-expand/src/attrs.rs b/crates/hir-expand/src/attrs.rs index c4937ae08b1e..1fdb5dc37b88 100644 --- a/crates/hir-expand/src/attrs.rs +++ b/crates/hir-expand/src/attrs.rs @@ -42,18 +42,18 @@ impl RawAttrs { pub fn new( db: &dyn ExpandDatabase, owner: &dyn ast::HasAttrs, - hygiene: SpanMapRef<'_>, + span_map: SpanMapRef<'_>, ) -> Self { let entries = collect_attrs(owner) .filter_map(|(id, attr)| match attr { Either::Left(attr) => { - attr.meta().and_then(|meta| Attr::from_src(db, meta, hygiene, id)) + attr.meta().and_then(|meta| Attr::from_src(db, meta, span_map, id)) } Either::Right(comment) => comment.doc_comment().map(|doc| Attr { id, input: Some(Interned::new(AttrInput::Literal(SmolStr::new(doc)))), path: Interned::new(ModPath::from(crate::name!(doc))), - ctxt: hygiene.span_for_range(comment.syntax().text_range()).ctx, + ctxt: span_map.span_for_range(comment.syntax().text_range()).ctx, }), }) .collect::>(); @@ -66,9 +66,9 @@ impl RawAttrs { pub fn from_attrs_owner( db: &dyn ExpandDatabase, owner: InFile<&dyn ast::HasAttrs>, - hygiene: SpanMapRef<'_>, + span_map: SpanMapRef<'_>, ) -> Self { - Self::new(db, owner.value, hygiene) + Self::new(db, owner.value, span_map) } pub fn merge(&self, other: Self) -> Self { @@ -216,10 +216,10 @@ impl Attr { fn from_src( db: &dyn ExpandDatabase, ast: ast::Meta, - hygiene: SpanMapRef<'_>, + span_map: SpanMapRef<'_>, id: AttrId, ) -> Option { - let path = Interned::new(ModPath::from_src(db, ast.path()?, hygiene)?); + let path = Interned::new(ModPath::from_src(db, ast.path()?, span_map)?); let input = if let Some(ast::Expr::Literal(lit)) = ast.expr() { let value = match lit.kind() { ast::LiteralKind::String(string) => string.value()?.into(), @@ -227,12 +227,12 @@ impl Attr { }; Some(Interned::new(AttrInput::Literal(value))) } else if let Some(tt) = ast.token_tree() { - let tree = syntax_node_to_token_tree(tt.syntax(), hygiene); + let tree = syntax_node_to_token_tree(tt.syntax(), span_map); Some(Interned::new(AttrInput::TokenTree(Box::new(tree)))) } else { None }; - Some(Attr { id, path, input, ctxt: hygiene.span_for_range(ast.syntax().text_range()).ctx }) + Some(Attr { id, path, input, ctxt: span_map.span_for_range(ast.syntax().text_range()).ctx }) } fn from_tt(db: &dyn ExpandDatabase, tt: &tt::Subtree, id: AttrId) -> Option { diff --git a/crates/hir-expand/src/eager.rs b/crates/hir-expand/src/eager.rs index bcb5383c3f9b..4499c2e69d65 100644 --- a/crates/hir-expand/src/eager.rs +++ b/crates/hir-expand/src/eager.rs @@ -127,7 +127,7 @@ fn lazy_expand( fn eager_macro_recur( db: &dyn ExpandDatabase, - hygiene: &ExpansionSpanMap, + span_map: &ExpansionSpanMap, curr: InFile, krate: CrateId, call_site: SyntaxContextId, @@ -170,7 +170,7 @@ fn eager_macro_recur( }; let def = match call .path() - .and_then(|path| ModPath::from_src(db, path, SpanMapRef::ExpansionSpanMap(hygiene))) + .and_then(|path| ModPath::from_src(db, path, SpanMapRef::ExpansionSpanMap(span_map))) { Some(path) => match macro_resolver(path.clone()) { Some(def) => def, diff --git a/crates/hir-expand/src/mod_path.rs b/crates/hir-expand/src/mod_path.rs index a518f7eb6a9f..9534b5039f68 100644 --- a/crates/hir-expand/src/mod_path.rs +++ b/crates/hir-expand/src/mod_path.rs @@ -47,9 +47,9 @@ impl ModPath { pub fn from_src( db: &dyn ExpandDatabase, path: ast::Path, - hygiene: SpanMapRef<'_>, + span_map: SpanMapRef<'_>, ) -> Option { - convert_path(db, None, path, hygiene) + convert_path(db, None, path, span_map) } pub fn from_segments(kind: PathKind, segments: impl IntoIterator) -> ModPath { @@ -194,10 +194,10 @@ fn convert_path( db: &dyn ExpandDatabase, prefix: Option, path: ast::Path, - hygiene: SpanMapRef<'_>, + span_map: SpanMapRef<'_>, ) -> Option { let prefix = match path.qualifier() { - Some(qual) => Some(convert_path(db, prefix, qual, hygiene)?), + Some(qual) => Some(convert_path(db, prefix, qual, span_map)?), None => prefix, }; @@ -211,7 +211,7 @@ fn convert_path( ModPath::from_kind( resolve_crate_root( db, - hygiene.span_for_range(name_ref.syntax().text_range()).ctx, + span_map.span_for_range(name_ref.syntax().text_range()).ctx, ) .map(PathKind::DollarCrate) .unwrap_or(PathKind::Crate), @@ -265,7 +265,7 @@ fn convert_path( // We follow what it did anyway :) if mod_path.segments.len() == 1 && mod_path.kind == PathKind::Plain { if let Some(_macro_call) = path.syntax().parent().and_then(ast::MacroCall::cast) { - let syn_ctx = hygiene.span_for_range(segment.syntax().text_range()).ctx; + let syn_ctx = span_map.span_for_range(segment.syntax().text_range()).ctx; if let Some(macro_call_id) = db.lookup_intern_syntax_context(syn_ctx).outer_expn { if db.lookup_intern_macro_call(macro_call_id).def.local_inner { mod_path.kind = match resolve_crate_root(db, syn_ctx) { diff --git a/crates/hir-ty/src/display.rs b/crates/hir-ty/src/display.rs index 0712fc1c50c7..3d426caf680e 100644 --- a/crates/hir-ty/src/display.rs +++ b/crates/hir-ty/src/display.rs @@ -1732,7 +1732,7 @@ impl HirDisplay for TypeRef { f.write_joined(bounds, " + ")?; } TypeRef::Macro(macro_call) => { - let ctx = hir_def::lower::LowerCtx::with_hygiene( + let ctx = hir_def::lower::LowerCtx::with_span_map( f.db.upcast(), f.db.span_map(macro_call.file_id), ); diff --git a/crates/hir/src/semantics.rs b/crates/hir/src/semantics.rs index e124e14a5422..666ffd390ae7 100644 --- a/crates/hir/src/semantics.rs +++ b/crates/hir/src/semantics.rs @@ -866,8 +866,8 @@ impl<'db> SemanticsImpl<'db> { pub fn resolve_trait(&self, path: &ast::Path) -> Option { let analyze = self.analyze(path.syntax())?; - let hygiene = self.db.span_map(analyze.file_id); - let ctx = LowerCtx::with_hygiene(self.db.upcast(), hygiene); + let span_map = self.db.span_map(analyze.file_id); + let ctx = LowerCtx::with_span_map(self.db.upcast(), span_map); let hir_path = Path::from_src(path.clone(), &ctx)?; match analyze.resolver.resolve_path_in_type_ns_fully(self.db.upcast(), &hir_path)? { TypeNs::TraitId(id) => Some(Trait { id }), diff --git a/crates/hir/src/source_analyzer.rs b/crates/hir/src/source_analyzer.rs index aaad82e12876..2339187122ee 100644 --- a/crates/hir/src/source_analyzer.rs +++ b/crates/hir/src/source_analyzer.rs @@ -595,7 +595,7 @@ impl SourceAnalyzer { } // This must be a normal source file rather than macro file. - let ctx = LowerCtx::with_hygiene(db.upcast(), db.span_map(self.file_id)); + let ctx = LowerCtx::with_span_map(db.upcast(), db.span_map(self.file_id)); let hir_path = Path::from_src(path.clone(), &ctx)?; // Case where path is a qualifier of a use tree, e.g. foo::bar::{Baz, Qux} where we are From 92d447f9766e07747815a9fc01d25f95df0be581 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 25 Nov 2023 17:10:18 +0100 Subject: [PATCH 12/24] =?UTF-8?q?=F0=9F=A7=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/hir-def/src/attr/tests.rs | 2 +- crates/hir-def/src/expander.rs | 2 +- crates/hir-def/src/lower.rs | 2 +- .../hir-def/src/nameres/tests/incremental.rs | 1 + crates/hir-def/src/path.rs | 4 +- crates/hir-def/src/path/lower.rs | 18 ++-- crates/hir-expand/src/ast_id_map.rs | 4 +- crates/hir-expand/src/attrs.rs | 8 +- crates/hir-expand/src/builtin_fn_macro.rs | 2 +- crates/hir-expand/src/db.rs | 89 +++++++------------ crates/hir-expand/src/eager.rs | 8 +- crates/hir-expand/src/hygiene.rs | 2 +- crates/hir-expand/src/lib.rs | 5 +- crates/hir-expand/src/span.rs | 3 +- crates/hir-ty/src/display.rs | 2 +- crates/hir/src/attrs.rs | 2 +- crates/hir/src/db.rs | 23 ++++- crates/hir/src/lib.rs | 13 +-- crates/hir/src/semantics.rs | 4 +- crates/hir/src/source_analyzer.rs | 4 +- crates/ide-completion/src/completions.rs | 2 +- crates/ide-db/src/apply_change.rs | 15 +++- crates/ide/src/hover.rs | 1 - crates/mbe/src/syntax_bridge.rs | 1 - crates/mbe/src/token_map.rs | 6 -- crates/proc-macro-api/src/msg.rs | 2 +- 26 files changed, 102 insertions(+), 123 deletions(-) diff --git a/crates/hir-def/src/attr/tests.rs b/crates/hir-def/src/attr/tests.rs index 0f21dc98299f..796f165c7c87 100644 --- a/crates/hir-def/src/attr/tests.rs +++ b/crates/hir-def/src/attr/tests.rs @@ -13,7 +13,7 @@ fn assert_parse_result(input: &str, expected: DocExpr) { let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); let tt = syntax_node_to_token_tree( tt.syntax(), - SpanMapRef::RealSpanMap(&RealSpanMap::empty(FileId(0))), + SpanMapRef::RealSpanMap(&RealSpanMap::absolute(FileId(0))), ); let cfg = DocExpr::parse(&tt); assert_eq!(cfg, expected); diff --git a/crates/hir-def/src/expander.rs b/crates/hir-def/src/expander.rs index 964f07072316..e36aa19b8ddc 100644 --- a/crates/hir-def/src/expander.rs +++ b/crates/hir-def/src/expander.rs @@ -131,7 +131,7 @@ impl Expander { pub(crate) fn parse_path(&mut self, db: &dyn DefDatabase, path: ast::Path) -> Option { let ctx = LowerCtx::new(db, self.span_map.clone(), self.current_file_id); - Path::from_src(path, &ctx) + Path::from_src(&ctx, path) } fn within_limit( diff --git a/crates/hir-def/src/lower.rs b/crates/hir-def/src/lower.rs index 6e7fb0845407..a3505b65fe72 100644 --- a/crates/hir-def/src/lower.rs +++ b/crates/hir-def/src/lower.rs @@ -40,7 +40,7 @@ impl<'a> LowerCtx<'a> { } pub(crate) fn lower_path(&self, ast: ast::Path) -> Option { - Path::from_src(ast, self) + Path::from_src(self, ast) } pub(crate) fn ast_id(&self, item: &N) -> Option> { diff --git a/crates/hir-def/src/nameres/tests/incremental.rs b/crates/hir-def/src/nameres/tests/incremental.rs index 3d7784bdd542..78cb78e833ec 100644 --- a/crates/hir-def/src/nameres/tests/incremental.rs +++ b/crates/hir-def/src/nameres/tests/incremental.rs @@ -11,6 +11,7 @@ fn check_def_map_is_not_recomputed(ra_fixture_initial: &str, ra_fixture_change: let (mut db, pos) = TestDB::with_position(ra_fixture_initial); let krate = { let crate_graph = db.crate_graph(); + // Some of these tests use minicore/proc-macros which will be injected as the first crate crate_graph.iter().last().unwrap() }; { diff --git a/crates/hir-def/src/path.rs b/crates/hir-def/src/path.rs index 3894172a5ad8..215c49d4c2ce 100644 --- a/crates/hir-def/src/path.rs +++ b/crates/hir-def/src/path.rs @@ -96,8 +96,8 @@ pub enum GenericArg { impl Path { /// Converts an `ast::Path` to `Path`. Works with use trees. /// It correctly handles `$crate` based path from macro call. - pub fn from_src(path: ast::Path, ctx: &LowerCtx<'_>) -> Option { - lower::lower_path(path, ctx) + pub fn from_src(ctx: &LowerCtx<'_>, path: ast::Path) -> Option { + lower::lower_path(ctx, path) } /// Converts a known mod path to `Path`. diff --git a/crates/hir-def/src/path/lower.rs b/crates/hir-def/src/path/lower.rs index 9ab4bdaa4ac8..39f1b6f1c06d 100644 --- a/crates/hir-def/src/path/lower.rs +++ b/crates/hir-def/src/path/lower.rs @@ -16,12 +16,9 @@ use crate::{ type_ref::{LifetimeRef, TypeBound, TypeRef}, }; -// fn resolve_crate_root - /// Converts an `ast::Path` to `Path`. Works with use trees. /// It correctly handles `$crate` based path from macro call. -// FIXME: flip the params -pub(super) fn lower_path(mut path: ast::Path, ctx: &LowerCtx<'_>) -> Option { +pub(super) fn lower_path(ctx: &LowerCtx<'_>, mut path: ast::Path) -> Option { let mut kind = PathKind::Plain; let mut type_anchor = None; let mut segments = Vec::new(); @@ -36,18 +33,15 @@ pub(super) fn lower_path(mut path: ast::Path, ctx: &LowerCtx<'_>) -> Option { - let name = if name_ref.text() == "$crate" { - kind = resolve_crate_root( + if name_ref.text() == "$crate" { + break kind = resolve_crate_root( ctx.db.upcast(), span_map.span_for_range(name_ref.syntax().text_range()).ctx, ) .map(PathKind::DollarCrate) .unwrap_or(PathKind::Crate); - - break; - } else { - name_ref.as_name() - }; + } + let name = name_ref.as_name(); let args = segment .generic_arg_list() .and_then(|it| lower_generic_args(ctx, it)) @@ -82,7 +76,7 @@ pub(super) fn lower_path(mut path: ast::Path, ctx: &LowerCtx<'_>) -> Option>::Foo desugars to Trait::Foo Some(trait_ref) => { let Path::Normal { mod_path, generic_args: path_generic_args, .. } = - Path::from_src(trait_ref.path()?, ctx)? + Path::from_src(ctx, trait_ref.path()?)? else { return None; }; diff --git a/crates/hir-expand/src/ast_id_map.rs b/crates/hir-expand/src/ast_id_map.rs index 2d24496ab701..be0b72f9dfa4 100644 --- a/crates/hir-expand/src/ast_id_map.rs +++ b/crates/hir-expand/src/ast_id_map.rs @@ -17,10 +17,10 @@ use profile::Count; use rustc_hash::FxHasher; use syntax::{ast, AstNode, AstPtr, SyntaxNode, SyntaxNodePtr}; -pub use base_db::span::ErasedFileAstId; - use crate::db; +pub use base_db::span::ErasedFileAstId; + /// `AstId` points to an AST node in any file. /// /// It is stable across reparses, and can be used as salsa key/value. diff --git a/crates/hir-expand/src/attrs.rs b/crates/hir-expand/src/attrs.rs index 1fdb5dc37b88..f1619db2420a 100644 --- a/crates/hir-expand/src/attrs.rs +++ b/crates/hir-expand/src/attrs.rs @@ -293,13 +293,8 @@ impl Attr { if tts.is_empty() { return None; } - // FIXME: Absolutely wrong - let call_site = match tts.first().unwrap() { - tt::TokenTree::Leaf(l) => l.span().ctx, - tt::TokenTree::Subtree(s) => s.delimiter.open.ctx, - }; // FIXME: This is necessarily a hack. It'd be nice if we could avoid allocation - // here. + // here or maybe just parse a mod path from a token tree directly let subtree = tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: tts.to_vec(), @@ -313,6 +308,7 @@ impl Attr { return None; } let path = meta.path()?; + let call_site = span_map.span_for_range(path.syntax().text_range()).ctx; Some(( ModPath::from_src(db, path, SpanMapRef::ExpansionSpanMap(&span_map))?, call_site, diff --git a/crates/hir-expand/src/builtin_fn_macro.rs b/crates/hir-expand/src/builtin_fn_macro.rs index 9fb04a2f1b08..459d1c868d80 100644 --- a/crates/hir-expand/src/builtin_fn_macro.rs +++ b/crates/hir-expand/src/builtin_fn_macro.rs @@ -553,7 +553,7 @@ pub(crate) fn include_arg_to_tt( let Some(EagerCallInfo { arg, arg_id, .. }) = loc.eager.as_deref() else { panic!("include_arg_to_tt called on non include macro call: {:?}", &loc.eager); }; - let path = parse_string(&arg.0)?; + let path = parse_string(&arg)?; let file_id = relative_file(db, *arg_id, &path, false)?; // why are we not going through a SyntaxNode here? diff --git a/crates/hir-expand/src/db.rs b/crates/hir-expand/src/db.rs index e31034884c5e..fed1705fb7d3 100644 --- a/crates/hir-expand/src/db.rs +++ b/crates/hir-expand/src/db.rs @@ -190,15 +190,16 @@ pub fn expand_speculative( speculative_args: &SyntaxNode, token_to_map: SyntaxToken, ) -> Option<(SyntaxNode, SyntaxToken)> { - // FIXME spanmaps let loc = db.lookup_intern_macro_call(actual_macro_call); // Build the subtree and token mapping for the speculative args let _censor = censor_for_macro_input(&loc, speculative_args); + let span_map = RealSpanMap::absolute(SpanAnchor::DUMMY.file_id); + let span_map = SpanMapRef::RealSpanMap(&span_map); let mut tt = mbe::syntax_node_to_token_tree( speculative_args, // we don't leak these spans into any query so its fine to make them absolute - SpanMapRef::RealSpanMap(&RealSpanMap::empty(SpanAnchor::DUMMY.file_id)), + span_map, ); let attr_arg = match loc.kind { @@ -216,10 +217,7 @@ pub fn expand_speculative( }?; match attr.token_tree() { Some(token_tree) => { - let mut tree = syntax_node_to_token_tree( - token_tree.syntax(), - SpanMapRef::RealSpanMap(&RealSpanMap::empty(SpanAnchor::DUMMY.file_id)), - ); + let mut tree = syntax_node_to_token_tree(token_tree.syntax(), span_map); tree.delimiter = tt::Delimiter::UNSPECIFIED; Some(tree) @@ -243,12 +241,7 @@ pub fn expand_speculative( MacroDefKind::BuiltInDerive(expander, ..) => { // this cast is a bit sus, can we avoid losing the typedness here? let adt = ast::Adt::cast(speculative_args.clone()).unwrap(); - expander.expand( - db, - actual_macro_call, - &adt, - SpanMapRef::RealSpanMap(&RealSpanMap::empty(SpanAnchor::DUMMY.file_id)), - ) + expander.expand(db, actual_macro_call, &adt, span_map) } MacroDefKind::Declarative(it) => { db.decl_macro_expander(loc.krate, it).expand_unhygienic(tt) @@ -305,6 +298,8 @@ fn parse_or_expand_with_err( } } +// FIXME: We should verify that the parsed node is one of the many macro node variants we expect +// instead of having it be untyped fn parse_macro_expansion( db: &dyn ExpandDatabase, macro_file: MacroFileId, @@ -330,6 +325,18 @@ fn parse_macro_expansion_error( .map(|it| it.0.errors().to_vec().into_boxed_slice()) } +fn parse_with_map(db: &dyn ExpandDatabase, file_id: HirFileId) -> (Parse, SpanMap) { + match file_id.repr() { + HirFileIdRepr::FileId(file_id) => { + (db.parse(file_id).to_syntax(), SpanMap::RealSpanMap(db.real_span_map(file_id))) + } + HirFileIdRepr::MacroFile(macro_file) => { + let (parse, map) = db.parse_macro_expansion(macro_file).value; + (parse, SpanMap::ExpansionSpanMap(map)) + } + } +} + fn macro_arg( db: &dyn ExpandDatabase, id: MacroCallId, @@ -361,32 +368,22 @@ fn macro_arg( .then(|| loc.eager.as_deref()) .flatten() { - ValueResult::ok(Some(Arc::new(arg.0.clone()))) + ValueResult::ok(Some(arg.clone())) } else { - //FIXME: clean this up, the ast id map lookup is done twice here - let (parse, map) = match loc.kind.file_id().repr() { - HirFileIdRepr::FileId(file_id) => { - let syntax = db.parse(file_id).to_syntax(); - - (syntax, SpanMap::RealSpanMap(db.real_span_map(file_id))) - } - HirFileIdRepr::MacroFile(macro_file) => { - let (parse, map) = db.parse_macro_expansion(macro_file).value; - (parse, SpanMap::ExpansionSpanMap(map)) - } - }; + let (parse, map) = parse_with_map(db, loc.kind.file_id()); let root = parse.syntax_node(); let syntax = match loc.kind { MacroCallKind::FnLike { ast_id, .. } => { let node = &ast_id.to_ptr(db).to_node(&root); let offset = node.syntax().text_range().start(); - match node.token_tree().map(|it| it.syntax().clone()) { + match node.token_tree() { Some(tt) => { - if let Some(e) = mismatched_delimiters(&tt) { + let tt = tt.syntax(); + if let Some(e) = mismatched_delimiters(tt) { return ValueResult::only_err(e); } - tt + tt.clone() } None => { return ValueResult::only_err(Arc::new(Box::new([ @@ -479,17 +476,8 @@ fn decl_macro_expander( id: AstId, ) -> Arc { let is_2021 = db.crate_graph()[def_crate].edition >= Edition::Edition2021; - let (root, map) = match id.file_id.repr() { - HirFileIdRepr::FileId(file_id) => { - // FIXME: Arc - // FIXME: id.to_ptr duplicated, expensive - (db.parse(file_id).syntax_node(), SpanMap::RealSpanMap(db.real_span_map(file_id))) - } - HirFileIdRepr::MacroFile(macro_file) => { - let (parse, map) = db.parse_macro_expansion(macro_file).value; - (parse.syntax_node(), SpanMap::ExpansionSpanMap(map)) - } - }; + let (root, map) = parse_with_map(db, id.file_id); + let root = root.syntax_node(); let transparency = |node| { // ... would be nice to have the item tree here @@ -568,21 +556,8 @@ fn macro_expand( let ExpandResult { value: tt, mut err } = match loc.def.kind { MacroDefKind::ProcMacro(..) => return db.expand_proc_macro(macro_call_id), MacroDefKind::BuiltInDerive(expander, ..) => { - // FIXME: add firewall query for this? - let hir_file_id = loc.kind.file_id(); - let (root, map) = match hir_file_id.repr() { - HirFileIdRepr::FileId(file_id) => { - // FIXME: query for span map - ( - db.parse(file_id).syntax_node(), - SpanMap::RealSpanMap(db.real_span_map(file_id)), - ) - } - HirFileIdRepr::MacroFile(macro_file) => { - let (parse, map) = db.parse_macro_expansion(macro_file).value; - (parse.syntax_node(), SpanMap::ExpansionSpanMap(map)) - } - }; + let (root, map) = parse_with_map(db, loc.kind.file_id()); + let root = root.syntax_node(); let MacroCallKind::Derive { ast_id, .. } = loc.kind else { unreachable!() }; let node = ast_id.to_ptr(db).to_node(&root); @@ -710,9 +685,9 @@ fn token_tree_to_syntax_node( ExpandTo::Type => mbe::TopEntryPoint::Type, ExpandTo::Expr => mbe::TopEntryPoint::Expr, }; - let mut tm = mbe::token_tree_to_syntax_node(tt, entry_point); + let (parse, mut span_map) = mbe::token_tree_to_syntax_node(tt, entry_point); // FIXME: now what the hell is going on here - tm.1.span_map.sort_by(|(_, a), (_, b)| { + span_map.span_map.sort_by(|(_, a), (_, b)| { a.anchor.file_id.cmp(&b.anchor.file_id).then_with(|| { let map = db.ast_id_map(a.anchor.file_id.into()); map.get_erased(a.anchor.ast_id) @@ -721,7 +696,7 @@ fn token_tree_to_syntax_node( .cmp(&map.get_erased(b.anchor.ast_id).text_range().start()) }) }); - tm + (parse, span_map) } fn check_tt_count(tt: &tt::Subtree) -> Result<(), ExpandResult>> { diff --git a/crates/hir-expand/src/eager.rs b/crates/hir-expand/src/eager.rs index 4499c2e69d65..deea59b93b62 100644 --- a/crates/hir-expand/src/eager.rs +++ b/crates/hir-expand/src/eager.rs @@ -81,7 +81,7 @@ pub fn expand_eager_macro_input( // FIXME: Spans! let mut subtree = mbe::syntax_node_to_token_tree( &expanded_eager_input, - RealSpanMap::empty(::DUMMY.file_id), + RealSpanMap::absolute(::DUMMY.file_id), ); subtree.delimiter = crate::tt::Delimiter::UNSPECIFIED; @@ -89,11 +89,7 @@ pub fn expand_eager_macro_input( let loc = MacroCallLoc { def, krate, - eager: Some(Box::new(EagerCallInfo { - arg: Arc::new((subtree,)), - arg_id, - error: err.clone(), - })), + eager: Some(Box::new(EagerCallInfo { arg: Arc::new(subtree), arg_id, error: err.clone() })), kind: MacroCallKind::FnLike { ast_id: call_id, expand_to }, call_site, }; diff --git a/crates/hir-expand/src/hygiene.rs b/crates/hir-expand/src/hygiene.rs index 66d9f679d894..8fdfa8af0cac 100644 --- a/crates/hir-expand/src/hygiene.rs +++ b/crates/hir-expand/src/hygiene.rs @@ -8,7 +8,7 @@ use base_db::span::{MacroCallId, SpanData, SyntaxContextId}; use crate::db::ExpandDatabase; -#[derive(Debug, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct SyntaxContextData { pub outer_expn: Option, pub outer_transparency: Transparency, diff --git a/crates/hir-expand/src/lib.rs b/crates/hir-expand/src/lib.rs index 13eb90b18a8e..4e5aa9031261 100644 --- a/crates/hir-expand/src/lib.rs +++ b/crates/hir-expand/src/lib.rs @@ -135,7 +135,7 @@ pub enum MacroDefKind { #[derive(Debug, Clone, PartialEq, Eq, Hash)] struct EagerCallInfo { /// The expanded argument of the eager macro. - arg: Arc<(tt::Subtree,)>, + arg: Arc, /// Call id of the eager macro's input file (this is the macro file for its fully expanded input). arg_id: MacroCallId, error: Option, @@ -537,8 +537,6 @@ impl MacroCallKind { FileRange { range, file_id } } - // FIXME: -> InFile it should be impossible for the token tree to be missing at - // this point! fn arg(&self, db: &dyn db::ExpandDatabase) -> InFile> { match self { MacroCallKind::FnLike { ast_id, .. } => { @@ -561,7 +559,6 @@ impl MacroCallKind { pub struct ExpansionInfo { pub expanded: InMacroFile, /// The argument TokenTree or item for attributes - // FIXME: Can this ever be `None`? arg: InFile>, /// The `macro_rules!` or attribute input. attr_input_or_mac_def: Option>, diff --git a/crates/hir-expand/src/span.rs b/crates/hir-expand/src/span.rs index c16d761c1719..589f415de566 100644 --- a/crates/hir-expand/src/span.rs +++ b/crates/hir-expand/src/span.rs @@ -77,7 +77,8 @@ pub struct RealSpanMap { } impl RealSpanMap { - pub fn empty(file_id: FileId) -> Self { + /// Creates a real file span map that returns absolute ranges (relative ranges to the root ast id). + pub fn absolute(file_id: FileId) -> Self { RealSpanMap { file_id, pairs: Box::from([(TextSize::new(0), ROOT_ERASED_FILE_AST_ID)]) } } diff --git a/crates/hir-ty/src/display.rs b/crates/hir-ty/src/display.rs index 3d426caf680e..a324129b351c 100644 --- a/crates/hir-ty/src/display.rs +++ b/crates/hir-ty/src/display.rs @@ -1738,7 +1738,7 @@ impl HirDisplay for TypeRef { ); let macro_call = macro_call.to_node(f.db.upcast()); match macro_call.path() { - Some(path) => match Path::from_src(path, &ctx) { + Some(path) => match Path::from_src(&ctx, path) { Some(path) => path.hir_fmt(f)?, None => write!(f, "{{macro}}")?, }, diff --git a/crates/hir/src/attrs.rs b/crates/hir/src/attrs.rs index 42d1515e5931..0ac1db9311b1 100644 --- a/crates/hir/src/attrs.rs +++ b/crates/hir/src/attrs.rs @@ -241,7 +241,7 @@ fn modpath_from_str(db: &dyn HirDatabase, link: &str) -> Option { ModPath::from_src( db.upcast(), ast_path, - SpanMapRef::RealSpanMap(&RealSpanMap::empty(FileId(0))), + SpanMapRef::RealSpanMap(&RealSpanMap::absolute(FileId(0))), ) }; diff --git a/crates/hir/src/db.rs b/crates/hir/src/db.rs index cc3e869aa7ce..b9112dee3166 100644 --- a/crates/hir/src/db.rs +++ b/crates/hir/src/db.rs @@ -3,10 +3,27 @@ //! we didn't do that. //! //! But we need this for at least LRU caching at the query level. -pub use hir_def::db::*; +pub use hir_def::db::{ + AttrsQuery, BlockDefMapQuery, BlockItemTreeQueryQuery, BodyQuery, BodyWithSourceMapQuery, + ConstDataQuery, ConstVisibilityQuery, CrateDefMapQueryQuery, CrateLangItemsQuery, + CrateSupportsNoStdQuery, DefDatabase, DefDatabaseStorage, EnumDataQuery, + EnumDataWithDiagnosticsQuery, ExprScopesQuery, ExternCrateDeclDataQuery, + FieldVisibilitiesQuery, FieldsAttrsQuery, FieldsAttrsSourceMapQuery, FileItemTreeQuery, + FunctionDataQuery, FunctionVisibilityQuery, GenericParamsQuery, ImplDataQuery, + ImplDataWithDiagnosticsQuery, ImportMapQuery, InternAnonymousConstQuery, InternBlockQuery, + InternConstQuery, InternDatabase, InternDatabaseStorage, InternEnumQuery, + InternExternBlockQuery, InternExternCrateQuery, InternFunctionQuery, InternImplQuery, + InternInTypeConstQuery, InternMacro2Query, InternMacroRulesQuery, InternProcMacroQuery, + InternStaticQuery, InternStructQuery, InternTraitAliasQuery, InternTraitQuery, + InternTypeAliasQuery, InternUnionQuery, InternUseQuery, LangAttrQuery, LangItemQuery, + Macro2DataQuery, MacroRulesDataQuery, ProcMacroDataQuery, StaticDataQuery, StructDataQuery, + StructDataWithDiagnosticsQuery, TraitAliasDataQuery, TraitDataQuery, + TraitDataWithDiagnosticsQuery, TypeAliasDataQuery, UnionDataQuery, + UnionDataWithDiagnosticsQuery, VariantsAttrsQuery, VariantsAttrsSourceMapQuery, +}; pub use hir_expand::db::{ AstIdMapQuery, DeclMacroExpanderQuery, ExpandDatabase, ExpandDatabaseStorage, - ExpandProcMacroQuery, InternMacroCallQuery, MacroArgQuery, MacroExpandQuery, - ParseMacroExpansionErrorQuery, ParseMacroExpansionQuery, + ExpandProcMacroQuery, IncludeExpandQuery, InternMacroCallQuery, MacroArgQuery, + MacroExpandQuery, ParseMacroExpansionErrorQuery, ParseMacroExpansionQuery, RealSpanMapQuery, }; pub use hir_ty::db::*; diff --git a/crates/hir/src/lib.rs b/crates/hir/src/lib.rs index aacc3b08cb79..7687fcbccaec 100644 --- a/crates/hir/src/lib.rs +++ b/crates/hir/src/lib.rs @@ -137,7 +137,13 @@ pub use { // These are negative re-exports: pub using these names is forbidden, they // should remain private to hir internals. #[allow(unused)] -use {hir_def::path::Path, hir_expand::name::AsName}; +use { + hir_def::path::Path, + hir_expand::{ + name::AsName, + span::{ExpansionSpanMap, RealSpanMap, SpanMap, SpanMapRef}, + }, +}; /// hir::Crate describes a single crate. It's the main interface with which /// a crate's dependencies interact. Mostly, it should be just a proxy for the @@ -3483,11 +3489,6 @@ impl Impl { self.id.lookup(db.upcast()).container.into() } - pub fn as_builtin_derive_attr(self, db: &dyn HirDatabase) -> Option> { - let src = self.source(db)?; - src.file_id.as_builtin_derive_attr_node(db.upcast()) - } - pub fn as_builtin_derive_path(self, db: &dyn HirDatabase) -> Option> { let src = self.source(db)?; diff --git a/crates/hir/src/semantics.rs b/crates/hir/src/semantics.rs index 666ffd390ae7..f6ee836c5294 100644 --- a/crates/hir/src/semantics.rs +++ b/crates/hir/src/semantics.rs @@ -868,7 +868,7 @@ impl<'db> SemanticsImpl<'db> { let analyze = self.analyze(path.syntax())?; let span_map = self.db.span_map(analyze.file_id); let ctx = LowerCtx::with_span_map(self.db.upcast(), span_map); - let hir_path = Path::from_src(path.clone(), &ctx)?; + let hir_path = Path::from_src(&ctx, path.clone())?; match analyze.resolver.resolve_path_in_type_ns_fully(self.db.upcast(), &hir_path)? { TypeNs::TraitId(id) => Some(Trait { id }), _ => None, @@ -1466,7 +1466,7 @@ impl SemanticsScope<'_> { /// necessary a heuristic, as it doesn't take hygiene into account. pub fn speculative_resolve(&self, path: &ast::Path) -> Option { let ctx = LowerCtx::with_file_id(self.db.upcast(), self.file_id); - let path = Path::from_src(path.clone(), &ctx)?; + let path = Path::from_src(&ctx, path.clone())?; resolve_hir_path(self.db, &self.resolver, &path) } diff --git a/crates/hir/src/source_analyzer.rs b/crates/hir/src/source_analyzer.rs index 2339187122ee..0f1093e6d143 100644 --- a/crates/hir/src/source_analyzer.rs +++ b/crates/hir/src/source_analyzer.rs @@ -483,7 +483,7 @@ impl SourceAnalyzer { macro_call: InFile<&ast::MacroCall>, ) -> Option { let ctx = LowerCtx::with_file_id(db.upcast(), macro_call.file_id); - let path = macro_call.value.path().and_then(|ast| Path::from_src(ast, &ctx))?; + let path = macro_call.value.path().and_then(|ast| Path::from_src(&ctx, ast))?; self.resolver .resolve_path_as_macro(db.upcast(), path.mod_path()?, Some(MacroSubNs::Bang)) .map(|(it, _)| it.into()) @@ -596,7 +596,7 @@ impl SourceAnalyzer { // This must be a normal source file rather than macro file. let ctx = LowerCtx::with_span_map(db.upcast(), db.span_map(self.file_id)); - let hir_path = Path::from_src(path.clone(), &ctx)?; + let hir_path = Path::from_src(&ctx, path.clone())?; // Case where path is a qualifier of a use tree, e.g. foo::bar::{Baz, Qux} where we are // trying to resolve foo::bar. diff --git a/crates/ide-completion/src/completions.rs b/crates/ide-completion/src/completions.rs index f49abcbae9bc..7d38c638a8ed 100644 --- a/crates/ide-completion/src/completions.rs +++ b/crates/ide-completion/src/completions.rs @@ -683,7 +683,7 @@ pub(super) fn complete_name_ref( ctx: &CompletionContext<'_>, NameRefContext { nameref, kind }: &NameRefContext, ) { - match dbg!(kind) { + match kind { NameRefKind::Path(path_ctx) => { flyimport::import_on_the_fly_path(acc, ctx, path_ctx); diff --git a/crates/ide-db/src/apply_change.rs b/crates/ide-db/src/apply_change.rs index dc77424c1314..2efe3ee1ed75 100644 --- a/crates/ide-db/src/apply_change.rs +++ b/crates/ide-db/src/apply_change.rs @@ -97,12 +97,14 @@ impl RootDatabase { // ExpandDatabase hir::db::AstIdMapQuery - hir::db::ParseMacroExpansionQuery + hir::db::DeclMacroExpanderQuery + hir::db::ExpandProcMacroQuery + hir::db::IncludeExpandQuery hir::db::InternMacroCallQuery hir::db::MacroArgQuery - hir::db::DeclMacroExpanderQuery hir::db::MacroExpandQuery - hir::db::ExpandProcMacroQuery + hir::db::ParseMacroExpansionQuery + hir::db::RealSpanMapQuery // DefDatabase hir::db::FileItemTreeQuery @@ -142,6 +144,13 @@ impl RootDatabase { hir::db::FunctionVisibilityQuery hir::db::ConstVisibilityQuery hir::db::CrateSupportsNoStdQuery + hir::db::BlockItemTreeQueryQuery + hir::db::ExternCrateDeclDataQuery + hir::db::LangAttrQuery + hir::db::InternAnonymousConstQuery + hir::db::InternExternCrateQuery + hir::db::InternInTypeConstQuery + hir::db::InternUseQuery // HirDatabase hir::db::InferQueryQuery diff --git a/crates/ide/src/hover.rs b/crates/ide/src/hover.rs index fc0c9ef17fa1..e0b64fe7988e 100644 --- a/crates/ide/src/hover.rs +++ b/crates/ide/src/hover.rs @@ -166,7 +166,6 @@ fn hover_simple( } else { sema.descend_into_macros_with_same_text(original_token.clone(), offset) }; - dbg!(&descended); let descended = || descended.iter(); let result = descended() diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 688ccb23252c..bbf49670ce59 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -565,7 +565,6 @@ impl Converter { #[derive(Debug)] enum SynToken { Ordinary(SyntaxToken), - // FIXME is this supposed to be `Punct`? Punct(SyntaxToken, usize), } diff --git a/crates/mbe/src/token_map.rs b/crates/mbe/src/token_map.rs index 978ee268c5cc..32f61af25e5d 100644 --- a/crates/mbe/src/token_map.rs +++ b/crates/mbe/src/token_map.rs @@ -6,12 +6,6 @@ use stdx::itertools::Itertools; use syntax::TextRange; use tt::Span; -// pub type HirFile = u32; -// pub type FileRange = (HirFile, TextRange); -// Option, LocalSyntaxContet -// pub type SyntaxContext = (); -// pub type LocalSyntaxContext = u32; - /// Maps absolute text ranges for the corresponding file to the relevant span data. #[derive(Debug, PartialEq, Eq, Clone, Hash)] // FIXME: Rename to SpanMap diff --git a/crates/proc-macro-api/src/msg.rs b/crates/proc-macro-api/src/msg.rs index 6a7329e322f3..4e6984f61b72 100644 --- a/crates/proc-macro-api/src/msg.rs +++ b/crates/proc-macro-api/src/msg.rs @@ -120,7 +120,7 @@ fn write_json(out: &mut impl Write, msg: &str) -> io::Result<()> { Ok(()) } -/* +/*TODO #[cfg(test)] mod tests { From 98cfdde8ba5c784fb5d7114070abb80e01a6d2bb Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Tue, 28 Nov 2023 10:55:21 +0100 Subject: [PATCH 13/24] Thinner TokenMap --- Cargo.lock | 2 +- crates/base-db/src/span.rs | 6 +- .../hir-def/src/macro_expansion_tests/mbe.rs | 2 +- crates/hir-expand/src/attrs.rs | 2 +- crates/hir-expand/src/db.rs | 18 +-- crates/hir-expand/src/lib.rs | 91 +++++------- crates/hir-expand/src/span.rs | 6 +- crates/hir/src/semantics.rs | 65 ++++---- crates/hir/src/semantics/source_to_def.rs | 4 +- crates/hir/src/source_analyzer.rs | 7 +- crates/mbe/src/syntax_bridge.rs | 36 ++--- crates/mbe/src/token_map.rs | 140 +++--------------- .../src/integrated_benchmarks.rs | 3 +- 13 files changed, 123 insertions(+), 259 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3618d69c749a..775231f3ea2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1752,7 +1752,7 @@ dependencies = [ "always-assert", "backtrace", "crossbeam-channel", - "itertools 0.12.0", + "itertools", "jod-thread", "libc", "miow", diff --git a/crates/base-db/src/span.rs b/crates/base-db/src/span.rs index 370c732813ac..607b8027ca95 100644 --- a/crates/base-db/src/span.rs +++ b/crates/base-db/src/span.rs @@ -33,10 +33,11 @@ impl SyntaxContext for SyntaxContextId { impl SyntaxContextId { // TODO: This is very much UB, salsa exposes no way to create an InternId in a const context // currently (which kind of makes sense but we need it here!) - pub const ROOT: Self = SyntaxContextId(unsafe { core::mem::transmute(1) }); + pub const ROOT: Self = SyntaxContextId(unsafe { InternId::new_unchecked(0) }); // TODO: This is very much UB, salsa exposes no way to create an InternId in a const context // currently (which kind of makes sense but we need it here!) - pub const SELF_REF: Self = SyntaxContextId(unsafe { core::mem::transmute(!0u32) }); + pub const SELF_REF: Self = + SyntaxContextId(unsafe { InternId::new_unchecked(InternId::MAX - 1) }); pub fn is_root(self) -> bool { self == Self::ROOT @@ -107,6 +108,7 @@ impl fmt::Debug for HirFileId { pub struct MacroFileId { pub macro_call_id: MacroCallId, } + /// `MacroCallId` identifies a particular macro invocation, like /// `println!("Hello, {}", world)`. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] diff --git a/crates/hir-def/src/macro_expansion_tests/mbe.rs b/crates/hir-def/src/macro_expansion_tests/mbe.rs index fc17dcde9a07..39079685a4d9 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe.rs @@ -36,7 +36,7 @@ macro_rules! f { } struct#FileId(0):1@58..64\2# MyTraitMap2#FileId(0):2@31..42\0# {#FileId(0):1@72..73\2# - map#FileId(0):1@86..89\2#:#FileId(0):1@89..90\2# #FileId(0):1@89..90\2#::#FileId(0):1@92..93\2#std#FileId(0):1@93..96\2#::#FileId(0):1@97..98\2#collections#FileId(0):1@98..109\2#::#FileId(0):1@110..111\2#HashSet#FileId(0):1@111..118\2#<#FileId(0):1@118..119\2#(#FileId(0):1@119..120\2#)#FileId(0):1@120..121\2#>#FileId(0):1@121..122\2#,#FileId(0):1@122..123\2# + map#FileId(0):1@86..89\2#:#FileId(0):1@89..90\2# #FileId(0):1@89..90\2#::#FileId(0):1@91..92\2#std#FileId(0):1@93..96\2#::#FileId(0):1@96..97\2#collections#FileId(0):1@98..109\2#::#FileId(0):1@109..110\2#HashSet#FileId(0):1@111..118\2#<#FileId(0):1@118..119\2#(#FileId(0):1@119..120\2#)#FileId(0):1@120..121\2#>#FileId(0):1@121..122\2#,#FileId(0):1@122..123\2# }#FileId(0):1@132..133\2# "#]], ); diff --git a/crates/hir-expand/src/attrs.rs b/crates/hir-expand/src/attrs.rs index f1619db2420a..23a8fffa8c29 100644 --- a/crates/hir-expand/src/attrs.rs +++ b/crates/hir-expand/src/attrs.rs @@ -308,7 +308,7 @@ impl Attr { return None; } let path = meta.path()?; - let call_site = span_map.span_for_range(path.syntax().text_range()).ctx; + let call_site = span_map.span_at(path.syntax().text_range().start()).ctx; Some(( ModPath::from_src(db, path, SpanMapRef::ExpansionSpanMap(&span_map))?, call_site, diff --git a/crates/hir-expand/src/db.rs b/crates/hir-expand/src/db.rs index fed1705fb7d3..601a754abbd4 100644 --- a/crates/hir-expand/src/db.rs +++ b/crates/hir-expand/src/db.rs @@ -254,7 +254,7 @@ pub fn expand_speculative( }; let expand_to = macro_expand_to(db, actual_macro_call); - let (node, rev_tmap) = token_tree_to_syntax_node(db, &speculative_expansion.value, expand_to); + let (node, rev_tmap) = token_tree_to_syntax_node(&speculative_expansion.value, expand_to); let syntax_node = node.syntax_node(); let token = rev_tmap @@ -312,7 +312,7 @@ fn parse_macro_expansion( tracing::debug!("expanded = {}", tt.as_debug_string()); tracing::debug!("kind = {:?}", expand_to); - let (parse, rev_token_map) = token_tree_to_syntax_node(db, &tt, expand_to); + let (parse, rev_token_map) = token_tree_to_syntax_node(&tt, expand_to); ExpandResult { value: (parse, Arc::new(rev_token_map)), err } } @@ -674,7 +674,6 @@ fn macro_expand_to(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandTo { } fn token_tree_to_syntax_node( - db: &dyn ExpandDatabase, tt: &tt::Subtree, expand_to: ExpandTo, ) -> (Parse, ExpansionSpanMap) { @@ -685,18 +684,7 @@ fn token_tree_to_syntax_node( ExpandTo::Type => mbe::TopEntryPoint::Type, ExpandTo::Expr => mbe::TopEntryPoint::Expr, }; - let (parse, mut span_map) = mbe::token_tree_to_syntax_node(tt, entry_point); - // FIXME: now what the hell is going on here - span_map.span_map.sort_by(|(_, a), (_, b)| { - a.anchor.file_id.cmp(&b.anchor.file_id).then_with(|| { - let map = db.ast_id_map(a.anchor.file_id.into()); - map.get_erased(a.anchor.ast_id) - .text_range() - .start() - .cmp(&map.get_erased(b.anchor.ast_id).text_range().start()) - }) - }); - (parse, span_map) + mbe::token_tree_to_syntax_node(tt, entry_point) } fn check_tt_count(tt: &tt::Subtree) -> Result<(), ExpandResult>> { diff --git a/crates/hir-expand/src/lib.rs b/crates/hir-expand/src/lib.rs index 4e5aa9031261..9027ea1c27b4 100644 --- a/crates/hir-expand/src/lib.rs +++ b/crates/hir-expand/src/lib.rs @@ -44,7 +44,7 @@ use crate::{ db::TokenExpander, mod_path::ModPath, proc_macro::ProcMacroExpander, - span::ExpansionSpanMap, + span::{ExpansionSpanMap, SpanMap}, }; pub use crate::ast_id_map::{AstId, ErasedAstId, ErasedFileAstId}; @@ -172,7 +172,6 @@ pub trait HirFileIdExt { /// For macro-expansion files, returns the file original source file the /// expansion originated from. fn original_file(self, db: &dyn db::ExpandDatabase) -> FileId; - fn expansion_level(self, db: &dyn db::ExpandDatabase) -> u32; /// If this is a macro call, returns the syntax node of the call. fn call_node(self, db: &dyn db::ExpandDatabase) -> Option>; @@ -218,18 +217,6 @@ impl HirFileIdExt for HirFileId { } } - fn expansion_level(self, db: &dyn db::ExpandDatabase) -> u32 { - let mut level = 0; - let mut curr = self; - while let Some(macro_file) = curr.macro_file() { - let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_file.macro_call_id); - - level += 1; - curr = loc.kind.file_id(); - } - level - } - fn call_node(self, db: &dyn db::ExpandDatabase) -> Option> { let macro_file = self.macro_file()?; let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_file.macro_call_id); @@ -330,6 +317,32 @@ impl HirFileIdExt for HirFileId { } } +pub trait MacroFileIdExt { + fn expansion_level(self, db: &dyn db::ExpandDatabase) -> u32; + fn expansion_info(self, db: &dyn db::ExpandDatabase) -> ExpansionInfo; +} + +impl MacroFileIdExt for MacroFileId { + fn expansion_level(self, db: &dyn db::ExpandDatabase) -> u32 { + let mut level = 0; + let mut macro_file = self; + loop { + let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_file.macro_call_id); + + level += 1; + macro_file = match loc.kind.file_id().repr() { + HirFileIdRepr::FileId(_) => break level, + HirFileIdRepr::MacroFile(it) => it, + }; + } + } + + /// Return expansion information if it is a macro-expansion file + fn expansion_info(self, db: &dyn db::ExpandDatabase) -> ExpansionInfo { + ExpansionInfo::new(db, self) + } +} + impl MacroDefId { pub fn as_lazy_macro( self, @@ -398,7 +411,7 @@ impl MacroCallLoc { match file_id.repr() { HirFileIdRepr::FileId(file_id) => db.real_span_map(file_id).span_for_range(range), HirFileIdRepr::MacroFile(m) => { - db.parse_macro_expansion(m).value.1.span_for_range(range) + db.parse_macro_expansion(m).value.1.span_at(range.start()) } } } @@ -565,9 +578,8 @@ pub struct ExpansionInfo { macro_def: TokenExpander, macro_arg: Arc, - exp_map: Arc, - /// [`None`] if the call is in a real file - arg_map: Option>, + pub exp_map: Arc, + arg_map: SpanMap, } impl ExpansionInfo { @@ -582,38 +594,14 @@ impl ExpansionInfo { /// Maps the passed in file range down into a macro expansion if it is the input to a macro call. pub fn map_range_down<'a>( &'a self, - db: &'a dyn db::ExpandDatabase, - FileRange { file_id, range: absolute_range }: FileRange, + span: SpanData, // FIXME: use this for range mapping, so that we can resolve inline format args _relative_token_offset: Option, // FIXME: ret ty should be wrapped in InMacroFile ) -> Option> + 'a> { - // search for all entries in the span map that have the given span and return the - // corresponding text ranges inside the expansion - // FIXME: Make this proper - let span_map = &self.exp_map.span_map; - let (start, end) = if span_map - .first() - .map_or(false, |(_, span)| span.anchor.file_id == file_id) - { - (0, span_map.partition_point(|a| a.1.anchor.file_id == file_id)) - } else { - let start = span_map.partition_point(|a| a.1.anchor.file_id != file_id); - (start, start + span_map[start..].partition_point(|a| a.1.anchor.file_id == file_id)) - }; - let tokens = span_map[start..end] - .iter() - .filter_map(move |(range, span)| { - // we need to resolve the relative ranges here to make sure that we are in fact - // considering differently anchored spans (this might occur with proc-macros) - let offset = db - .ast_id_map(span.anchor.file_id.into()) - .get_erased(span.anchor.ast_id) - .text_range() - .start(); - let abs_range = span.range + offset; - absolute_range.eq(&abs_range).then_some(*range) - }) + let tokens = self + .exp_map + .ranges_with_span(span) .flat_map(move |range| self.expanded.value.covering_element(range).into_token()); Some(tokens.map(move |token| InFile::new(self.expanded.file_id.into(), token))) @@ -626,7 +614,7 @@ impl ExpansionInfo { range: TextRange, ) -> (FileRange, SyntaxContextId) { debug_assert!(self.expanded.value.text_range().contains_range(range)); - let span = self.exp_map.span_for_range(range); + let span = self.exp_map.span_at(range.start()); let anchor_offset = db .ast_id_map(span.anchor.file_id.into()) .get_erased(span.anchor.ast_id) @@ -672,15 +660,15 @@ impl ExpansionInfo { token: TextRange, ) -> InFile> { debug_assert!(self.expanded.value.text_range().contains_range(token)); - let span = self.exp_map.span_for_range(token); + let span = self.exp_map.span_at(token.start()); match &self.arg_map { - None => { + SpanMap::RealSpanMap(_) => { let file_id = span.anchor.file_id.into(); let anchor_offset = db.ast_id_map(file_id).get_erased(span.anchor.ast_id).text_range().start(); InFile { file_id, value: smallvec::smallvec![span.range + anchor_offset] } } - Some(arg_map) => { + SpanMap::ExpansionSpanMap(arg_map) => { let arg_range = self .arg .value @@ -701,8 +689,7 @@ impl ExpansionInfo { let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_file.macro_call_id); let arg_tt = loc.kind.arg(db); - let arg_map = - arg_tt.file_id.macro_file().map(|file| db.parse_macro_expansion(file).value.1); + let arg_map = db.span_map(arg_tt.file_id); let macro_def = db.macro_expander(loc.def); let (parse, exp_map) = db.parse_macro_expansion(macro_file).value; diff --git a/crates/hir-expand/src/span.rs b/crates/hir-expand/src/span.rs index 589f415de566..c2399259facb 100644 --- a/crates/hir-expand/src/span.rs +++ b/crates/hir-expand/src/span.rs @@ -11,7 +11,7 @@ use crate::db::ExpandDatabase; pub type ExpansionSpanMap = TokenMap; /// Spanmap for a macro file or a real file -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] pub enum SpanMap { /// Spanmap for a macro file ExpansionSpanMap(Arc), @@ -46,7 +46,7 @@ impl mbe::SpanMapper for RealSpanMap { impl SpanMap { pub fn span_for_range(&self, range: TextRange) -> SpanData { match self { - Self::ExpansionSpanMap(span_map) => span_map.span_for_range(range), + Self::ExpansionSpanMap(span_map) => span_map.span_at(range.start()), Self::RealSpanMap(span_map) => span_map.span_for_range(range), } } @@ -62,7 +62,7 @@ impl SpanMap { impl SpanMapRef<'_> { pub fn span_for_range(self, range: TextRange) -> SpanData { match self { - Self::ExpansionSpanMap(span_map) => span_map.span_for_range(range), + Self::ExpansionSpanMap(span_map) => span_map.span_at(range.start()), Self::RealSpanMap(span_map) => span_map.span_for_range(range), } } diff --git a/crates/hir/src/semantics.rs b/crates/hir/src/semantics.rs index f6ee836c5294..38c4f081b760 100644 --- a/crates/hir/src/semantics.rs +++ b/crates/hir/src/semantics.rs @@ -17,6 +17,7 @@ use hir_def::{ }; use hir_expand::{ db::ExpandDatabase, files::InRealFile, name::AsName, ExpansionInfo, HirFileIdExt, MacroCallId, + MacroFileId, MacroFileIdExt, }; use itertools::Itertools; use rustc_hash::{FxHashMap, FxHashSet}; @@ -117,11 +118,11 @@ pub struct Semantics<'db, DB> { pub struct SemanticsImpl<'db> { pub db: &'db dyn HirDatabase, s2d_cache: RefCell, - expansion_info_cache: RefCell>>, - // Rootnode to HirFileId cache + expansion_info_cache: RefCell>, + /// Rootnode to HirFileId cache cache: RefCell>, - // MacroCall to its expansion's HirFileId cache - macro_call_cache: RefCell, HirFileId>>, + /// MacroCall to its expansion's MacroFileId cache + macro_call_cache: RefCell, MacroFileId>>, } impl fmt::Debug for Semantics<'_, DB> { @@ -258,7 +259,7 @@ impl<'db> SemanticsImpl<'db> { pub fn expand(&self, macro_call: &ast::MacroCall) -> Option { let sa = self.analyze_no_infer(macro_call.syntax())?; let file_id = sa.expand(self.db, InFile::new(sa.file_id, macro_call))?; - let node = self.parse_or_expand(file_id); + let node = self.parse_or_expand(file_id.into()); Some(node) } @@ -527,6 +528,7 @@ impl<'db> SemanticsImpl<'db> { res } + // FIXME: should only take real file inputs for simplicity fn descend_into_macros_impl( &self, token: SyntaxToken, @@ -537,31 +539,22 @@ impl<'db> SemanticsImpl<'db> { ) { // FIXME: Clean this up let _p = profile::span("descend_into_macros"); - let parent = match token.parent() { + let sa = match token.parent().and_then(|parent| self.analyze_no_infer(&parent)) { Some(it) => it, None => return, }; - let sa = match self.analyze_no_infer(&parent) { - Some(it) => it, - None => return, - }; - let def_map = sa.resolver.def_map(); - let absolute_range = match sa.file_id.repr() { + + let mut cache = self.expansion_info_cache.borrow_mut(); + let mut mcache = self.macro_call_cache.borrow_mut(); + let span = match sa.file_id.repr() { base_db::span::HirFileIdRepr::FileId(file_id) => { - FileRange { file_id, range: token.text_range() } - } - base_db::span::HirFileIdRepr::MacroFile(m) => { - let span = - self.db.parse_macro_expansion(m).value.1.span_for_range(token.text_range()); - let range = span.range - + self - .db - .ast_id_map(span.anchor.file_id.into()) - .get_erased(span.anchor.ast_id) - .text_range() - .start(); - FileRange { file_id: span.anchor.file_id, range } + self.db.real_span_map(file_id).span_for_range(token.text_range()) } + base_db::span::HirFileIdRepr::MacroFile(macro_file) => cache + .entry(macro_file) + .or_insert_with(|| macro_file.expansion_info(self.db.upcast())) + .exp_map + .span_at(token.text_range().start()), }; // fetch span information of token in real file, then use that look through expansions of @@ -569,24 +562,21 @@ impl<'db> SemanticsImpl<'db> { // what about things where spans change? Due to being joined etc, that is we don't find the // exact span anymore? + let def_map = sa.resolver.def_map(); let mut stack: SmallVec<[_; 4]> = smallvec![InFile::new(sa.file_id, token)]; - let mut cache = self.expansion_info_cache.borrow_mut(); - let mut mcache = self.macro_call_cache.borrow_mut(); let mut process_expansion_for_token = |stack: &mut SmallVec<_>, macro_file, _token: InFile<&_>| { let expansion_info = cache .entry(macro_file) - .or_insert_with(|| macro_file.expansion_info(self.db.upcast())) - .as_ref()?; + .or_insert_with(|| macro_file.expansion_info(self.db.upcast())); { let InFile { file_id, value } = expansion_info.expanded(); self.cache(value, file_id); } - let mapped_tokens = - expansion_info.map_range_down(self.db.upcast(), absolute_range, None)?; + let mapped_tokens = expansion_info.map_range_down(span, None)?; let len = stack.len(); // requeue the tokens we got from mapping our current token down @@ -599,9 +589,9 @@ impl<'db> SemanticsImpl<'db> { // either due to not being in a macro-call or because its unused push it into the result vec, // otherwise push the remapped tokens back into the queue as they can potentially be remapped again. while let Some(token) = stack.pop() { - self.db.unwind_if_cancelled(); let was_not_remapped = (|| { // First expand into attribute invocations + let containing_attribute_macro_call = self.with_ctx(|ctx| { token.value.parent_ancestors().filter_map(ast::Item::cast).find_map(|item| { if item.attrs().next().is_none() { @@ -612,7 +602,7 @@ impl<'db> SemanticsImpl<'db> { }) }); if let Some(call_id) = containing_attribute_macro_call { - let file_id = call_id.as_file(); + let file_id = call_id.as_macro_file(); return process_expansion_for_token(&mut stack, file_id, token.as_ref()); } @@ -629,7 +619,8 @@ impl<'db> SemanticsImpl<'db> { } if let Some(macro_call) = ast::MacroCall::cast(parent.clone()) { - let mcall = token.with_value(macro_call); + let mcall: hir_expand::files::InFileWrapper = + token.with_value(macro_call); let file_id = match mcache.get(&mcall) { Some(&it) => it, None => { @@ -659,7 +650,7 @@ impl<'db> SemanticsImpl<'db> { match derive_call { Some(call_id) => { // resolved to a derive - let file_id = call_id.as_file(); + let file_id = call_id.as_macro_file(); return process_expansion_for_token( &mut stack, file_id, @@ -698,7 +689,7 @@ impl<'db> SemanticsImpl<'db> { for (.., derive) in helpers.iter().filter(|(helper, ..)| *helper == attr_name) { res = res.or(process_expansion_for_token( &mut stack, - derive.as_file(), + derive.as_macro_file(), token.as_ref(), )); } @@ -1052,7 +1043,7 @@ impl<'db> SemanticsImpl<'db> { fn with_ctx) -> T, T>(&self, f: F) -> T { let mut cache = self.s2d_cache.borrow_mut(); - let mut ctx = SourceToDefCtx { db: self.db, cache: &mut cache }; + let mut ctx = SourceToDefCtx { db: self.db, dynmap_cache: &mut cache }; f(&mut ctx) } diff --git a/crates/hir/src/semantics/source_to_def.rs b/crates/hir/src/semantics/source_to_def.rs index 5b20c873157d..df8c1e904fe8 100644 --- a/crates/hir/src/semantics/source_to_def.rs +++ b/crates/hir/src/semantics/source_to_def.rs @@ -112,7 +112,7 @@ pub(super) type SourceToDefCache = FxHashMap<(ChildContainer, HirFileId), DynMap pub(super) struct SourceToDefCtx<'a, 'b> { pub(super) db: &'b dyn HirDatabase, - pub(super) cache: &'a mut SourceToDefCache, + pub(super) dynmap_cache: &'a mut SourceToDefCache, } impl SourceToDefCtx<'_, '_> { @@ -300,7 +300,7 @@ impl SourceToDefCtx<'_, '_> { fn cache_for(&mut self, container: ChildContainer, file_id: HirFileId) -> &DynMap { let db = self.db; - self.cache + self.dynmap_cache .entry((container, file_id)) .or_insert_with(|| container.child_by_source(db, file_id)) } diff --git a/crates/hir/src/source_analyzer.rs b/crates/hir/src/source_analyzer.rs index 0f1093e6d143..8b1148368961 100644 --- a/crates/hir/src/source_analyzer.rs +++ b/crates/hir/src/source_analyzer.rs @@ -29,7 +29,7 @@ use hir_expand::{ mod_path::path, name, name::{AsName, Name}, - HirFileId, HirFileIdExt, InFile, + HirFileId, HirFileIdExt, InFile, MacroFileId, MacroFileIdExt, }; use hir_ty::{ diagnostics::{ @@ -753,14 +753,15 @@ impl SourceAnalyzer { &self, db: &dyn HirDatabase, macro_call: InFile<&ast::MacroCall>, - ) -> Option { + ) -> Option { let krate = self.resolver.krate(); let macro_call_id = macro_call.as_call_id(db.upcast(), krate, |path| { self.resolver .resolve_path_as_macro(db.upcast(), &path, Some(MacroSubNs::Bang)) .map(|(it, _)| macro_id_to_def_id(db.upcast(), it)) })?; - Some(macro_call_id.as_file()).filter(|it| it.expansion_level(db.upcast()) < 64) + // why the 64? + Some(macro_call_id.as_macro_file()).filter(|it| it.expansion_level(db.upcast()) < 64) } pub(crate) fn resolve_variant( diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index bbf49670ce59..5d802ba86c0c 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -23,7 +23,7 @@ pub trait SpanMapper { impl SpanMapper for TokenMap { fn span_for(&self, range: TextRange) -> S { - self.span_for_range(range) + self.span_at(range.start()) } } @@ -152,8 +152,8 @@ where { let mut map = TokenMap::empty(); node.descendants_with_tokens().filter_map(NodeOrToken::into_token).for_each(|t| { - map.insert( - t.text_range(), + map.push( + t.text_range().start(), SpanData { range: t.text_range() - anchor_offset, anchor, ctx: Ctx::DUMMY }, ); }); @@ -730,15 +730,13 @@ where self.inner.start_node(SyntaxKind::NAME_REF); self.inner.token(SyntaxKind::INT_NUMBER, left); self.inner.finish_node(); - let range = TextRange::at(self.text_pos, TextSize::of(left)); - self.token_map.insert(range, span); + self.token_map.push(self.text_pos + TextSize::of(left), span); // here we move the exit up, the original exit has been deleted in process self.inner.finish_node(); self.inner.token(SyntaxKind::DOT, "."); - let range = TextRange::at(range.end(), TextSize::of(".")); - self.token_map.insert(range, span); + self.token_map.push(self.text_pos + TextSize::of(left) + TextSize::of("."), span); if has_pseudo_dot { assert!(right.is_empty(), "{left}.{right}"); @@ -746,8 +744,7 @@ where assert!(!right.is_empty(), "{left}.{right}"); self.inner.start_node(SyntaxKind::NAME_REF); self.inner.token(SyntaxKind::INT_NUMBER, right); - let range = TextRange::at(range.end(), TextSize::of(right)); - self.token_map.insert(range, span); + self.token_map.push(self.text_pos + TextSize::of(text), span); self.inner.finish_node(); // the parser creates an unbalanced start node, we are required to close it here @@ -772,7 +769,7 @@ where break; } last = self.cursor; - let text: &str = loop { + let (text, span) = loop { break match self.cursor.token_tree() { Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => { // Mark the range if needed @@ -788,19 +785,13 @@ where } tt::Leaf::Literal(lit) => (lit.text.as_str(), lit.span), }; - let range = TextRange::at(self.text_pos, TextSize::of(text)); - self.token_map.insert(range, span); self.cursor = self.cursor.bump(); - text + (text, span) } Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => { self.cursor = self.cursor.subtree().unwrap(); match delim_to_str(subtree.delimiter.kind, false) { - Some(it) => { - let range = TextRange::at(self.text_pos, TextSize::of(it)); - self.token_map.insert(range, subtree.delimiter.open); - it - } + Some(it) => (it, subtree.delimiter.open), None => continue, } } @@ -808,11 +799,7 @@ where let parent = self.cursor.end().unwrap(); self.cursor = self.cursor.bump(); match delim_to_str(parent.delimiter.kind, true) { - Some(it) => { - let range = TextRange::at(self.text_pos, TextSize::of(it)); - self.token_map.insert(range, parent.delimiter.close); - it - } + Some(it) => (it, parent.delimiter.close), None => continue, } } @@ -820,6 +807,7 @@ where }; self.buf += text; self.text_pos += TextSize::of(text); + self.token_map.push(self.text_pos, span); } self.inner.token(kind, self.buf.as_str()); @@ -839,8 +827,8 @@ where // need to add whitespace either. if curr.spacing == tt::Spacing::Alone && curr.char != ';' && next.char != '\'' { self.inner.token(WHITESPACE, " "); - self.token_map.insert(TextRange::at(self.text_pos, TextSize::of(' ')), curr.span); self.text_pos += TextSize::of(' '); + self.token_map.push(self.text_pos, curr.span); } } } diff --git a/crates/mbe/src/token_map.rs b/crates/mbe/src/token_map.rs index 32f61af25e5d..c2ec30ca72fb 100644 --- a/crates/mbe/src/token_map.rs +++ b/crates/mbe/src/token_map.rs @@ -3,7 +3,7 @@ use std::hash::Hash; use stdx::itertools::Itertools; -use syntax::TextRange; +use syntax::{TextRange, TextSize}; use tt::Span; /// Maps absolute text ranges for the corresponding file to the relevant span data. @@ -12,138 +12,46 @@ use tt::Span; pub struct TokenMap { // FIXME: This needs to be sorted by (FileId, AstId) // Then we can do a binary search on the file id, - // then a bin search on the ast id - pub span_map: Vec<(TextRange, S)>, - // span_map2: rustc_hash::FxHashMap, + // then a bin search on the ast id? + spans: Vec<(TextSize, S)>, } impl TokenMap { - pub fn empty() -> Self { - Self { span_map: Vec::new() } + pub(crate) fn empty() -> Self { + Self { spans: Vec::new() } } - pub fn finish(&mut self) { - debug_assert_eq!( - self.span_map - .iter() - .sorted_by_key(|it| (it.0.start(), it.0.end())) - .tuple_windows() - .find(|(range, next)| range.0.end() != next.0.start()), - None, - "span map has holes!" - ); - self.span_map.shrink_to_fit(); + pub(crate) fn finish(&mut self) { + assert!(self.spans.iter().tuple_windows().all(|(a, b)| a.0 < b.0)); + self.spans.shrink_to_fit(); } - pub(crate) fn insert(&mut self, range: TextRange, span: S) { - self.span_map.push((range, span)); + pub(crate) fn push(&mut self, offset: TextSize, span: S) { + self.spans.push((offset, span)); } pub fn ranges_with_span(&self, span: S) -> impl Iterator + '_ { // FIXME: linear search - // FIXME: Disregards resolving spans to get more matches! See ExpansionInfo::map_token_down - self.span_map.iter().filter_map( - move |(range, s)| { - if s == &span { - Some(*range) - } else { - None - } - }, - ) + self.spans.iter().enumerate().filter_map(move |(idx, &(end, s))| { + if s != span { + return None; + } + let start = idx.checked_sub(1).map_or(TextSize::new(0), |prev| self.spans[prev].0); + Some(TextRange::new(start, end)) + }) } // FIXME: We need APIs for fetching the span of a token as well as for a whole node. The node // one *is* fallible though. - // Token span fetching technically only needs an offset really, as the entire file span is - // populated, where node fetching is more like fetching the spans at all source positions, and - // then we need to verify that all those positions have the same context, if not we fail! But - // how do we handle them having different span ranges? - - pub fn span_for_range(&self, range: TextRange) -> S { - // TODO FIXME: make this proper - self.span_map - .iter() - .filter_map(|(r, s)| Some((r, s, r.intersect(range).filter(|it| !it.is_empty())?))) - .max_by_key(|(_, _, intersection)| intersection.len()) - .map_or_else( - || panic!("no span for range {:?} in {:#?}", range, self.span_map), - |(_, &s, _)| s, - ) + pub fn span_at(&self, offset: TextSize) -> S { + let entry = self.spans.partition_point(|&(it, _)| it <= offset); + self.spans[entry].1 } pub fn spans_for_node_range(&self, range: TextRange) -> impl Iterator + '_ { - // TODO FIXME: make this proper - self.span_map - .iter() - .filter(move |(r, _)| r.intersect(range).filter(|it| !it.is_empty()).is_some()) - .map(|&(_, s)| s) + let (start, end) = (range.start(), range.end()); + let start_entry = self.spans.partition_point(|&(it, _)| it <= start); + let end_entry = self.spans[start_entry..].partition_point(|&(it, _)| it <= end); // FIXME: this might be wrong? + (&self.spans[start_entry..][..end_entry]).iter().map(|&(_, s)| s) } - - // pub fn ranges_by_token( - // &self, - // token_id: tt::TokenId, - // kind: SyntaxKind, - // ) -> impl Iterator + '_ { - // self.entries - // .iter() - // .filter(move |&&(tid, _)| tid == token_id) - // .filter_map(move |(_, range)| range.by_kind(kind)) - // } - - // pub(crate) fn remove_delim(&mut self, idx: usize) { - // // FIXME: This could be accidentally quadratic - // self.entries.remove(idx); - // } - - // pub fn entries(&self) -> impl Iterator + '_ { - // self.entries.iter().filter_map(|&(tid, tr)| match tr { - // TokenTextRange::Token(range) => Some((tid, range)), - // TokenTextRange::Delimiter(_) => None, - // }) - // } - - // pub fn filter(&mut self, id: impl Fn(tt::TokenId) -> bool) { - // self.entries.retain(|&(tid, _)| id(tid)); - // } - // pub fn synthetic_token_id(&self, token_id: tt::TokenId) -> Option { - // self.synthetic_entries.iter().find(|(tid, _)| *tid == token_id).map(|(_, id)| *id) - // } - - // pub fn first_range_by_token( - // &self, - // token_id: tt::TokenId, - // kind: SyntaxKind, - // ) -> Option { - // self.ranges_by_token(token_id, kind).next() - // } - - // pub(crate) fn insert(&mut self, token_id: tt::TokenId, relative_range: TextRange) { - // self.entries.push((token_id, TokenTextRange::Token(relative_range))); - // } - - // pub(crate) fn insert_synthetic(&mut self, token_id: tt::TokenId, id: SyntheticTokenId) { - // self.synthetic_entries.push((token_id, id)); - // } - - // pub(crate) fn insert_delim( - // &mut self, - // token_id: tt::TokenId, - // open_relative_range: TextRange, - // close_relative_range: TextRange, - // ) -> usize { - // let res = self.entries.len(); - // let cover = open_relative_range.cover(close_relative_range); - - // self.entries.push((token_id, TokenTextRange::Delimiter(cover))); - // res - // } - - // pub(crate) fn update_close_delim(&mut self, idx: usize, close_relative_range: TextRange) { - // let (_, token_text_range) = &mut self.entries[idx]; - // if let TokenTextRange::Delimiter(dim) = token_text_range { - // let cover = dim.cover(close_relative_range); - // *token_text_range = TokenTextRange::Delimiter(cover); - // } - // } } diff --git a/crates/rust-analyzer/src/integrated_benchmarks.rs b/crates/rust-analyzer/src/integrated_benchmarks.rs index b7e839ac2a5e..a29e18811d88 100644 --- a/crates/rust-analyzer/src/integrated_benchmarks.rs +++ b/crates/rust-analyzer/src/integrated_benchmarks.rs @@ -56,8 +56,7 @@ fn integrated_highlighting_benchmark() { analysis.highlight_as_html(file_id, false).unwrap(); } - profile::init_from("*>100"); - // let _s = profile::heartbeat_span(); + profile::init_from("*>1"); { let _it = stdx::timeit("change"); From b98597f06d34efabe95828f9e6d9161fd608bead Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Tue, 28 Nov 2023 16:28:51 +0100 Subject: [PATCH 14/24] Re-enable proc-macros --- Cargo.lock | 3 + crates/base-db/src/fixture.rs | 12 + crates/base-db/src/input.rs | 3 + crates/base-db/src/span.rs | 2 + .../hir-def/src/macro_expansion_tests/mod.rs | 5 +- crates/hir-expand/src/db.rs | 55 ++- crates/hir-expand/src/fixup.rs | 273 ++++++------- crates/hir-expand/src/proc_macro.rs | 8 +- crates/load-cargo/src/lib.rs | 24 +- crates/mbe/src/expander/transcriber.rs | 2 +- crates/mbe/src/lib.rs | 3 +- crates/mbe/src/syntax_bridge.rs | 105 +++-- crates/proc-macro-api/Cargo.toml | 7 +- crates/proc-macro-api/src/lib.rs | 39 +- crates/proc-macro-api/src/msg.rs | 127 ++++-- crates/proc-macro-api/src/msg/flat.rs | 381 +++++++++--------- crates/proc-macro-srv-cli/src/main.rs | 6 +- crates/proc-macro-srv/src/dylib.rs | 10 +- crates/proc-macro-srv/src/lib.rs | 38 +- crates/proc-macro-srv/src/proc_macros.rs | 39 +- crates/proc-macro-srv/src/server.rs | 47 ++- .../proc-macro-srv/src/server/token_stream.rs | 64 +-- crates/proc-macro-srv/src/tests/utils.rs | 25 +- .../src/integrated_benchmarks.rs | 2 +- 24 files changed, 787 insertions(+), 493 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 775231f3ea2e..c71ca2f21260 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1255,6 +1255,9 @@ checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" name = "proc-macro-api" version = "0.0.0" dependencies = [ + "base-db", + "indexmap", + "la-arena 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "memmap2", "object 0.32.0", "paths", diff --git a/crates/base-db/src/fixture.rs b/crates/base-db/src/fixture.rs index 7236b56f6d47..cfba01a03298 100644 --- a/crates/base-db/src/fixture.rs +++ b/crates/base-db/src/fixture.rs @@ -543,6 +543,9 @@ impl ProcMacroExpander for IdentityProcMacroExpander { subtree: &Subtree, _: Option<&Subtree>, _: &Env, + _: SpanData, + _: SpanData, + _: SpanData, ) -> Result, ProcMacroExpansionError> { Ok(subtree.clone()) } @@ -557,6 +560,9 @@ impl ProcMacroExpander for AttributeInputReplaceProcMacroExpander { _: &Subtree, attrs: Option<&Subtree>, _: &Env, + _: SpanData, + _: SpanData, + _: SpanData, ) -> Result, ProcMacroExpansionError> { attrs .cloned() @@ -572,6 +578,9 @@ impl ProcMacroExpander for MirrorProcMacroExpander { input: &Subtree, _: Option<&Subtree>, _: &Env, + _: SpanData, + _: SpanData, + _: SpanData, ) -> Result, ProcMacroExpansionError> { fn traverse(input: &Subtree) -> Subtree { let mut token_trees = vec![]; @@ -599,6 +608,9 @@ impl ProcMacroExpander for ShortenProcMacroExpander { input: &Subtree, _: Option<&Subtree>, _: &Env, + _: SpanData, + _: SpanData, + _: SpanData, ) -> Result, ProcMacroExpansionError> { return Ok(traverse(input)); diff --git a/crates/base-db/src/input.rs b/crates/base-db/src/input.rs index 0b04a91f626b..12b449932dd7 100644 --- a/crates/base-db/src/input.rs +++ b/crates/base-db/src/input.rs @@ -262,6 +262,9 @@ pub trait ProcMacroExpander: fmt::Debug + Send + Sync + RefUnwindSafe { subtree: &tt::Subtree, attrs: Option<&tt::Subtree>, env: &Env, + def_site: SpanData, + call_site: SpanData, + mixed_site: SpanData, ) -> Result, ProcMacroExpansionError>; } diff --git a/crates/base-db/src/span.rs b/crates/base-db/src/span.rs index 607b8027ca95..d2b40ecdd214 100644 --- a/crates/base-db/src/span.rs +++ b/crates/base-db/src/span.rs @@ -38,6 +38,8 @@ impl SyntaxContextId { // currently (which kind of makes sense but we need it here!) pub const SELF_REF: Self = SyntaxContextId(unsafe { InternId::new_unchecked(InternId::MAX - 1) }); + /// Used syntax fixups + pub const FAKE: Self = SyntaxContextId(unsafe { InternId::new_unchecked(InternId::MAX - 2) }); pub fn is_root(self) -> bool { self == Self::ROOT diff --git a/crates/hir-def/src/macro_expansion_tests/mod.rs b/crates/hir-def/src/macro_expansion_tests/mod.rs index 355b82a5f42c..bcbf4047caa2 100644 --- a/crates/hir-def/src/macro_expansion_tests/mod.rs +++ b/crates/hir-def/src/macro_expansion_tests/mod.rs @@ -16,7 +16,7 @@ mod proc_macros; use std::{iter, ops::Range, sync}; -use base_db::{fixture::WithFixture, ProcMacro, SourceDatabase}; +use base_db::{fixture::WithFixture, span::SpanData, ProcMacro, SourceDatabase}; use expect_test::Expect; use hir_expand::{db::ExpandDatabase, span::SpanMapRef, HirFileIdExt, InFile, MacroFileId}; use stdx::format_to; @@ -307,6 +307,9 @@ impl base_db::ProcMacroExpander for IdentityWhenValidProcMacroExpander { subtree: &Subtree, _: Option<&Subtree>, _: &base_db::Env, + _: SpanData, + _: SpanData, + _: SpanData, ) -> Result { let (parse, _) = ::mbe::token_tree_to_syntax_node(subtree, ::mbe::TopEntryPoint::MacroItems); diff --git a/crates/hir-expand/src/db.rs b/crates/hir-expand/src/db.rs index 601a754abbd4..1d55aaf1703f 100644 --- a/crates/hir-expand/src/db.rs +++ b/crates/hir-expand/src/db.rs @@ -233,7 +233,17 @@ pub fn expand_speculative( let speculative_expansion = match loc.def.kind { MacroDefKind::ProcMacro(expander, ..) => { tt.delimiter = tt::Delimiter::UNSPECIFIED; - expander.expand(db, loc.def.krate, loc.krate, &tt, attr_arg.as_ref()) + let call_site = loc.span(db); + expander.expand( + db, + loc.def.krate, + loc.krate, + &tt, + attr_arg.as_ref(), + call_site, + call_site, + call_site, + ) } MacroDefKind::BuiltInAttr(BuiltinAttrExpander::Derive, _) => { pseudo_derive_attr_expansion(&tt, attr_arg.as_ref()?) @@ -398,17 +408,23 @@ fn macro_arg( MacroCallKind::Attr { ast_id, .. } => ast_id.to_ptr(db).to_node(&root).syntax().clone(), }; let censor = censor_for_macro_input(&loc, &syntax); - // let mut fixups = fixup::fixup_syntax(&node); - // fixups.replace.extend(censor.into_iter().map(|node| (node.into(), Vec::new()))); - // let (mut tt, tmap, _) = mbe::syntax_node_to_token_tree_with_modifications( - // &node, - // fixups.token_map, - // fixups.next_id, - // fixups.replace, - // fixups.append, - // ); - - let mut tt = mbe::syntax_node_to_token_tree_censored(&syntax, map.as_ref(), censor); + let mut tt = match loc.kind { + MacroCallKind::FnLike { .. } => { + mbe::syntax_node_to_token_tree_censored(&syntax, map.as_ref(), censor) + } + MacroCallKind::Derive { .. } | MacroCallKind::Attr { .. } => { + // let mut fixups = crate::fixup::fixup_syntax(&syntax); + // fixups.replace.extend(censor.into_iter().map(|node| (node.into(), Vec::new()))); + // let (mut tt, tmap, _) = mbe::syntax_node_to_token_tree_with_modifications( + // &node, + // fixups.token_map, + // fixups.next_id, + // fixups.replace, + // fixups.append, + // ); + mbe::syntax_node_to_token_tree_censored(&syntax, map.as_ref(), censor) + } + }; if loc.def.is_proc_macro() { // proc macros expect their inputs without parentheses, MBEs expect it with them included @@ -658,8 +674,19 @@ fn expand_proc_macro(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult None, }; - let ExpandResult { value: tt, err } = - expander.expand(db, loc.def.krate, loc.krate, ¯o_arg, attr_arg); + let call_site = loc.span(db); + let ExpandResult { value: tt, err } = expander.expand( + db, + loc.def.krate, + loc.krate, + ¯o_arg, + attr_arg, + // FIXME + call_site, + call_site, + // FIXME + call_site, + ); // Set a hard limit for the expanded tt if let Err(value) = check_tt_count(&tt) { diff --git a/crates/hir-expand/src/fixup.rs b/crates/hir-expand/src/fixup.rs index e6e8d8c02992..326ff1dec484 100644 --- a/crates/hir-expand/src/fixup.rs +++ b/crates/hir-expand/src/fixup.rs @@ -2,25 +2,30 @@ //! fix up syntax errors in the code we're passing to them. use std::mem; -use mbe::{SyntheticToken, SyntheticTokenId, TokenMap}; +use base_db::{ + span::{ErasedFileAstId, SpanAnchor, SpanData, SyntaxContextId}, + FileId, +}; +use la_arena::RawIdx; +use mbe::TokenMap; use rustc_hash::FxHashMap; use smallvec::SmallVec; use syntax::{ ast::{self, AstNode, HasLoopBody}, - match_ast, SyntaxElement, SyntaxKind, SyntaxNode, TextRange, + match_ast, SyntaxElement, SyntaxKind, SyntaxNode, TextRange, TextSize, }; -use tt::token_id::Subtree; +use tt::Spacing; + +use crate::tt::{Ident, Leaf, Punct, Subtree}; /// The result of calculating fixes for a syntax node -- a bunch of changes /// (appending to and replacing nodes), the information that is needed to /// reverse those changes afterwards, and a token map. #[derive(Debug, Default)] pub(crate) struct SyntaxFixups { - pub(crate) append: FxHashMap>, - pub(crate) replace: FxHashMap>, + pub(crate) append: FxHashMap>, + pub(crate) replace: FxHashMap>, pub(crate) undo_info: SyntaxFixupUndoInfo, - pub(crate) token_map: TokenMap, - pub(crate) next_id: u32, } /// This is the information needed to reverse the fixups. @@ -29,21 +34,25 @@ pub struct SyntaxFixupUndoInfo { original: Box<[Subtree]>, } -const EMPTY_ID: SyntheticTokenId = SyntheticTokenId(!0); +// censoring -> just don't convert the node +// replacement -> censor + append +// append -> insert a fake node, here we need to assemble some dummy span that we can figure out how +// to remove later pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { let mut append = FxHashMap::::default(); let mut replace = FxHashMap::::default(); let mut preorder = node.preorder(); let mut original = Vec::new(); - let mut token_map = TokenMap::default(); - let mut next_id = 0; + let dummy_range = TextRange::empty(TextSize::new(0)); + let dummy_anchor = + SpanAnchor { file_id: FileId(!0), ast_id: ErasedFileAstId::from_raw(RawIdx::from(0)) }; + let fake_span = + SpanData { range: dummy_range, anchor: dummy_anchor, ctx: SyntaxContextId::FAKE }; while let Some(event) = preorder.next() { - let node = match event { - syntax::WalkEvent::Enter(node) => node, - syntax::WalkEvent::Leave(_) => continue, - }; + let syntax::WalkEvent::Enter(node) = event else { continue }; + /* TODO if can_handle_error(&node) && has_error_to_handle(&node) { // the node contains an error node, we have to completely replace it by something valid let (original_tree, new_tmap, new_next_id) = @@ -68,6 +77,7 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { preorder.skip_subtree(); continue; } + */ // In some other situations, we can fix things by just appending some tokens. let end_range = TextRange::empty(node.text_range().end()); match_ast! { @@ -76,36 +86,32 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { if it.name_ref().is_none() { // incomplete field access: some_expr.| append.insert(node.clone().into(), vec![ - SyntheticToken { - kind: SyntaxKind::IDENT, + Leaf::Ident(Ident { text: "__ra_fixup".into(), - range: end_range, - id: EMPTY_ID, - }, + span: fake_span + }), ]); } }, ast::ExprStmt(it) => { if it.semicolon_token().is_none() { append.insert(node.clone().into(), vec![ - SyntheticToken { - kind: SyntaxKind::SEMICOLON, - text: ";".into(), - range: end_range, - id: EMPTY_ID, - }, + Leaf::Punct(Punct { + char: ';', + spacing: Spacing::Alone, + span: fake_span + }), ]); } }, ast::LetStmt(it) => { if it.semicolon_token().is_none() { append.insert(node.clone().into(), vec![ - SyntheticToken { - kind: SyntaxKind::SEMICOLON, - text: ";".into(), - range: end_range, - id: EMPTY_ID, - }, + Leaf::Punct(Punct { + char: ';', + spacing: Spacing::Alone, + span: fake_span + }), ]); } }, @@ -117,28 +123,25 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { None => continue, }; append.insert(if_token.into(), vec![ - SyntheticToken { - kind: SyntaxKind::IDENT, + Leaf::Ident(Ident { text: "__ra_fixup".into(), - range: end_range, - id: EMPTY_ID, - }, + span: fake_span + }), ]); } if it.then_branch().is_none() { append.insert(node.clone().into(), vec![ - SyntheticToken { - kind: SyntaxKind::L_CURLY, - text: "{".into(), - range: end_range, - id: EMPTY_ID, - }, - SyntheticToken { - kind: SyntaxKind::R_CURLY, - text: "}".into(), - range: end_range, - id: EMPTY_ID, - }, + // FIXME: THis should be a subtree no? + Leaf::Punct(Punct { + char: '{', + spacing: Spacing::Alone, + span: fake_span + }), + Leaf::Punct(Punct { + char: '}', + spacing: Spacing::Alone, + span: fake_span + }), ]); } }, @@ -150,46 +153,42 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { None => continue, }; append.insert(while_token.into(), vec![ - SyntheticToken { - kind: SyntaxKind::IDENT, + Leaf::Ident(Ident { text: "__ra_fixup".into(), - range: end_range, - id: EMPTY_ID, - }, + span: fake_span + }), ]); } if it.loop_body().is_none() { append.insert(node.clone().into(), vec![ - SyntheticToken { - kind: SyntaxKind::L_CURLY, - text: "{".into(), - range: end_range, - id: EMPTY_ID, - }, - SyntheticToken { - kind: SyntaxKind::R_CURLY, - text: "}".into(), - range: end_range, - id: EMPTY_ID, - }, + // FIXME: THis should be a subtree no? + Leaf::Punct(Punct { + char: '{', + spacing: Spacing::Alone, + span: fake_span + }), + Leaf::Punct(Punct { + char: '}', + spacing: Spacing::Alone, + span: fake_span + }), ]); } }, ast::LoopExpr(it) => { if it.loop_body().is_none() { append.insert(node.clone().into(), vec![ - SyntheticToken { - kind: SyntaxKind::L_CURLY, - text: "{".into(), - range: end_range, - id: EMPTY_ID, - }, - SyntheticToken { - kind: SyntaxKind::R_CURLY, - text: "}".into(), - range: end_range, - id: EMPTY_ID, - }, + // FIXME: THis should be a subtree no? + Leaf::Punct(Punct { + char: '{', + spacing: Spacing::Alone, + span: fake_span + }), + Leaf::Punct(Punct { + char: '}', + spacing: Spacing::Alone, + span: fake_span + }), ]); } }, @@ -201,29 +200,26 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { None => continue }; append.insert(match_token.into(), vec![ - SyntheticToken { - kind: SyntaxKind::IDENT, + Leaf::Ident(Ident { text: "__ra_fixup".into(), - range: end_range, - id: EMPTY_ID - }, + span: fake_span + }), ]); } if it.match_arm_list().is_none() { // No match arms append.insert(node.clone().into(), vec![ - SyntheticToken { - kind: SyntaxKind::L_CURLY, - text: "{".into(), - range: end_range, - id: EMPTY_ID, - }, - SyntheticToken { - kind: SyntaxKind::R_CURLY, - text: "}".into(), - range: end_range, - id: EMPTY_ID, - }, + // FIXME: THis should be a subtree no? + Leaf::Punct(Punct { + char: '{', + spacing: Spacing::Alone, + span: fake_span + }), + Leaf::Punct(Punct { + char: '}', + spacing: Spacing::Alone, + span: fake_span + }), ]); } }, @@ -234,10 +230,15 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { }; let [pat, in_token, iter] = [ - (SyntaxKind::UNDERSCORE, "_"), - (SyntaxKind::IN_KW, "in"), - (SyntaxKind::IDENT, "__ra_fixup") - ].map(|(kind, text)| SyntheticToken { kind, text: text.into(), range: end_range, id: EMPTY_ID}); + "_", + "in", + "__ra_fixup" + ].map(|text| + Leaf::Ident(Ident { + text: text.into(), + span: fake_span + }), + ); if it.pat().is_none() && it.in_token().is_none() && it.iterable().is_none() { append.insert(for_token.into(), vec![pat, in_token, iter]); @@ -248,18 +249,17 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { if it.loop_body().is_none() { append.insert(node.clone().into(), vec![ - SyntheticToken { - kind: SyntaxKind::L_CURLY, - text: "{".into(), - range: end_range, - id: EMPTY_ID, - }, - SyntheticToken { - kind: SyntaxKind::R_CURLY, - text: "}".into(), - range: end_range, - id: EMPTY_ID, - }, + // FIXME: THis should be a subtree no? + Leaf::Punct(Punct { + char: '{', + spacing: Spacing::Alone, + span: fake_span + }), + Leaf::Punct(Punct { + char: '}', + spacing: Spacing::Alone, + span: fake_span + }), ]); } }, @@ -270,8 +270,6 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { SyntaxFixups { append, replace, - token_map, - next_id, undo_info: SyntaxFixupUndoInfo { original: original.into_boxed_slice() }, } } @@ -288,40 +286,33 @@ fn has_error_to_handle(node: &SyntaxNode) -> bool { has_error(node) || node.children().any(|c| !can_handle_error(&c) && has_error_to_handle(&c)) } -pub(crate) fn reverse_fixups( - tt: &mut Subtree, - token_map: &TokenMap, - undo_info: &SyntaxFixupUndoInfo, -) { +pub(crate) fn reverse_fixups(tt: &mut Subtree, undo_info: &SyntaxFixupUndoInfo) { let tts = std::mem::take(&mut tt.token_trees); tt.token_trees = tts .into_iter() + // delete all fake nodes .filter(|tt| match tt { - tt::TokenTree::Leaf(leaf) => { - token_map.synthetic_token_id(*leaf.span()) != Some(EMPTY_ID) - } - tt::TokenTree::Subtree(st) => { - token_map.synthetic_token_id(st.delimiter.open) != Some(EMPTY_ID) - } - }) - .flat_map(|tt| match tt { - tt::TokenTree::Subtree(mut tt) => { - reverse_fixups(&mut tt, token_map, undo_info); - SmallVec::from_const([tt.into()]) - } - tt::TokenTree::Leaf(leaf) => { - if let Some(id) = token_map.synthetic_token_id(*leaf.span()) { - let original = undo_info.original[id.0 as usize].clone(); - if original.delimiter.kind == tt::DelimiterKind::Invisible { - original.token_trees.into() - } else { - SmallVec::from_const([original.into()]) - } - } else { - SmallVec::from_const([leaf.into()]) - } - } + tt::TokenTree::Leaf(leaf) => leaf.span().ctx != SyntaxContextId::FAKE, + tt::TokenTree::Subtree(st) => st.delimiter.open.ctx != SyntaxContextId::FAKE, }) + // .flat_map(|tt| match tt { TODO + // tt::TokenTree::Subtree(mut tt) => { + // reverse_fixups(&mut tt, undo_info); + // SmallVec::from_const([tt.into()]) + // } + // tt::TokenTree::Leaf(leaf) => { + // if let Some(id) = leaf.span().anchor { + // let original = undo_info.original[id.0 as usize].clone(); + // if original.delimiter.kind == tt::DelimiterKind::Invisible { + // original.token_trees.into() + // } else { + // SmallVec::from_const([original.into()]) + // } + // } else { + // SmallVec::from_const([leaf.into()]) + // } + // } + // }) .collect(); } diff --git a/crates/hir-expand/src/proc_macro.rs b/crates/hir-expand/src/proc_macro.rs index 41675c630dcf..04b5b7b0b6a3 100644 --- a/crates/hir-expand/src/proc_macro.rs +++ b/crates/hir-expand/src/proc_macro.rs @@ -1,6 +1,6 @@ //! Proc Macro Expander stub -use base_db::{CrateId, ProcMacroExpansionError, ProcMacroId, ProcMacroKind}; +use base_db::{span::SpanData, CrateId, ProcMacroExpansionError, ProcMacroId, ProcMacroKind}; use stdx::never; use crate::{db::ExpandDatabase, tt, ExpandError, ExpandResult}; @@ -33,6 +33,9 @@ impl ProcMacroExpander { calling_crate: CrateId, tt: &tt::Subtree, attr_arg: Option<&tt::Subtree>, + def_site: SpanData, + call_site: SpanData, + mixed_site: SpanData, ) -> ExpandResult { match self.proc_macro_id { ProcMacroId(DUMMY_ID) => { @@ -68,7 +71,8 @@ impl ProcMacroExpander { let krate_graph = db.crate_graph(); // Proc macros have access to the environment variables of the invoking crate. let env = &krate_graph[calling_crate].env; - match proc_macro.expander.expand(tt, attr_arg, env) { + match proc_macro.expander.expand(tt, attr_arg, env, def_site, call_site, mixed_site) + { Ok(t) => ExpandResult::ok(t), Err(err) => match err { // Don't discard the item in case something unexpected happened while expanding attributes diff --git a/crates/load-cargo/src/lib.rs b/crates/load-cargo/src/lib.rs index 4d1319094933..ed4175c45830 100644 --- a/crates/load-cargo/src/lib.rs +++ b/crates/load-cargo/src/lib.rs @@ -376,16 +376,16 @@ impl ProcMacroExpander for Expander { subtree: &tt::Subtree, attrs: Option<&tt::Subtree>, env: &Env, + def_site: SpanData, + call_site: SpanData, + mixed_site: SpanData, ) -> Result, ProcMacroExpansionError> { - let _ = (subtree, attrs, env); - - // let env = env.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(); - // match self.0.expand(subtree, attrs, env) { - // Ok(Ok(subtree)) => Ok(subtree), - // Ok(Err(err)) => Err(ProcMacroExpansionError::Panic(err.0)), - // Err(err) => Err(ProcMacroExpansionError::System(err.to_string())), - // } - todo!() + let env = env.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(); + match self.0.expand(subtree, attrs, env, def_site, call_site, mixed_site) { + Ok(Ok(subtree)) => Ok(subtree), + Ok(Err(err)) => Err(ProcMacroExpansionError::Panic(err.0)), + Err(err) => Err(ProcMacroExpansionError::System(err.to_string())), + } } } @@ -399,6 +399,9 @@ impl ProcMacroExpander for IdentityExpander { subtree: &tt::Subtree, _: Option<&tt::Subtree>, _: &Env, + _: SpanData, + _: SpanData, + _: SpanData, ) -> Result, ProcMacroExpansionError> { Ok(subtree.clone()) } @@ -414,6 +417,9 @@ impl ProcMacroExpander for EmptyExpander { _: &tt::Subtree, _: Option<&tt::Subtree>, _: &Env, + _: SpanData, + _: SpanData, + _: SpanData, ) -> Result, ProcMacroExpansionError> { Ok(tt::Subtree::empty()) } diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs index c8b326fa6c53..8aedd7314063 100644 --- a/crates/mbe/src/expander/transcriber.rs +++ b/crates/mbe/src/expander/transcriber.rs @@ -51,7 +51,7 @@ impl Bindings { marker(&mut span); let subtree = tt::Subtree { delimiter: tt::Delimiter { - // TODO split span + // FIXME split span open: span, close: span, kind: delimiter.kind, diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index 1b13b39f0177..fdf97ad538cb 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -34,7 +34,8 @@ pub use tt::{Delimiter, DelimiterKind, Punct, SyntaxContext}; pub use crate::{ syntax_bridge::{ - map_from_syntax_node, parse_exprs_with_sep, parse_to_token_tree, syntax_node_to_token_tree, + map_from_syntax_node, parse_exprs_with_sep, parse_to_token_tree, + parse_to_token_tree_static_span, syntax_node_to_token_tree, syntax_node_to_token_tree_censored, token_tree_to_syntax_node, SpanMapper, }, token_map::TokenMap, diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 5d802ba86c0c..c61c52628607 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -63,7 +63,7 @@ pub(crate) mod dummy_test_span_utils { /// Convert the syntax node to a `TokenTree` (what macro /// will consume). -/// TODO: Flesh out the doc comment more thoroughly +/// FIXME: Flesh out the doc comment more thoroughly pub fn syntax_node_to_token_tree( node: &SyntaxNode, map: SpanMap, @@ -179,6 +179,19 @@ where Some(convert_tokens(&mut conv)) } +/// Convert a string to a `TokenTree` +pub fn parse_to_token_tree_static_span(span: S, text: &str) -> Option> +where + S: Span, +{ + let lexed = parser::LexedStr::new(text); + if lexed.errors().next().is_some() { + return None; + } + let mut conv = StaticRawConverter { lexed, pos: 0, span }; + Some(convert_tokens(&mut conv)) +} + /// Split token tree with separate expr: $($e:expr)SEP* pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec> { if tt.token_trees.is_empty() { @@ -213,12 +226,10 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec(conv: &mut C) -> tt::Subtree> +fn convert_tokens(conv: &mut C) -> tt::Subtree where - C: TokenConverter, - Ctx: SyntaxContext, - SpanData: Span, - Anchor: Copy, + C: TokenConverter, + S: Span, { let entry = tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: vec![] }; let mut stack = NonEmptyVec::new(entry); @@ -452,6 +463,12 @@ struct RawConverter<'a, Anchor> { pos: usize, anchor: Anchor, } +/// A raw token (straight from lexer) converter that gives every token the same span. +struct StaticRawConverter<'a, S> { + lexed: parser::LexedStr<'a>, + pos: usize, + span: S, +} trait SrcToken: std::fmt::Debug { fn kind(&self, ctx: &Ctx) -> SyntaxKind; @@ -461,20 +478,16 @@ trait SrcToken: std::fmt::Debug { fn to_text(&self, ctx: &Ctx) -> SmolStr; } -trait TokenConverter: Sized { +trait TokenConverter: Sized { type Token: SrcToken; - fn convert_doc_comment( - &self, - token: &Self::Token, - span: SpanData, - ) -> Option>>>; + fn convert_doc_comment(&self, token: &Self::Token, span: S) -> Option>>; fn bump(&mut self) -> Option<(Self::Token, TextRange)>; fn peek(&self) -> Option; - fn span_for(&self, range: TextRange) -> SpanData; + fn span_for(&self, range: TextRange) -> S; } impl SrcToken> for usize { @@ -491,7 +504,22 @@ impl SrcToken> for usize { } } -impl TokenConverter for RawConverter<'_, Anchor> +impl SrcToken> for usize { + fn kind(&self, ctx: &StaticRawConverter<'_, S>) -> SyntaxKind { + ctx.lexed.kind(*self) + } + + fn to_char(&self, ctx: &StaticRawConverter<'_, S>) -> Option { + ctx.lexed.text(*self).chars().next() + } + + fn to_text(&self, ctx: &StaticRawConverter<'_, S>) -> SmolStr { + ctx.lexed.text(*self).into() + } +} + +impl TokenConverter> + for RawConverter<'_, Anchor> where SpanData: Span, { @@ -530,6 +558,41 @@ where } } +impl TokenConverter for StaticRawConverter<'_, S> +where + S: Span, +{ + type Token = usize; + + fn convert_doc_comment(&self, &token: &usize, span: S) -> Option>> { + let text = self.lexed.text(token); + convert_doc_comment(&doc_comment(text), span) + } + + fn bump(&mut self) -> Option<(Self::Token, TextRange)> { + if self.pos == self.lexed.len() { + return None; + } + let token = self.pos; + self.pos += 1; + let range = self.lexed.text_range(token); + let range = TextRange::new(range.start.try_into().ok()?, range.end.try_into().ok()?); + + Some((token, range)) + } + + fn peek(&self) -> Option { + if self.pos == self.lexed.len() { + return None; + } + Some(self.pos) + } + + fn span_for(&self, _: TextRange) -> S { + self.span + } +} + struct Converter { current: Option, preorder: PreorderWithTokens, @@ -596,17 +659,13 @@ impl SrcToken> for SynToken { } } -impl TokenConverter for Converter +impl TokenConverter for Converter where - SpanData: Span, - SpanMap: SpanMapper>, + S: Span, + SpanMap: SpanMapper, { type Token = SynToken; - fn convert_doc_comment( - &self, - token: &Self::Token, - span: SpanData, - ) -> Option>>> { + fn convert_doc_comment(&self, token: &Self::Token, span: S) -> Option>> { convert_doc_comment(token.token(), span) } @@ -661,7 +720,7 @@ where Some(token) } - fn span_for(&self, range: TextRange) -> SpanData { + fn span_for(&self, range: TextRange) -> S { self.map.span_for(range) } } diff --git a/crates/proc-macro-api/Cargo.toml b/crates/proc-macro-api/Cargo.toml index 4c87c89adde9..2cbbc9489a29 100644 --- a/crates/proc-macro-api/Cargo.toml +++ b/crates/proc-macro-api/Cargo.toml @@ -25,6 +25,7 @@ tracing.workspace = true triomphe.workspace = true memmap2 = "0.5.4" snap = "1.1.0" +indexmap = "2.1.0" # local deps paths.workspace = true @@ -32,5 +33,7 @@ tt.workspace = true stdx.workspace = true profile.workspace = true text-size.workspace = true -# Intentionally *not* depend on anything salsa-related -# base-db.workspace = true +# Ideally this crate would not depend on salsa things, but we need span information here which wraps +# InternIds for the syntax context +base-db.workspace = true +la-arena.workspace = true diff --git a/crates/proc-macro-api/src/lib.rs b/crates/proc-macro-api/src/lib.rs index 7a3580f814e2..9fc5a53d935f 100644 --- a/crates/proc-macro-api/src/lib.rs +++ b/crates/proc-macro-api/src/lib.rs @@ -11,6 +11,8 @@ pub mod msg; mod process; mod version; +use base_db::span::SpanData; +use indexmap::IndexSet; use paths::AbsPathBuf; use std::{fmt, io, sync::Mutex}; use triomphe::Arc; @@ -18,7 +20,7 @@ use triomphe::Arc; use serde::{Deserialize, Serialize}; use crate::{ - msg::{flat::SerializableSpan, ExpandMacro, FlatTree, PanicMessage}, + msg::{ExpandMacro, ExpnGlobals, FlatTree, PanicMessage, HAS_GLOBAL_SPANS}, process::ProcMacroProcessSrv, }; @@ -132,32 +134,49 @@ impl ProcMacro { self.kind } - pub fn expand>( + pub fn expand( &self, - subtree: &tt::Subtree, - attr: Option<&tt::Subtree>, + subtree: &tt::Subtree, + attr: Option<&tt::Subtree>, env: Vec<(String, String)>, - ) -> Result, PanicMessage>, ServerError> { + def_site: SpanData, + call_site: SpanData, + mixed_site: SpanData, + ) -> Result, PanicMessage>, ServerError> { let version = self.process.lock().unwrap_or_else(|e| e.into_inner()).version(); let current_dir = env .iter() .find(|(name, _)| name == "CARGO_MANIFEST_DIR") .map(|(_, value)| value.clone()); + let mut span_data_table = IndexSet::default(); + let def_site = span_data_table.insert_full(def_site).0; + let call_site = span_data_table.insert_full(call_site).0; + let mixed_site = span_data_table.insert_full(mixed_site).0; let task = ExpandMacro { - macro_body: FlatTree::new(subtree, version), + macro_body: FlatTree::new(subtree, version, &mut span_data_table), macro_name: self.name.to_string(), - attributes: attr.map(|subtree| FlatTree::new(subtree, version)), + attributes: attr.map(|subtree| FlatTree::new(subtree, version, &mut span_data_table)), lib: self.dylib_path.to_path_buf().into(), env, current_dir, + has_global_spans: ExpnGlobals { + serialize: version >= HAS_GLOBAL_SPANS, + def_site, + call_site, + mixed_site, + }, }; - let request = msg::Request::ExpandMacro(task); - let response = self.process.lock().unwrap_or_else(|e| e.into_inner()).send_task(request)?; + let response = self + .process + .lock() + .unwrap_or_else(|e| e.into_inner()) + .send_task(msg::Request::ExpandMacro(task))?; + match response { msg::Response::ExpandMacro(it) => { - Ok(it.map(|tree| FlatTree::to_subtree(tree, version))) + Ok(it.map(|tree| FlatTree::to_subtree_resolved(tree, version, &span_data_table))) } msg::Response::ListMacros(..) | msg::Response::ApiVersionCheck(..) => { Err(ServerError { message: "unexpected response".to_string(), io: None }) diff --git a/crates/proc-macro-api/src/msg.rs b/crates/proc-macro-api/src/msg.rs index 4e6984f61b72..ddac514ff709 100644 --- a/crates/proc-macro-api/src/msg.rs +++ b/crates/proc-macro-api/src/msg.rs @@ -10,21 +10,15 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize}; use crate::ProcMacroKind; -pub use crate::msg::flat::FlatTree; +pub use crate::msg::flat::{FlatTree, TokenId}; // The versions of the server protocol pub const NO_VERSION_CHECK_VERSION: u32 = 0; pub const VERSION_CHECK_VERSION: u32 = 1; pub const ENCODE_CLOSE_SPAN_VERSION: u32 = 2; -/// This version changes how spans are encoded, kind of. Prior to this version, -/// spans were represented as a single u32 which effectively forced spans to be -/// token ids. Starting with this version, the span fields are still u32, -/// but if the size of the span is greater than 1 then the span data is encoded in -/// an additional vector where the span represents the offset into that vector. -/// This allows encoding bigger spans while supporting the previous versions. -pub const VARIABLE_SIZED_SPANS: u32 = 2; +pub const HAS_GLOBAL_SPANS: u32 = 3; -pub const CURRENT_API_VERSION: u32 = VARIABLE_SIZED_SPANS; +pub const CURRENT_API_VERSION: u32 = HAS_GLOBAL_SPANS; #[derive(Debug, Serialize, Deserialize)] pub enum Request { @@ -66,6 +60,24 @@ pub struct ExpandMacro { pub env: Vec<(String, String)>, pub current_dir: Option, + /// marker for serde skip stuff + #[serde(skip_serializing_if = "ExpnGlobals::skip_serializing_if")] + pub has_global_spans: ExpnGlobals, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ExpnGlobals { + #[serde(skip_serializing)] + pub serialize: bool, + pub def_site: usize, + pub call_site: usize, + pub mixed_site: usize, +} + +impl ExpnGlobals { + fn skip_serializing_if(&self) -> bool { + !self.serialize + } } pub trait Message: Serialize + DeserializeOwned { @@ -120,38 +132,89 @@ fn write_json(out: &mut impl Write, msg: &str) -> io::Result<()> { Ok(()) } -/*TODO - #[cfg(test)] mod tests { - use tt::{ - Delimiter, DelimiterKind, Ident, Leaf, Literal, Punct, Spacing, SpanAnchor, Subtree, - TokenId, TokenTree, + use base_db::{ + span::{ErasedFileAstId, SpanAnchor, SpanData, SyntaxContextId}, + FileId, }; + use la_arena::RawIdx; + use text_size::{TextRange, TextSize}; + use tt::{Delimiter, DelimiterKind, Ident, Leaf, Literal, Punct, Spacing, Subtree, TokenTree}; use super::*; - fn fixture_token_tree() -> Subtree { - let mut subtree = Subtree { delimiter: Delimiter::unspecified(), token_trees: Vec::new() }; - subtree - .token_trees - .push(TokenTree::Leaf(Ident { text: "struct".into(), span: TokenId(0) }.into())); - subtree - .token_trees - .push(TokenTree::Leaf(Ident { text: "Foo".into(), span: TokenId(1) }.into())); + fn fixture_token_tree() -> Subtree { + let anchor = + SpanAnchor { file_id: FileId(0), ast_id: ErasedFileAstId::from_raw(RawIdx::from(0)) }; + let mut subtree = Subtree { + delimiter: Delimiter { + open: SpanData { + range: TextRange::empty(TextSize::new(0)), + anchor, + ctx: SyntaxContextId::ROOT, + }, + close: SpanData { + range: TextRange::empty(TextSize::new(13)), + anchor, + ctx: SyntaxContextId::ROOT, + }, + kind: DelimiterKind::Invisible, + }, + token_trees: Vec::new(), + }; + subtree.token_trees.push(TokenTree::Leaf( + Ident { + text: "struct".into(), + span: SpanData { + range: TextRange::at(TextSize::new(0), TextSize::of("struct")), + anchor, + ctx: SyntaxContextId::ROOT, + }, + } + .into(), + )); + subtree.token_trees.push(TokenTree::Leaf( + Ident { + text: "Foo".into(), + span: SpanData { + range: TextRange::at(TextSize::new(5), TextSize::of("Foo")), + anchor, + ctx: SyntaxContextId::ROOT, + }, + } + .into(), + )); subtree.token_trees.push(TokenTree::Leaf(Leaf::Literal(Literal { text: "Foo".into(), - span: TokenId::DUMMY, + + span: SpanData { + range: TextRange::at(TextSize::new(8), TextSize::of("Foo")), + anchor, + ctx: SyntaxContextId::ROOT, + }, }))); subtree.token_trees.push(TokenTree::Leaf(Leaf::Punct(Punct { char: '@', - span: TokenId::DUMMY, + span: SpanData { + range: TextRange::at(TextSize::new(11), TextSize::of('@')), + anchor, + ctx: SyntaxContextId::ROOT, + }, spacing: Spacing::Joint, }))); subtree.token_trees.push(TokenTree::Subtree(Subtree { delimiter: Delimiter { - open: TokenId(2), - close: TokenId::DUMMY, + open: SpanData { + range: TextRange::at(TextSize::new(12), TextSize::of('{')), + anchor, + ctx: SyntaxContextId::ROOT, + }, + close: SpanData { + range: TextRange::at(TextSize::new(13), TextSize::of('}')), + anchor, + ctx: SyntaxContextId::ROOT, + }, kind: DelimiterKind::Brace, }, token_trees: vec![], @@ -162,20 +225,26 @@ mod tests { #[test] fn test_proc_macro_rpc_works() { let tt = fixture_token_tree(); + let mut span_data_table = Default::default(); let task = ExpandMacro { - macro_body: FlatTree::new(&tt, CURRENT_API_VERSION), + macro_body: FlatTree::new(&tt, CURRENT_API_VERSION, &mut span_data_table), macro_name: Default::default(), attributes: None, lib: std::env::current_dir().unwrap(), env: Default::default(), current_dir: Default::default(), + has_global_spans: ExpnGlobals { + serialize: true, + def_site: 0, + call_site: 0, + mixed_site: 0, + }, }; let json = serde_json::to_string(&task).unwrap(); // println!("{}", json); let back: ExpandMacro = serde_json::from_str(&json).unwrap(); - assert_eq!(tt, back.macro_body.to_subtree(CURRENT_API_VERSION)); + assert_eq!(tt, back.macro_body.to_subtree_resolved(CURRENT_API_VERSION, &span_data_table)); } } -*/ diff --git a/crates/proc-macro-api/src/msg/flat.rs b/crates/proc-macro-api/src/msg/flat.rs index f29aac529568..baf8bbad4bf2 100644 --- a/crates/proc-macro-api/src/msg/flat.rs +++ b/crates/proc-macro-api/src/msg/flat.rs @@ -37,40 +37,40 @@ use std::collections::{HashMap, VecDeque}; +use base_db::span::SpanData; +use indexmap::IndexSet; use serde::{Deserialize, Serialize}; -use text_size::TextRange; -use tt::{Span, SyntaxContext}; -use crate::msg::{ENCODE_CLOSE_SPAN_VERSION, VARIABLE_SIZED_SPANS}; +use crate::msg::ENCODE_CLOSE_SPAN_VERSION; -pub trait SerializableSpan: Span { - fn into_u32(self) -> [u32; L]; - fn from_u32(input: [u32; L]) -> Self; +pub type SpanDataIndexMap = IndexSet; + +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct TokenId(pub u32); + +impl std::fmt::Debug for TokenId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +impl TokenId { + pub const DEF_SITE: Self = TokenId(0); + pub const CALL_SITE: Self = TokenId(0); + pub const MIXED_SITE: Self = TokenId(0); } -// impl SerializableSpan<1> for tt::TokenId { -// fn into_u32(self) -> [u32; 1] { -// [self.0] -// } -// fn from_u32([input]: [u32; 1]) -> Self { -// tt::TokenId(input) -// } -// } - -impl SerializableSpan<3> for tt::SpanData -where - Anchor: From + Into, - Self: Span, - Ctx: SyntaxContext, -{ - fn into_u32(self) -> [u32; 3] { - [self.anchor.into(), self.range.start().into(), self.range.end().into()] + +impl tt::Span for TokenId { + const DUMMY: Self = TokenId(!0); + + type Anchor = (); + + fn anchor(self) -> Self::Anchor { + () } - fn from_u32([file_id, start, end]: [u32; 3]) -> Self { - tt::SpanData { - anchor: file_id.into(), - range: TextRange::new(start.into(), end.into()), - ctx: Ctx::DUMMY, - } + + fn mk(_: Self::Anchor, _: text_size::TextRange) -> Self { + Self::DUMMY } } @@ -82,82 +82,41 @@ pub struct FlatTree { ident: Vec, token_tree: Vec, text: Vec, - #[serde(skip_serializing_if = "SpanMap::do_serialize")] - #[serde(default)] - span_map: SpanMap, -} - -#[derive(Serialize, Deserialize, Debug)] -struct SpanMap { - #[serde(skip_serializing)] - serialize: bool, - span_size: u32, - spans: Vec, -} - -impl Default for SpanMap { - fn default() -> Self { - Self { serialize: false, span_size: 1, spans: Default::default() } - } -} - -impl SpanMap { - fn serialize_span>(&mut self, span: S) -> u32 { - let u32s = span.into_u32(); - if L == 1 { - u32s[0] - } else { - let offset = self.spans.len() as u32; - self.spans.extend(u32s); - offset - } - } - fn deserialize_span>(&self, offset: u32) -> S { - S::from_u32(if L == 1 { - [offset].as_ref().try_into().unwrap() - } else { - self.spans[offset as usize..][..L].try_into().unwrap() - }) - } } -impl SpanMap { - fn do_serialize(&self) -> bool { - self.serialize - } -} - -struct SubtreeRepr { - open: S, - close: S, +struct SubtreeRepr { + open: TokenId, + close: TokenId, kind: tt::DelimiterKind, tt: [u32; 2], } -struct LiteralRepr { - id: S, +struct LiteralRepr { + id: TokenId, text: u32, } -struct PunctRepr { - id: S, +struct PunctRepr { + id: TokenId, char: char, spacing: tt::Spacing, } -struct IdentRepr { - id: S, +struct IdentRepr { + id: TokenId, text: u32, } impl FlatTree { - pub fn new>( - subtree: &tt::Subtree, + pub fn new( + subtree: &tt::Subtree, version: u32, + span_data_table: &mut SpanDataIndexMap, ) -> FlatTree { let mut w = Writer { string_table: HashMap::new(), work: VecDeque::new(), + span_data_table, subtree: Vec::new(), literal: Vec::new(), @@ -167,78 +126,111 @@ impl FlatTree { text: Vec::new(), }; w.write(subtree); - assert!(L == 1 || version >= VARIABLE_SIZED_SPANS); - let mut span_map = SpanMap { - serialize: version >= VARIABLE_SIZED_SPANS && L != 1, - span_size: L as u32, - spans: Vec::new(), - }; - return FlatTree { + + FlatTree { subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - write_vec(&mut span_map, w.subtree, SubtreeRepr::write_with_close_span) + write_vec(w.subtree, SubtreeRepr::write_with_close_span) } else { - write_vec(&mut span_map, w.subtree, SubtreeRepr::write) + write_vec(w.subtree, SubtreeRepr::write) }, - literal: write_vec(&mut span_map, w.literal, LiteralRepr::write), - punct: write_vec(&mut span_map, w.punct, PunctRepr::write), - ident: write_vec(&mut span_map, w.ident, IdentRepr::write), + literal: write_vec(w.literal, LiteralRepr::write), + punct: write_vec(w.punct, PunctRepr::write), + ident: write_vec(w.ident, IdentRepr::write), token_tree: w.token_tree, text: w.text, - span_map, + } + } + + pub fn new_raw(subtree: &tt::Subtree, version: u32) -> FlatTree { + let mut w = Writer { + string_table: HashMap::new(), + work: VecDeque::new(), + span_data_table: &mut (), + + subtree: Vec::new(), + literal: Vec::new(), + punct: Vec::new(), + ident: Vec::new(), + token_tree: Vec::new(), + text: Vec::new(), }; + w.write(subtree); - fn write_vec [u32; N], const N: usize>( - map: &mut SpanMap, - xs: Vec, - f: F, - ) -> Vec { - xs.into_iter().flat_map(|it| f(it, map)).collect() + FlatTree { + subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { + write_vec(w.subtree, SubtreeRepr::write_with_close_span) + } else { + write_vec(w.subtree, SubtreeRepr::write) + }, + literal: write_vec(w.literal, LiteralRepr::write), + punct: write_vec(w.punct, PunctRepr::write), + ident: write_vec(w.ident, IdentRepr::write), + token_tree: w.token_tree, + text: w.text, } } - pub fn to_subtree>( + pub fn to_subtree_resolved( self, version: u32, - ) -> tt::Subtree { - assert!((version >= VARIABLE_SIZED_SPANS || L == 1) && L as u32 == self.span_map.span_size); - return Reader { + span_data_table: &SpanDataIndexMap, + ) -> tt::Subtree { + Reader { subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - read_vec(&self.span_map, self.subtree, SubtreeRepr::read_with_close_span) + read_vec(self.subtree, SubtreeRepr::read_with_close_span) } else { - read_vec(&self.span_map, self.subtree, SubtreeRepr::read) + read_vec(self.subtree, SubtreeRepr::read) }, - literal: read_vec(&self.span_map, self.literal, LiteralRepr::read), - punct: read_vec(&self.span_map, self.punct, PunctRepr::read), - ident: read_vec(&self.span_map, self.ident, IdentRepr::read), + literal: read_vec(self.literal, LiteralRepr::read), + punct: read_vec(self.punct, PunctRepr::read), + ident: read_vec(self.ident, IdentRepr::read), token_tree: self.token_tree, text: self.text, + span_data_table, } - .read(); - - fn read_vec T, const N: usize>( - map: &SpanMap, - xs: Vec, - f: F, - ) -> Vec { - let mut chunks = xs.chunks_exact(N); - let res = chunks.by_ref().map(|chunk| f(chunk.try_into().unwrap(), map)).collect(); - assert!(chunks.remainder().is_empty()); - res + .read() + } + + pub fn to_subtree_unresolved(self, version: u32) -> tt::Subtree { + Reader { + subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { + read_vec(self.subtree, SubtreeRepr::read_with_close_span) + } else { + read_vec(self.subtree, SubtreeRepr::read) + }, + literal: read_vec(self.literal, LiteralRepr::read), + punct: read_vec(self.punct, PunctRepr::read), + ident: read_vec(self.ident, IdentRepr::read), + token_tree: self.token_tree, + text: self.text, + span_data_table: &(), } + .read() } } -impl> SubtreeRepr { - fn write(self, map: &mut SpanMap) -> [u32; 4] { +fn read_vec T, const N: usize>(xs: Vec, f: F) -> Vec { + let mut chunks = xs.chunks_exact(N); + let res = chunks.by_ref().map(|chunk| f(chunk.try_into().unwrap())).collect(); + assert!(chunks.remainder().is_empty()); + res +} + +fn write_vec [u32; N], const N: usize>(xs: Vec, f: F) -> Vec { + xs.into_iter().flat_map(f).collect() +} + +impl SubtreeRepr { + fn write(self) -> [u32; 4] { let kind = match self.kind { tt::DelimiterKind::Invisible => 0, tt::DelimiterKind::Parenthesis => 1, tt::DelimiterKind::Brace => 2, tt::DelimiterKind::Bracket => 3, }; - [map.serialize_span(self.open), kind, self.tt[0], self.tt[1]] + [self.open.0, kind, self.tt[0], self.tt[1]] } - fn read([open, kind, lo, len]: [u32; 4], map: &SpanMap) -> Self { + fn read([open, kind, lo, len]: [u32; 4]) -> SubtreeRepr { let kind = match kind { 0 => tt::DelimiterKind::Invisible, 1 => tt::DelimiterKind::Parenthesis, @@ -246,24 +238,18 @@ impl> SubtreeRepr { 3 => tt::DelimiterKind::Bracket, other => panic!("bad kind {other}"), }; - SubtreeRepr { open: map.deserialize_span(open), close: S::DUMMY, kind, tt: [lo, len] } + SubtreeRepr { open: TokenId(open), close: TokenId(!0), kind, tt: [lo, len] } } - fn write_with_close_span(self, map: &mut SpanMap) -> [u32; 5] { + fn write_with_close_span(self) -> [u32; 5] { let kind = match self.kind { tt::DelimiterKind::Invisible => 0, tt::DelimiterKind::Parenthesis => 1, tt::DelimiterKind::Brace => 2, tt::DelimiterKind::Bracket => 3, }; - [ - map.serialize_span(self.open), - map.serialize_span(self.close), - kind, - self.tt[0], - self.tt[1], - ] + [self.open.0, self.close.0, kind, self.tt[0], self.tt[1]] } - fn read_with_close_span([open, close, kind, lo, len]: [u32; 5], map: &SpanMap) -> Self { + fn read_with_close_span([open, close, kind, lo, len]: [u32; 5]) -> SubtreeRepr { let kind = match kind { 0 => tt::DelimiterKind::Invisible, 1 => tt::DelimiterKind::Parenthesis, @@ -271,64 +257,86 @@ impl> SubtreeRepr { 3 => tt::DelimiterKind::Bracket, other => panic!("bad kind {other}"), }; - SubtreeRepr { - open: map.deserialize_span(open), - close: map.deserialize_span(close), - kind, - tt: [lo, len], - } + SubtreeRepr { open: TokenId(open), close: TokenId(close), kind, tt: [lo, len] } } } -impl> LiteralRepr { - fn write(self, map: &mut SpanMap) -> [u32; 2] { - [map.serialize_span(self.id), self.text] +impl LiteralRepr { + fn write(self) -> [u32; 2] { + [self.id.0, self.text] } - fn read([id, text]: [u32; 2], map: &SpanMap) -> Self { - LiteralRepr { id: map.deserialize_span(id), text } + fn read([id, text]: [u32; 2]) -> LiteralRepr { + LiteralRepr { id: TokenId(id), text } } } -impl> PunctRepr { - fn write(self, map: &mut SpanMap) -> [u32; 3] { +impl PunctRepr { + fn write(self) -> [u32; 3] { let spacing = match self.spacing { tt::Spacing::Alone => 0, tt::Spacing::Joint => 1, }; - [map.serialize_span(self.id), self.char as u32, spacing] + [self.id.0, self.char as u32, spacing] } - fn read([id, char, spacing]: [u32; 3], map: &SpanMap) -> Self { + fn read([id, char, spacing]: [u32; 3]) -> PunctRepr { let spacing = match spacing { 0 => tt::Spacing::Alone, 1 => tt::Spacing::Joint, other => panic!("bad spacing {other}"), }; - PunctRepr { id: map.deserialize_span(id), char: char.try_into().unwrap(), spacing } + PunctRepr { id: TokenId(id), char: char.try_into().unwrap(), spacing } } } -impl> IdentRepr { - fn write(self, map: &mut SpanMap) -> [u32; 2] { - [map.serialize_span(self.id), self.text] +impl IdentRepr { + fn write(self) -> [u32; 2] { + [self.id.0, self.text] } - fn read(data: [u32; 2], map: &SpanMap) -> Self { - IdentRepr { id: map.deserialize_span(data[0]), text: data[1] } + fn read(data: [u32; 2]) -> IdentRepr { + IdentRepr { id: TokenId(data[0]), text: data[1] } } } -struct Writer<'a, const L: usize, S> { +trait Span: Copy { + type Table; + fn token_id_of(table: &mut Self::Table, s: Self) -> TokenId; + fn span_for_token_id(table: &Self::Table, id: TokenId) -> Self; +} + +impl Span for TokenId { + type Table = (); + fn token_id_of((): &mut Self::Table, token_id: Self) -> TokenId { + token_id + } + + fn span_for_token_id((): &Self::Table, id: TokenId) -> Self { + id + } +} +impl Span for SpanData { + type Table = IndexSet; + fn token_id_of(table: &mut Self::Table, span: Self) -> TokenId { + TokenId(table.insert_full(span).0 as u32) + } + fn span_for_token_id(table: &Self::Table, id: TokenId) -> Self { + *table.get_index(id.0 as usize).unwrap_or_else(|| &table[0]) + } +} + +struct Writer<'a, 'span, S: Span> { work: VecDeque<(usize, &'a tt::Subtree)>, string_table: HashMap<&'a str, u32>, + span_data_table: &'span mut S::Table, - subtree: Vec>, - literal: Vec>, - punct: Vec>, - ident: Vec>, + subtree: Vec, + literal: Vec, + punct: Vec, + ident: Vec, token_tree: Vec, text: Vec, } -impl<'a, const L: usize, S: Copy> Writer<'a, L, S> { +impl<'a, 'span, S: Span> Writer<'a, 'span, S> { fn write(&mut self, root: &'a tt::Subtree) { self.enqueue(root); while let Some((idx, subtree)) = self.work.pop_front() { @@ -336,6 +344,10 @@ impl<'a, const L: usize, S: Copy> Writer<'a, L, S> { } } + fn token_id_of(&mut self, span: S) -> TokenId { + S::token_id_of(self.span_data_table, span) + } + fn subtree(&mut self, idx: usize, subtree: &'a tt::Subtree) { let mut first_tt = self.token_tree.len(); let n_tt = subtree.token_trees.len(); @@ -353,22 +365,21 @@ impl<'a, const L: usize, S: Copy> Writer<'a, L, S> { tt::Leaf::Literal(lit) => { let idx = self.literal.len() as u32; let text = self.intern(&lit.text); - self.literal.push(LiteralRepr { id: lit.span, text }); + let id = self.token_id_of(lit.span); + self.literal.push(LiteralRepr { id, text }); idx << 2 | 0b01 } tt::Leaf::Punct(punct) => { let idx = self.punct.len() as u32; - self.punct.push(PunctRepr { - char: punct.char, - spacing: punct.spacing, - id: punct.span, - }); + let id = self.token_id_of(punct.span); + self.punct.push(PunctRepr { char: punct.char, spacing: punct.spacing, id }); idx << 2 | 0b10 } tt::Leaf::Ident(ident) => { let idx = self.ident.len() as u32; let text = self.intern(&ident.text); - self.ident.push(IdentRepr { id: ident.span, text }); + let id = self.token_id_of(ident.span); + self.ident.push(IdentRepr { id, text }); idx << 2 | 0b11 } }, @@ -380,8 +391,8 @@ impl<'a, const L: usize, S: Copy> Writer<'a, L, S> { fn enqueue(&mut self, subtree: &'a tt::Subtree) -> u32 { let idx = self.subtree.len(); - let open = subtree.delimiter.open; - let close = subtree.delimiter.close; + let open = self.token_id_of(subtree.delimiter.open); + let close = self.token_id_of(subtree.delimiter.close); let delimiter_kind = subtree.delimiter.kind; self.subtree.push(SubtreeRepr { open, close, kind: delimiter_kind, tt: [!0, !0] }); self.work.push_back((idx, subtree)); @@ -398,23 +409,29 @@ impl<'a, const L: usize, S: Copy> Writer<'a, L, S> { } } -struct Reader { - subtree: Vec>, - literal: Vec>, - punct: Vec>, - ident: Vec>, +struct Reader<'span, S: Span> { + subtree: Vec, + literal: Vec, + punct: Vec, + ident: Vec, token_tree: Vec, text: Vec, + span_data_table: &'span S::Table, } -impl> Reader { +impl<'span, S: Span> Reader<'span, S> { pub(crate) fn read(self) -> tt::Subtree { let mut res: Vec>> = vec![None; self.subtree.len()]; + let read_span = |id| S::span_for_token_id(self.span_data_table, id); for i in (0..self.subtree.len()).rev() { let repr = &self.subtree[i]; let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize]; let s = tt::Subtree { - delimiter: tt::Delimiter { open: repr.open, close: repr.close, kind: repr.kind }, + delimiter: tt::Delimiter { + open: read_span(repr.open), + close: read_span(repr.close), + kind: repr.kind, + }, token_trees: token_trees .iter() .copied() @@ -429,7 +446,7 @@ impl> Reader { let repr = &self.literal[idx]; tt::Leaf::Literal(tt::Literal { text: self.text[repr.text as usize].as_str().into(), - span: repr.id, + span: read_span(repr.id), }) .into() } @@ -438,7 +455,7 @@ impl> Reader { tt::Leaf::Punct(tt::Punct { char: repr.char, spacing: repr.spacing, - span: repr.id, + span: read_span(repr.id), }) .into() } @@ -446,7 +463,7 @@ impl> Reader { let repr = &self.ident[idx]; tt::Leaf::Ident(tt::Ident { text: self.text[repr.text as usize].as_str().into(), - span: repr.id, + span: read_span(repr.id), }) .into() } diff --git a/crates/proc-macro-srv-cli/src/main.rs b/crates/proc-macro-srv-cli/src/main.rs index ea65c33604f8..50ce586fc429 100644 --- a/crates/proc-macro-srv-cli/src/main.rs +++ b/crates/proc-macro-srv-cli/src/main.rs @@ -18,14 +18,12 @@ fn main() -> std::io::Result<()> { run() } -#[cfg(not(FALSE))] -#[cfg(not(feature = "sysroot-abi"))] +#[cfg(not(any(feature = "sysroot-abi", rust_analyzer)))] fn run() -> io::Result<()> { panic!("proc-macro-srv-cli requires the `sysroot-abi` feature to be enabled"); } -#[cfg(FALSE)] -#[cfg(feature = "sysroot-abi")] +#[cfg(any(feature = "sysroot-abi", rust_analyzer))] fn run() -> io::Result<()> { use proc_macro_api::msg::{self, Message}; diff --git a/crates/proc-macro-srv/src/dylib.rs b/crates/proc-macro-srv/src/dylib.rs index dd05e250c2de..80bce3af1a00 100644 --- a/crates/proc-macro-srv/src/dylib.rs +++ b/crates/proc-macro-srv/src/dylib.rs @@ -11,7 +11,7 @@ use libloading::Library; use memmap2::Mmap; use object::Object; use paths::AbsPath; -use proc_macro_api::{read_dylib_info, ProcMacroKind}; +use proc_macro_api::{msg::TokenId, read_dylib_info, ProcMacroKind}; const NEW_REGISTRAR_SYMBOL: &str = "_rustc_proc_macro_decls_"; @@ -152,8 +152,14 @@ impl Expander { macro_name: &str, macro_body: &crate::tt::Subtree, attributes: Option<&crate::tt::Subtree>, + def_site: TokenId, + call_site: TokenId, + mixed_site: TokenId, ) -> Result { - let result = self.inner.proc_macros.expand(macro_name, macro_body, attributes); + let result = self + .inner + .proc_macros + .expand(macro_name, macro_body, attributes, def_site, call_site, mixed_site); result.map_err(|e| e.as_str().unwrap_or_else(|| "".to_string())) } diff --git a/crates/proc-macro-srv/src/lib.rs b/crates/proc-macro-srv/src/lib.rs index bd0d1b79fa1f..32a07a84777b 100644 --- a/crates/proc-macro-srv/src/lib.rs +++ b/crates/proc-macro-srv/src/lib.rs @@ -10,7 +10,6 @@ //! * By **copying** the whole rustc `lib_proc_macro` code, we are able to build this with `stable` //! rustc rather than `unstable`. (Although in general ABI compatibility is still an issue)… -#![cfg(FALSE)] // TODO #![cfg(any(feature = "sysroot-abi", rust_analyzer))] #![feature(proc_macro_internals, proc_macro_diagnostic, proc_macro_span)] #![warn(rust_2018_idioms, unused_lifetimes, semicolon_in_expressions_from_macros)] @@ -32,12 +31,25 @@ use std::{ time::SystemTime, }; +use ::tt::Span; use proc_macro_api::{ - msg::{self, CURRENT_API_VERSION}, + msg::{self, ExpnGlobals, TokenId, CURRENT_API_VERSION, HAS_GLOBAL_SPANS}, ProcMacroKind, }; -use ::tt::token_id as tt; +mod tt { + pub use proc_macro_api::msg::TokenId; + + pub use ::tt::*; + + pub type Subtree = ::tt::Subtree; + pub type TokenTree = ::tt::TokenTree; + pub type Delimiter = ::tt::Delimiter; + pub type Leaf = ::tt::Leaf; + pub type Literal = ::tt::Literal; + pub type Punct = ::tt::Punct; + pub type Ident = ::tt::Ident; +} // see `build.rs` include!(concat!(env!("OUT_DIR"), "/rustc_version.rs")); @@ -71,16 +83,28 @@ impl ProcMacroSrv { None => None, }; - let macro_body = task.macro_body.to_subtree(CURRENT_API_VERSION); - let attributes = task.attributes.map(|it| it.to_subtree(CURRENT_API_VERSION)); + let ExpnGlobals { def_site, call_site, mixed_site, .. } = task.has_global_spans; + let def_site = TokenId(def_site as u32); + let call_site = TokenId(call_site as u32); + let mixed_site = TokenId(mixed_site as u32); + + let macro_body = task.macro_body.to_subtree_unresolved(CURRENT_API_VERSION); + let attributes = task.attributes.map(|it| it.to_subtree_unresolved(CURRENT_API_VERSION)); let result = thread::scope(|s| { let thread = thread::Builder::new() .stack_size(EXPANDER_STACK_SIZE) .name(task.macro_name.clone()) .spawn_scoped(s, || { expander - .expand(&task.macro_name, ¯o_body, attributes.as_ref()) - .map(|it| msg::FlatTree::new(&it, CURRENT_API_VERSION)) + .expand( + &task.macro_name, + ¯o_body, + attributes.as_ref(), + def_site, + call_site, + mixed_site, + ) + .map(|it| msg::FlatTree::new_raw(&it, CURRENT_API_VERSION)) }); let res = match thread { Ok(handle) => handle.join(), diff --git a/crates/proc-macro-srv/src/proc_macros.rs b/crates/proc-macro-srv/src/proc_macros.rs index 3c6f32033192..4f87fa281b77 100644 --- a/crates/proc-macro-srv/src/proc_macros.rs +++ b/crates/proc-macro-srv/src/proc_macros.rs @@ -1,7 +1,7 @@ //! Proc macro ABI use libloading::Library; -use proc_macro_api::{ProcMacroKind, RustCInfo}; +use proc_macro_api::{msg::TokenId, ProcMacroKind, RustCInfo}; use crate::{dylib::LoadProcMacroDylibError, server::SYMBOL_INTERNER, tt}; @@ -45,6 +45,9 @@ impl ProcMacros { macro_name: &str, macro_body: &tt::Subtree, attributes: Option<&tt::Subtree>, + def_site: TokenId, + call_site: TokenId, + mixed_site: TokenId, ) -> Result { let parsed_body = crate::server::TokenStream::with_subtree(macro_body.clone()); @@ -59,34 +62,56 @@ impl ProcMacros { } if *trait_name == macro_name => { let res = client.run( &proc_macro::bridge::server::SameThread, - crate::server::RustAnalyzer { interner: &SYMBOL_INTERNER }, + crate::server::RustAnalyzer { + interner: &SYMBOL_INTERNER, + call_site, + def_site, + mixed_site, + }, parsed_body, true, ); - return res.map(|it| it.into_subtree()).map_err(crate::PanicMessage::from); + return res + .map(|it| it.into_subtree(call_site)) + .map_err(crate::PanicMessage::from); } proc_macro::bridge::client::ProcMacro::Bang { name, client } if *name == macro_name => { let res = client.run( &proc_macro::bridge::server::SameThread, - crate::server::RustAnalyzer { interner: &SYMBOL_INTERNER }, + crate::server::RustAnalyzer { + interner: &SYMBOL_INTERNER, + call_site, + def_site, + mixed_site, + }, parsed_body, true, ); - return res.map(|it| it.into_subtree()).map_err(crate::PanicMessage::from); + return res + .map(|it| it.into_subtree(call_site)) + .map_err(crate::PanicMessage::from); } proc_macro::bridge::client::ProcMacro::Attr { name, client } if *name == macro_name => { let res = client.run( &proc_macro::bridge::server::SameThread, - crate::server::RustAnalyzer { interner: &SYMBOL_INTERNER }, + crate::server::RustAnalyzer { + interner: &SYMBOL_INTERNER, + + call_site, + def_site, + mixed_site, + }, parsed_attributes, parsed_body, true, ); - return res.map(|it| it.into_subtree()).map_err(crate::PanicMessage::from); + return res + .map(|it| it.into_subtree(call_site)) + .map_err(crate::PanicMessage::from); } _ => continue, } diff --git a/crates/proc-macro-srv/src/server.rs b/crates/proc-macro-srv/src/server.rs index fe18451d3848..fc080eccc0f4 100644 --- a/crates/proc-macro-srv/src/server.rs +++ b/crates/proc-macro-srv/src/server.rs @@ -11,6 +11,7 @@ use proc_macro::bridge::{self, server}; mod token_stream; +use proc_macro_api::msg::TokenId; pub use token_stream::TokenStream; use token_stream::TokenStreamBuilder; @@ -43,6 +44,9 @@ pub struct FreeFunctions; pub struct RustAnalyzer { // FIXME: store span information here. pub(crate) interner: SymbolInternerRef, + pub call_site: TokenId, + pub def_site: TokenId, + pub mixed_site: TokenId, } impl server::Types for RustAnalyzer { @@ -69,7 +73,7 @@ impl server::FreeFunctions for RustAnalyzer { kind: bridge::LitKind::Err, symbol: Symbol::intern(self.interner, s), suffix: None, - span: tt::TokenId::unspecified(), + span: self.call_site, }) } @@ -83,7 +87,7 @@ impl server::TokenStream for RustAnalyzer { stream.is_empty() } fn from_str(&mut self, src: &str) -> Self::TokenStream { - src.parse().expect("cannot parse string") + Self::TokenStream::from_str(src, self.call_site).expect("cannot parse string") } fn to_string(&mut self, stream: &Self::TokenStream) -> String { stream.to_string() @@ -280,7 +284,7 @@ impl server::Span for RustAnalyzer { } fn recover_proc_macro_span(&mut self, _id: usize) -> Self::Span { // FIXME stub - tt::TokenId::unspecified() + self.call_site } /// Recent feature, not yet in the proc_macro /// @@ -317,15 +321,15 @@ impl server::Span for RustAnalyzer { } fn resolved_at(&mut self, _span: Self::Span, _at: Self::Span) -> Self::Span { // FIXME handle span - tt::TokenId::unspecified() + self.call_site } fn end(&mut self, _self_: Self::Span) -> Self::Span { - tt::TokenId::unspecified() + self.call_site } fn start(&mut self, _self_: Self::Span) -> Self::Span { - tt::TokenId::unspecified() + self.call_site } fn line(&mut self, _span: Self::Span) -> usize { @@ -349,9 +353,9 @@ impl server::Symbol for RustAnalyzer { impl server::Server for RustAnalyzer { fn globals(&mut self) -> bridge::ExpnGlobals { bridge::ExpnGlobals { - def_site: Span::unspecified(), - call_site: Span::unspecified(), - mixed_site: Span::unspecified(), + def_site: self.def_site, + call_site: self.call_site, + mixed_site: self.mixed_site, } } @@ -422,6 +426,8 @@ impl LiteralFormatter { #[cfg(test)] mod tests { + use ::tt::Span; + use super::*; #[test] @@ -430,16 +436,16 @@ mod tests { token_trees: vec![ tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: "struct".into(), - span: tt::TokenId::unspecified(), + span: tt::TokenId::DUMMY, })), tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: "T".into(), - span: tt::TokenId::unspecified(), + span: tt::TokenId::DUMMY, })), tt::TokenTree::Subtree(tt::Subtree { delimiter: tt::Delimiter { - open: tt::TokenId::unspecified(), - close: tt::TokenId::unspecified(), + open: tt::TokenId::DUMMY, + close: tt::TokenId::DUMMY, kind: tt::DelimiterKind::Brace, }, token_trees: vec![], @@ -452,33 +458,32 @@ mod tests { #[test] fn test_ra_server_from_str() { - use std::str::FromStr; let subtree_paren_a = tt::TokenTree::Subtree(tt::Subtree { delimiter: tt::Delimiter { - open: tt::TokenId::unspecified(), - close: tt::TokenId::unspecified(), + open: tt::TokenId::DUMMY, + close: tt::TokenId::DUMMY, kind: tt::DelimiterKind::Parenthesis, }, token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: "a".into(), - span: tt::TokenId::unspecified(), + span: tt::TokenId::DUMMY, }))], }); - let t1 = TokenStream::from_str("(a)").unwrap(); + let t1 = TokenStream::from_str("(a)", tt::TokenId::DUMMY).unwrap(); assert_eq!(t1.token_trees.len(), 1); assert_eq!(t1.token_trees[0], subtree_paren_a); - let t2 = TokenStream::from_str("(a);").unwrap(); + let t2 = TokenStream::from_str("(a);", tt::TokenId::DUMMY).unwrap(); assert_eq!(t2.token_trees.len(), 2); assert_eq!(t2.token_trees[0], subtree_paren_a); - let underscore = TokenStream::from_str("_").unwrap(); + let underscore = TokenStream::from_str("_", tt::TokenId::DUMMY).unwrap(); assert_eq!( underscore.token_trees[0], tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: "_".into(), - span: tt::TokenId::unspecified(), + span: tt::TokenId::DUMMY, })) ); } diff --git a/crates/proc-macro-srv/src/server/token_stream.rs b/crates/proc-macro-srv/src/server/token_stream.rs index 2589d8b64d48..36be88250388 100644 --- a/crates/proc-macro-srv/src/server/token_stream.rs +++ b/crates/proc-macro-srv/src/server/token_stream.rs @@ -1,5 +1,7 @@ //! TokenStream implementation used by sysroot ABI +use proc_macro_api::msg::TokenId; + use crate::tt::{self, TokenTree}; #[derive(Debug, Default, Clone)] @@ -20,8 +22,15 @@ impl TokenStream { } } - pub(crate) fn into_subtree(self) -> tt::Subtree { - tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: self.token_trees } + pub(crate) fn into_subtree(self, call_site: TokenId) -> tt::Subtree { + tt::Subtree { + delimiter: tt::Delimiter { + open: call_site, + close: call_site, + kind: tt::DelimiterKind::Invisible, + }, + token_trees: self.token_trees, + } } pub(super) fn is_empty(&self) -> bool { @@ -84,7 +93,7 @@ pub(super) struct TokenStreamBuilder { /// pub(super)lic implementation details for the `TokenStream` type, such as iterators. pub(super) mod token_stream { - use std::str::FromStr; + use proc_macro_api::msg::TokenId; use super::{tt, TokenStream, TokenTree}; @@ -109,14 +118,15 @@ pub(super) mod token_stream { /// /// NOTE: some errors may cause panics instead of returning `LexError`. We reserve the right to /// change these errors into `LexError`s later. - impl FromStr for TokenStream { - type Err = LexError; + #[rustfmt::skip] + impl /*FromStr for*/ TokenStream { + // type Err = LexError; - fn from_str(src: &str) -> Result { - let (subtree, _token_map) = - mbe::parse_to_token_tree(src).ok_or("Failed to parse from mbe")?; + pub(crate) fn from_str(src: &str, call_site: TokenId) -> Result { + let subtree = + mbe::parse_to_token_tree_static_span(call_site, src).ok_or("Failed to parse from mbe")?; - let subtree = subtree_replace_token_ids_with_unspecified(subtree); + let subtree = subtree_replace_token_ids_with_call_site(subtree,call_site); Ok(TokenStream::with_subtree(subtree)) } } @@ -127,43 +137,39 @@ pub(super) mod token_stream { } } - fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree { + fn subtree_replace_token_ids_with_call_site( + subtree: tt::Subtree, + call_site: TokenId, + ) -> tt::Subtree { tt::Subtree { - delimiter: tt::Delimiter { - open: tt::TokenId::UNSPECIFIED, - close: tt::TokenId::UNSPECIFIED, - ..subtree.delimiter - }, + delimiter: tt::Delimiter { open: call_site, close: call_site, ..subtree.delimiter }, token_trees: subtree .token_trees .into_iter() - .map(token_tree_replace_token_ids_with_unspecified) + .map(|it| token_tree_replace_token_ids_with_call_site(it, call_site)) .collect(), } } - fn token_tree_replace_token_ids_with_unspecified(tt: tt::TokenTree) -> tt::TokenTree { + fn token_tree_replace_token_ids_with_call_site( + tt: tt::TokenTree, + call_site: TokenId, + ) -> tt::TokenTree { match tt { tt::TokenTree::Leaf(leaf) => { - tt::TokenTree::Leaf(leaf_replace_token_ids_with_unspecified(leaf)) + tt::TokenTree::Leaf(leaf_replace_token_ids_with_call_site(leaf, call_site)) } tt::TokenTree::Subtree(subtree) => { - tt::TokenTree::Subtree(subtree_replace_token_ids_with_unspecified(subtree)) + tt::TokenTree::Subtree(subtree_replace_token_ids_with_call_site(subtree, call_site)) } } } - fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf { + fn leaf_replace_token_ids_with_call_site(leaf: tt::Leaf, call_site: TokenId) -> tt::Leaf { match leaf { - tt::Leaf::Literal(lit) => { - tt::Leaf::Literal(tt::Literal { span: tt::TokenId::unspecified(), ..lit }) - } - tt::Leaf::Punct(punct) => { - tt::Leaf::Punct(tt::Punct { span: tt::TokenId::unspecified(), ..punct }) - } - tt::Leaf::Ident(ident) => { - tt::Leaf::Ident(tt::Ident { span: tt::TokenId::unspecified(), ..ident }) - } + tt::Leaf::Literal(lit) => tt::Leaf::Literal(tt::Literal { span: call_site, ..lit }), + tt::Leaf::Punct(punct) => tt::Leaf::Punct(tt::Punct { span: call_site, ..punct }), + tt::Leaf::Ident(ident) => tt::Leaf::Ident(tt::Ident { span: call_site, ..ident }), } } } diff --git a/crates/proc-macro-srv/src/tests/utils.rs b/crates/proc-macro-srv/src/tests/utils.rs index 49b4d973b63b..ccfefafb2cfc 100644 --- a/crates/proc-macro-srv/src/tests/utils.rs +++ b/crates/proc-macro-srv/src/tests/utils.rs @@ -1,18 +1,19 @@ //! utils used in proc-macro tests use expect_test::Expect; -use std::str::FromStr; +use proc_macro_api::msg::TokenId; +use tt::Span; use crate::{dylib, proc_macro_test_dylib_path, ProcMacroSrv}; -fn parse_string(code: &str) -> Option { +fn parse_string(code: &str, call_site: TokenId) -> Option { // This is a bit strange. We need to parse a string into a token stream into // order to create a tt::SubTree from it in fixtures. `into_subtree` is // implemented by all the ABIs we have so we arbitrarily choose one ABI to // write a `parse_string` function for and use that. The tests don't really // care which ABI we're using as the `into_subtree` function isn't part of // the ABI and shouldn't change between ABI versions. - crate::server::TokenStream::from_str(code).ok() + crate::server::TokenStream::from_str(code, call_site).ok() } pub fn assert_expand(macro_name: &str, ra_fixture: &str, expect: Expect) { @@ -24,12 +25,22 @@ pub fn assert_expand_attr(macro_name: &str, ra_fixture: &str, attr_args: &str, e } fn assert_expand_impl(macro_name: &str, input: &str, attr: Option<&str>, expect: Expect) { + let call_site = TokenId::DUMMY; let path = proc_macro_test_dylib_path(); let expander = dylib::Expander::new(&path).unwrap(); - let fixture = parse_string(input).unwrap(); - let attr = attr.map(|attr| parse_string(attr).unwrap().into_subtree()); - - let res = expander.expand(macro_name, &fixture.into_subtree(), attr.as_ref()).unwrap(); + let fixture = parse_string(input, call_site).unwrap(); + let attr = attr.map(|attr| parse_string(attr, call_site).unwrap().into_subtree(call_site)); + + let res = expander + .expand( + macro_name, + &fixture.into_subtree(call_site), + attr.as_ref(), + TokenId::DUMMY, + TokenId::DUMMY, + TokenId::DUMMY, + ) + .unwrap(); expect.assert_eq(&format!("{res:?}")); } diff --git a/crates/rust-analyzer/src/integrated_benchmarks.rs b/crates/rust-analyzer/src/integrated_benchmarks.rs index a29e18811d88..a865d9e4ab82 100644 --- a/crates/rust-analyzer/src/integrated_benchmarks.rs +++ b/crates/rust-analyzer/src/integrated_benchmarks.rs @@ -56,7 +56,7 @@ fn integrated_highlighting_benchmark() { analysis.highlight_as_html(file_id, false).unwrap(); } - profile::init_from("*>1"); + profile::init_from("*>100"); { let _it = stdx::timeit("change"); From 7a8c4c001ba82e360d77d17a5815f2c6fd50e3e5 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Tue, 28 Nov 2023 16:50:05 +0100 Subject: [PATCH 15/24] Turn macro_expand from query to normal function --- crates/base-db/src/span.rs | 12 ++++-------- .../hir-def/src/macro_expansion_tests/proc_macros.rs | 1 - crates/hir-expand/src/db.rs | 5 +---- crates/hir-expand/src/files.rs | 1 + crates/hir-expand/src/fixup.rs | 4 ++-- crates/hir-expand/src/span.rs | 2 ++ crates/hir/src/db.rs | 4 ++-- crates/ide-completion/src/tests/proc_macros.rs | 2 -- crates/ide-db/src/apply_change.rs | 2 +- crates/ide-db/src/lib.rs | 7 ------- crates/proc-macro-api/src/msg/flat.rs | 8 +------- crates/proc-macro-srv/src/lib.rs | 3 +-- 12 files changed, 15 insertions(+), 36 deletions(-) diff --git a/crates/base-db/src/span.rs b/crates/base-db/src/span.rs index d2b40ecdd214..420c669641cc 100644 --- a/crates/base-db/src/span.rs +++ b/crates/base-db/src/span.rs @@ -1,4 +1,4 @@ -/// File and span related types. +//! File and span related types. // FIXME: This should probably be moved into its own crate. use std::fmt; @@ -26,19 +26,15 @@ impl fmt::Display for SyntaxContextId { impl SyntaxContext for SyntaxContextId { const DUMMY: Self = Self::ROOT; - // veykril(HACK): salsa doesn't allow us fetching the id of the current input to be allocated so - // we need a special value that behaves as the current context. } // inherent trait impls please tyvm impl SyntaxContextId { - // TODO: This is very much UB, salsa exposes no way to create an InternId in a const context - // currently (which kind of makes sense but we need it here!) pub const ROOT: Self = SyntaxContextId(unsafe { InternId::new_unchecked(0) }); - // TODO: This is very much UB, salsa exposes no way to create an InternId in a const context - // currently (which kind of makes sense but we need it here!) + // veykril(HACK): salsa doesn't allow us fetching the id of the current input to be allocated so + // we need a special value that behaves as the current context. pub const SELF_REF: Self = SyntaxContextId(unsafe { InternId::new_unchecked(InternId::MAX - 1) }); - /// Used syntax fixups + // Used for syntax fixups pub const FAKE: Self = SyntaxContextId(unsafe { InternId::new_unchecked(InternId::MAX - 2) }); pub fn is_root(self) -> bool { diff --git a/crates/hir-def/src/macro_expansion_tests/proc_macros.rs b/crates/hir-def/src/macro_expansion_tests/proc_macros.rs index 548f22499b31..060b8aa8c193 100644 --- a/crates/hir-def/src/macro_expansion_tests/proc_macros.rs +++ b/crates/hir-def/src/macro_expansion_tests/proc_macros.rs @@ -74,7 +74,6 @@ fn foo() { } #[test] -#[ignore] // TODO fn attribute_macro_syntax_completion_2() { // common case of dot completion while typing check( diff --git a/crates/hir-expand/src/db.rs b/crates/hir-expand/src/db.rs index 1d55aaf1703f..00a7b0cb3fef 100644 --- a/crates/hir-expand/src/db.rs +++ b/crates/hir-expand/src/db.rs @@ -146,9 +146,6 @@ pub trait ExpandDatabase: SourceDatabase { id: AstId, ) -> Arc; - /// Expand macro call to a token tree. - // This query is LRU cached - fn macro_expand(&self, macro_call: MacroCallId) -> ExpandResult>; #[salsa::invoke(crate::builtin_fn_macro::include_arg_to_tt)] fn include_expand( &self, @@ -315,7 +312,7 @@ fn parse_macro_expansion( macro_file: MacroFileId, ) -> ExpandResult<(Parse, Arc)> { let _p = profile::span("parse_macro_expansion"); - let mbe::ValueResult { value: tt, err } = db.macro_expand(macro_file.macro_call_id); + let mbe::ValueResult { value: tt, err } = macro_expand(db, macro_file.macro_call_id); let expand_to = macro_expand_to(db, macro_file.macro_call_id); diff --git a/crates/hir-expand/src/files.rs b/crates/hir-expand/src/files.rs index 57a7fa5ec398..36eb2dbb27c4 100644 --- a/crates/hir-expand/src/files.rs +++ b/crates/hir-expand/src/files.rs @@ -1,3 +1,4 @@ +//! Things to wrap other things in file ids. use std::iter; use base_db::{ diff --git a/crates/hir-expand/src/fixup.rs b/crates/hir-expand/src/fixup.rs index 326ff1dec484..8fa7d5762958 100644 --- a/crates/hir-expand/src/fixup.rs +++ b/crates/hir-expand/src/fixup.rs @@ -52,7 +52,7 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { while let Some(event) = preorder.next() { let syntax::WalkEvent::Enter(node) = event else { continue }; - /* TODO + /* if can_handle_error(&node) && has_error_to_handle(&node) { // the node contains an error node, we have to completely replace it by something valid let (original_tree, new_tmap, new_next_id) = @@ -295,7 +295,7 @@ pub(crate) fn reverse_fixups(tt: &mut Subtree, undo_info: &SyntaxFixupUndoInfo) tt::TokenTree::Leaf(leaf) => leaf.span().ctx != SyntaxContextId::FAKE, tt::TokenTree::Subtree(st) => st.delimiter.open.ctx != SyntaxContextId::FAKE, }) - // .flat_map(|tt| match tt { TODO + // .flat_map(|tt| match tt { // tt::TokenTree::Subtree(mut tt) => { // reverse_fixups(&mut tt, undo_info); // SmallVec::from_const([tt.into()]) diff --git a/crates/hir-expand/src/span.rs b/crates/hir-expand/src/span.rs index c2399259facb..1703923d11a1 100644 --- a/crates/hir-expand/src/span.rs +++ b/crates/hir-expand/src/span.rs @@ -1,3 +1,5 @@ +//! Spanmaps allow turning absolute ranges into relative ranges for incrementality purposes as well +//! as associating spans with text ranges in a particular file. use base_db::{ span::{ErasedFileAstId, SpanAnchor, SpanData, SyntaxContextId, ROOT_ERASED_FILE_AST_ID}, FileId, diff --git a/crates/hir/src/db.rs b/crates/hir/src/db.rs index b9112dee3166..ff6f987aa128 100644 --- a/crates/hir/src/db.rs +++ b/crates/hir/src/db.rs @@ -23,7 +23,7 @@ pub use hir_def::db::{ }; pub use hir_expand::db::{ AstIdMapQuery, DeclMacroExpanderQuery, ExpandDatabase, ExpandDatabaseStorage, - ExpandProcMacroQuery, IncludeExpandQuery, InternMacroCallQuery, MacroArgQuery, - MacroExpandQuery, ParseMacroExpansionErrorQuery, ParseMacroExpansionQuery, RealSpanMapQuery, + ExpandProcMacroQuery, IncludeExpandQuery, InternMacroCallQuery, InternSyntaxContextQuery, + MacroArgQuery, ParseMacroExpansionErrorQuery, ParseMacroExpansionQuery, RealSpanMapQuery, }; pub use hir_ty::db::*; diff --git a/crates/ide-completion/src/tests/proc_macros.rs b/crates/ide-completion/src/tests/proc_macros.rs index e55d1f315fed..2d6234e310c6 100644 --- a/crates/ide-completion/src/tests/proc_macros.rs +++ b/crates/ide-completion/src/tests/proc_macros.rs @@ -9,7 +9,6 @@ fn check(ra_fixture: &str, expect: Expect) { } #[test] -#[ignore] // todo fn complete_dot_in_attr() { check( r#" @@ -41,7 +40,6 @@ fn main() { } #[test] -#[ignore] // TODO fn complete_dot_in_attr2() { check( r#" diff --git a/crates/ide-db/src/apply_change.rs b/crates/ide-db/src/apply_change.rs index 2efe3ee1ed75..67e686dad11e 100644 --- a/crates/ide-db/src/apply_change.rs +++ b/crates/ide-db/src/apply_change.rs @@ -101,8 +101,8 @@ impl RootDatabase { hir::db::ExpandProcMacroQuery hir::db::IncludeExpandQuery hir::db::InternMacroCallQuery + hir::db::InternSyntaxContextQuery hir::db::MacroArgQuery - hir::db::MacroExpandQuery hir::db::ParseMacroExpansionQuery hir::db::RealSpanMapQuery diff --git a/crates/ide-db/src/lib.rs b/crates/ide-db/src/lib.rs index 38c9a3538f3a..cbd51e67810c 100644 --- a/crates/ide-db/src/lib.rs +++ b/crates/ide-db/src/lib.rs @@ -157,7 +157,6 @@ impl RootDatabase { base_db::ParseQuery.in_db_mut(self).set_lru_capacity(lru_capacity); // macro expansions are usually rather small, so we can afford to keep more of them alive hir::db::ParseMacroExpansionQuery.in_db_mut(self).set_lru_capacity(4 * lru_capacity); - hir::db::MacroExpandQuery.in_db_mut(self).set_lru_capacity(4 * lru_capacity); } pub fn update_lru_capacities(&mut self, lru_capacities: &FxHashMap, usize>) { @@ -175,12 +174,6 @@ impl RootDatabase { .copied() .unwrap_or(4 * base_db::DEFAULT_PARSE_LRU_CAP), ); - hir_db::MacroExpandQuery.in_db_mut(self).set_lru_capacity( - lru_capacities - .get(stringify!(MacroExpandQuery)) - .copied() - .unwrap_or(4 * base_db::DEFAULT_PARSE_LRU_CAP), - ); macro_rules! update_lru_capacity_per_query { ($( $module:ident :: $query:ident )*) => {$( diff --git a/crates/proc-macro-api/src/msg/flat.rs b/crates/proc-macro-api/src/msg/flat.rs index baf8bbad4bf2..43840fa333d7 100644 --- a/crates/proc-macro-api/src/msg/flat.rs +++ b/crates/proc-macro-api/src/msg/flat.rs @@ -43,7 +43,7 @@ use serde::{Deserialize, Serialize}; use crate::msg::ENCODE_CLOSE_SPAN_VERSION; -pub type SpanDataIndexMap = IndexSet; +type SpanDataIndexMap = IndexSet; #[derive(Clone, Copy, PartialEq, Eq, Hash)] pub struct TokenId(pub u32); @@ -54,12 +54,6 @@ impl std::fmt::Debug for TokenId { } } -impl TokenId { - pub const DEF_SITE: Self = TokenId(0); - pub const CALL_SITE: Self = TokenId(0); - pub const MIXED_SITE: Self = TokenId(0); -} - impl tt::Span for TokenId { const DUMMY: Self = TokenId(!0); diff --git a/crates/proc-macro-srv/src/lib.rs b/crates/proc-macro-srv/src/lib.rs index 32a07a84777b..dd327681c4c0 100644 --- a/crates/proc-macro-srv/src/lib.rs +++ b/crates/proc-macro-srv/src/lib.rs @@ -31,9 +31,8 @@ use std::{ time::SystemTime, }; -use ::tt::Span; use proc_macro_api::{ - msg::{self, ExpnGlobals, TokenId, CURRENT_API_VERSION, HAS_GLOBAL_SPANS}, + msg::{self, ExpnGlobals, TokenId, CURRENT_API_VERSION}, ProcMacroKind, }; From f48fa0c6cba2d442523daced9db09576e4b11e78 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Wed, 29 Nov 2023 15:48:40 +0100 Subject: [PATCH 16/24] Re-implement syntax fixups --- crates/base-db/src/span.rs | 2 - crates/hir-expand/src/db.rs | 103 ++++-- crates/hir-expand/src/fixup.rs | 206 ++++++----- crates/hir-expand/src/lib.rs | 13 +- .../src/handlers/unresolved_module.rs | 2 +- crates/mbe/src/lib.rs | 2 +- crates/mbe/src/syntax_bridge.rs | 344 +++++++++++------- 7 files changed, 408 insertions(+), 264 deletions(-) diff --git a/crates/base-db/src/span.rs b/crates/base-db/src/span.rs index 420c669641cc..dbccbb382a7e 100644 --- a/crates/base-db/src/span.rs +++ b/crates/base-db/src/span.rs @@ -34,8 +34,6 @@ impl SyntaxContextId { // we need a special value that behaves as the current context. pub const SELF_REF: Self = SyntaxContextId(unsafe { InternId::new_unchecked(InternId::MAX - 1) }); - // Used for syntax fixups - pub const FAKE: Self = SyntaxContextId(unsafe { InternId::new_unchecked(InternId::MAX - 2) }); pub fn is_root(self) -> bool { self == Self::ROOT diff --git a/crates/hir-expand/src/db.rs b/crates/hir-expand/src/db.rs index 00a7b0cb3fef..0135d97f1cd1 100644 --- a/crates/hir-expand/src/db.rs +++ b/crates/hir-expand/src/db.rs @@ -9,6 +9,7 @@ use base_db::{ use either::Either; use limit::Limit; use mbe::{syntax_node_to_token_tree, ValueResult}; +use rustc_hash::FxHashSet; use syntax::{ ast::{self, HasAttrs, HasDocComments}, AstNode, Parse, SyntaxError, SyntaxNode, SyntaxToken, T, @@ -20,6 +21,7 @@ use crate::{ attrs::RawAttrs, builtin_attr_macro::pseudo_derive_attr_expansion, builtin_fn_macro::EagerExpander, + fixup::{self, SyntaxFixupUndoInfo}, hygiene::{self, SyntaxContextData, Transparency}, span::{RealSpanMap, SpanMap, SpanMapRef}, tt, AstId, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander, EagerCallInfo, @@ -135,7 +137,7 @@ pub trait ExpandDatabase: SourceDatabase { fn macro_arg( &self, id: MacroCallId, - ) -> ValueResult>, Arc>>; + ) -> ValueResult, SyntaxFixupUndoInfo)>, Arc>>; /// Fetches the expander for this macro. #[salsa::transparent] fn macro_expander(&self, id: MacroDefId) -> TokenExpander; @@ -189,15 +191,33 @@ pub fn expand_speculative( ) -> Option<(SyntaxNode, SyntaxToken)> { let loc = db.lookup_intern_macro_call(actual_macro_call); - // Build the subtree and token mapping for the speculative args - let _censor = censor_for_macro_input(&loc, speculative_args); let span_map = RealSpanMap::absolute(SpanAnchor::DUMMY.file_id); let span_map = SpanMapRef::RealSpanMap(&span_map); - let mut tt = mbe::syntax_node_to_token_tree( - speculative_args, - // we don't leak these spans into any query so its fine to make them absolute - span_map, - ); + + // Build the subtree and token mapping for the speculative args + let (mut tt, undo_info) = match loc.kind { + MacroCallKind::FnLike { .. } => { + (mbe::syntax_node_to_token_tree(speculative_args, span_map), SyntaxFixupUndoInfo::NONE) + } + MacroCallKind::Derive { .. } | MacroCallKind::Attr { .. } => { + let censor = censor_for_macro_input(&loc, speculative_args); + let mut fixups = fixup::fixup_syntax(span_map, speculative_args); + fixups.append.retain(|it, _| match it { + syntax::NodeOrToken::Node(it) => !censor.contains(it), + syntax::NodeOrToken::Token(_) => true, + }); + fixups.remove.extend(censor); + ( + mbe::syntax_node_to_token_tree_modified( + speculative_args, + span_map, + fixups.append, + fixups.remove, + ), + fixups.undo_info, + ) + } + }; let attr_arg = match loc.kind { MacroCallKind::Attr { invoc_attr_index, .. } => { @@ -227,7 +247,7 @@ pub fn expand_speculative( // Do the actual expansion, we need to directly expand the proc macro due to the attribute args // Otherwise the expand query will fetch the non speculative attribute args and pass those instead. - let speculative_expansion = match loc.def.kind { + let mut speculative_expansion = match loc.def.kind { MacroDefKind::ProcMacro(expander, ..) => { tt.delimiter = tt::Delimiter::UNSPECIFIED; let call_site = loc.span(db); @@ -261,6 +281,7 @@ pub fn expand_speculative( }; let expand_to = macro_expand_to(db, actual_macro_call); + fixup::reverse_fixups(&mut speculative_expansion.value, &undo_info); let (node, rev_tmap) = token_tree_to_syntax_node(&speculative_expansion.value, expand_to); let syntax_node = node.syntax_node(); @@ -347,7 +368,9 @@ fn parse_with_map(db: &dyn ExpandDatabase, file_id: HirFileId) -> (Parse ValueResult>, Arc>> { + // FIXME: consider the following by putting fixup info into eager call info args + // ) -> ValueResult>, Arc>> { +) -> ValueResult, SyntaxFixupUndoInfo)>, Arc>> { let mismatched_delimiters = |arg: &SyntaxNode| { let first = arg.first_child_or_token().map_or(T![.], |it| it.kind()); let last = arg.last_child_or_token().map_or(T![.], |it| it.kind()); @@ -375,7 +398,7 @@ fn macro_arg( .then(|| loc.eager.as_deref()) .flatten() { - ValueResult::ok(Some(arg.clone())) + ValueResult::ok(Some((arg.clone(), SyntaxFixupUndoInfo::NONE))) } else { let (parse, map) = parse_with_map(db, loc.kind.file_id()); let root = parse.syntax_node(); @@ -404,22 +427,27 @@ fn macro_arg( } MacroCallKind::Attr { ast_id, .. } => ast_id.to_ptr(db).to_node(&root).syntax().clone(), }; - let censor = censor_for_macro_input(&loc, &syntax); - let mut tt = match loc.kind { + let (mut tt, undo_info) = match loc.kind { MacroCallKind::FnLike { .. } => { - mbe::syntax_node_to_token_tree_censored(&syntax, map.as_ref(), censor) + (mbe::syntax_node_to_token_tree(&syntax, map.as_ref()), SyntaxFixupUndoInfo::NONE) } MacroCallKind::Derive { .. } | MacroCallKind::Attr { .. } => { - // let mut fixups = crate::fixup::fixup_syntax(&syntax); - // fixups.replace.extend(censor.into_iter().map(|node| (node.into(), Vec::new()))); - // let (mut tt, tmap, _) = mbe::syntax_node_to_token_tree_with_modifications( - // &node, - // fixups.token_map, - // fixups.next_id, - // fixups.replace, - // fixups.append, - // ); - mbe::syntax_node_to_token_tree_censored(&syntax, map.as_ref(), censor) + let censor = censor_for_macro_input(&loc, &syntax); + let mut fixups = fixup::fixup_syntax(map.as_ref(), &syntax); + fixups.append.retain(|it, _| match it { + syntax::NodeOrToken::Node(it) => !censor.contains(it), + syntax::NodeOrToken::Token(_) => true, + }); + fixups.remove.extend(censor); + ( + mbe::syntax_node_to_token_tree_modified( + &syntax, + map, + fixups.append, + fixups.remove, + ), + fixups.undo_info, + ) } }; @@ -430,15 +458,15 @@ fn macro_arg( if matches!(loc.def.kind, MacroDefKind::BuiltInEager(..)) { match parse.errors() { - [] => ValueResult::ok(Some(Arc::new(tt))), + [] => ValueResult::ok(Some((Arc::new(tt), undo_info))), errors => ValueResult::new( - Some(Arc::new(tt)), + Some((Arc::new(tt), undo_info)), // Box::<[_]>::from(res.errors()), not stable yet Arc::new(errors.to_vec().into_boxed_slice()), ), } } else { - ValueResult::ok(Some(Arc::new(tt))) + ValueResult::ok(Some((Arc::new(tt), undo_info))) } } } @@ -447,7 +475,7 @@ fn macro_arg( /// Certain macro calls expect some nodes in the input to be preprocessed away, namely: /// - derives expect all `#[derive(..)]` invocations up to the currently invoked one to be stripped /// - attributes expect the invoking attribute to be stripped -fn censor_for_macro_input(loc: &MacroCallLoc, node: &SyntaxNode) -> Vec { +fn censor_for_macro_input(loc: &MacroCallLoc, node: &SyntaxNode) -> FxHashSet { // FIXME: handle `cfg_attr` (|| { let censor = match loc.kind { @@ -574,13 +602,13 @@ fn macro_expand( let MacroCallKind::Derive { ast_id, .. } = loc.kind else { unreachable!() }; let node = ast_id.to_ptr(db).to_node(&root); - // FIXME: we might need to remove the spans from the input to the derive macro here + // FIXME: Use censoring let _censor = censor_for_macro_input(&loc, node.syntax()); expander.expand(db, macro_call_id, &node, map.as_ref()) } _ => { let ValueResult { value, err } = db.macro_arg(macro_call_id); - let Some(macro_arg) = value else { + let Some((macro_arg, undo_info)) = value else { return ExpandResult { value: Arc::new(tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, @@ -608,7 +636,7 @@ fn macro_expand( // As such we just return the input subtree here. MacroDefKind::BuiltInEager(..) if loc.eager.is_none() => { return ExpandResult { - value: Arc::new(arg.clone()), + value: macro_arg.clone(), err: err.map(|err| { let mut buf = String::new(); for err in &**err { @@ -624,7 +652,11 @@ fn macro_expand( MacroDefKind::BuiltInEager(it, _) => { it.expand(db, macro_call_id, &arg).map_err(Into::into) } - MacroDefKind::BuiltInAttr(it, _) => it.expand(db, macro_call_id, &arg), + MacroDefKind::BuiltInAttr(it, _) => { + let mut res = it.expand(db, macro_call_id, &arg); + fixup::reverse_fixups(&mut res.value, &undo_info); + res + } _ => unreachable!(), } } @@ -647,9 +679,8 @@ fn macro_expand( } fn expand_proc_macro(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult> { - // FIXME: Syntax fix ups let loc = db.lookup_intern_macro_call(id); - let Some(macro_arg) = db.macro_arg(id).value else { + let Some((macro_arg, undo_info)) = db.macro_arg(id).value else { return ExpandResult { value: Arc::new(tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, @@ -672,7 +703,7 @@ fn expand_proc_macro(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult ExpandResult>, - pub(crate) replace: FxHashMap>, + pub(crate) remove: FxHashSet, pub(crate) undo_info: SyntaxFixupUndoInfo, } /// This is the information needed to reverse the fixups. -#[derive(Debug, Default, PartialEq, Eq)] +#[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct SyntaxFixupUndoInfo { - original: Box<[Subtree]>, + // FIXME: ThinArc<[Subtree]> + original: Option>>, +} + +impl SyntaxFixupUndoInfo { + pub(crate) const NONE: Self = SyntaxFixupUndoInfo { original: None }; } // censoring -> just don't convert the node @@ -39,47 +46,45 @@ pub struct SyntaxFixupUndoInfo { // append -> insert a fake node, here we need to assemble some dummy span that we can figure out how // to remove later -pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { +pub(crate) fn fixup_syntax(span_map: SpanMapRef<'_>, node: &SyntaxNode) -> SyntaxFixups { let mut append = FxHashMap::::default(); - let mut replace = FxHashMap::::default(); + let mut remove = FxHashSet::::default(); let mut preorder = node.preorder(); let mut original = Vec::new(); let dummy_range = TextRange::empty(TextSize::new(0)); + // we use a file id of `FileId(!0)` to signal a fake node, and the text range's start offset as + // the index into the replacement vec but only if the end points to !0 let dummy_anchor = - SpanAnchor { file_id: FileId(!0), ast_id: ErasedFileAstId::from_raw(RawIdx::from(0)) }; - let fake_span = - SpanData { range: dummy_range, anchor: dummy_anchor, ctx: SyntaxContextId::FAKE }; + SpanAnchor { file_id: FileId(!0), ast_id: ErasedFileAstId::from_raw(RawIdx::from(!0)) }; + let fake_span = |range| SpanData { + range: dummy_range, + anchor: dummy_anchor, + ctx: span_map.span_for_range(range).ctx, + }; while let Some(event) = preorder.next() { let syntax::WalkEvent::Enter(node) = event else { continue }; - /* + let node_range = node.text_range(); if can_handle_error(&node) && has_error_to_handle(&node) { + remove.insert(node.clone().into()); // the node contains an error node, we have to completely replace it by something valid - let (original_tree, new_tmap, new_next_id) = - mbe::syntax_node_to_token_tree_with_modifications( - &node, - mem::take(&mut token_map), - next_id, - Default::default(), - Default::default(), - ); - token_map = new_tmap; - next_id = new_next_id; + let original_tree = mbe::syntax_node_to_token_tree(&node, span_map); let idx = original.len() as u32; original.push(original_tree); - let replacement = SyntheticToken { - kind: SyntaxKind::IDENT, + let replacement = Leaf::Ident(Ident { text: "__ra_fixup".into(), - range: node.text_range(), - id: SyntheticTokenId(idx), - }; - replace.insert(node.clone().into(), vec![replacement]); + span: SpanData { + range: TextRange::new(TextSize::new(idx), TextSize::new(!0)), + anchor: dummy_anchor, + ctx: span_map.span_for_range(node_range).ctx, + }, + }); + append.insert(node.clone().into(), vec![replacement]); preorder.skip_subtree(); continue; } - */ + // In some other situations, we can fix things by just appending some tokens. - let end_range = TextRange::empty(node.text_range().end()); match_ast! { match node { ast::FieldExpr(it) => { @@ -88,7 +93,7 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { append.insert(node.clone().into(), vec![ Leaf::Ident(Ident { text: "__ra_fixup".into(), - span: fake_span + span: fake_span(node_range), }), ]); } @@ -99,7 +104,7 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { Leaf::Punct(Punct { char: ';', spacing: Spacing::Alone, - span: fake_span + span: fake_span(node_range), }), ]); } @@ -110,7 +115,7 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { Leaf::Punct(Punct { char: ';', spacing: Spacing::Alone, - span: fake_span + span: fake_span(node_range) }), ]); } @@ -125,7 +130,7 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { append.insert(if_token.into(), vec![ Leaf::Ident(Ident { text: "__ra_fixup".into(), - span: fake_span + span: fake_span(node_range) }), ]); } @@ -135,12 +140,12 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { Leaf::Punct(Punct { char: '{', spacing: Spacing::Alone, - span: fake_span + span: fake_span(node_range) }), Leaf::Punct(Punct { char: '}', spacing: Spacing::Alone, - span: fake_span + span: fake_span(node_range) }), ]); } @@ -155,7 +160,7 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { append.insert(while_token.into(), vec![ Leaf::Ident(Ident { text: "__ra_fixup".into(), - span: fake_span + span: fake_span(node_range) }), ]); } @@ -165,12 +170,12 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { Leaf::Punct(Punct { char: '{', spacing: Spacing::Alone, - span: fake_span + span: fake_span(node_range) }), Leaf::Punct(Punct { char: '}', spacing: Spacing::Alone, - span: fake_span + span: fake_span(node_range) }), ]); } @@ -182,12 +187,12 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { Leaf::Punct(Punct { char: '{', spacing: Spacing::Alone, - span: fake_span + span: fake_span(node_range) }), Leaf::Punct(Punct { char: '}', spacing: Spacing::Alone, - span: fake_span + span: fake_span(node_range) }), ]); } @@ -202,7 +207,7 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { append.insert(match_token.into(), vec![ Leaf::Ident(Ident { text: "__ra_fixup".into(), - span: fake_span + span: fake_span(node_range) }), ]); } @@ -213,12 +218,12 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { Leaf::Punct(Punct { char: '{', spacing: Spacing::Alone, - span: fake_span + span: fake_span(node_range) }), Leaf::Punct(Punct { char: '}', spacing: Spacing::Alone, - span: fake_span + span: fake_span(node_range) }), ]); } @@ -236,7 +241,7 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { ].map(|text| Leaf::Ident(Ident { text: text.into(), - span: fake_span + span: fake_span(node_range) }), ); @@ -253,12 +258,12 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { Leaf::Punct(Punct { char: '{', spacing: Spacing::Alone, - span: fake_span + span: fake_span(node_range) }), Leaf::Punct(Punct { char: '}', spacing: Spacing::Alone, - span: fake_span + span: fake_span(node_range) }), ]); } @@ -267,10 +272,13 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups { } } } + let needs_fixups = !append.is_empty() || !original.is_empty(); SyntaxFixups { append, - replace, - undo_info: SyntaxFixupUndoInfo { original: original.into_boxed_slice() }, + remove, + undo_info: SyntaxFixupUndoInfo { + original: needs_fixups.then(|| Arc::new(original.into_boxed_slice())), + }, } } @@ -287,42 +295,55 @@ fn has_error_to_handle(node: &SyntaxNode) -> bool { } pub(crate) fn reverse_fixups(tt: &mut Subtree, undo_info: &SyntaxFixupUndoInfo) { + let Some(undo_info) = undo_info.original.as_deref() else { return }; + let undo_info = &**undo_info; + reverse_fixups_(tt, undo_info); +} + +fn reverse_fixups_(tt: &mut Subtree, undo_info: &[Subtree]) { let tts = std::mem::take(&mut tt.token_trees); tt.token_trees = tts .into_iter() // delete all fake nodes .filter(|tt| match tt { - tt::TokenTree::Leaf(leaf) => leaf.span().ctx != SyntaxContextId::FAKE, - tt::TokenTree::Subtree(st) => st.delimiter.open.ctx != SyntaxContextId::FAKE, + tt::TokenTree::Leaf(leaf) => { + let span = leaf.span(); + span.anchor.file_id != FileId(!0) || span.range.end() == TextSize::new(!0) + } + tt::TokenTree::Subtree(_) => true, + }) + .flat_map(|tt| match tt { + tt::TokenTree::Subtree(mut tt) => { + reverse_fixups_(&mut tt, undo_info); + SmallVec::from_const([tt.into()]) + } + tt::TokenTree::Leaf(leaf) => { + if leaf.span().anchor.file_id == FileId(!0) { + let original = undo_info[u32::from(leaf.span().range.start()) as usize].clone(); + if original.delimiter.kind == tt::DelimiterKind::Invisible { + original.token_trees.into() + } else { + SmallVec::from_const([original.into()]) + } + } else { + SmallVec::from_const([leaf.into()]) + } + } }) - // .flat_map(|tt| match tt { - // tt::TokenTree::Subtree(mut tt) => { - // reverse_fixups(&mut tt, undo_info); - // SmallVec::from_const([tt.into()]) - // } - // tt::TokenTree::Leaf(leaf) => { - // if let Some(id) = leaf.span().anchor { - // let original = undo_info.original[id.0 as usize].clone(); - // if original.delimiter.kind == tt::DelimiterKind::Invisible { - // original.token_trees.into() - // } else { - // SmallVec::from_const([original.into()]) - // } - // } else { - // SmallVec::from_const([leaf.into()]) - // } - // } - // }) .collect(); } #[cfg(test)] mod tests { + use base_db::FileId; use expect_test::{expect, Expect}; + use triomphe::Arc; - use crate::tt; - - use super::reverse_fixups; + use crate::{ + fixup::reverse_fixups, + span::{RealSpanMap, SpanMap}, + tt, + }; // The following three functions are only meant to check partial structural equivalence of // `TokenTree`s, see the last assertion in `check()`. @@ -352,13 +373,13 @@ mod tests { #[track_caller] fn check(ra_fixture: &str, mut expect: Expect) { let parsed = syntax::SourceFile::parse(ra_fixture); - let fixups = super::fixup_syntax(&parsed.syntax_node()); - let (mut tt, tmap, _) = mbe::syntax_node_to_token_tree_with_modifications( + let span_map = SpanMap::RealSpanMap(Arc::new(RealSpanMap::absolute(FileId(0)))); + let fixups = super::fixup_syntax(span_map.as_ref(), &parsed.syntax_node()); + let mut tt = mbe::syntax_node_to_token_tree_modified( &parsed.syntax_node(), - fixups.token_map, - fixups.next_id, - fixups.replace, + span_map.as_ref(), fixups.append, + fixups.remove, ); let actual = format!("{tt}\n"); @@ -374,14 +395,15 @@ mod tests { parse.syntax_node() ); - reverse_fixups(&mut tt, &tmap, &fixups.undo_info); + reverse_fixups(&mut tt, &fixups.undo_info); // the fixed-up + reversed version should be equivalent to the original input // modulo token IDs and `Punct`s' spacing. - let (original_as_tt, _) = mbe::syntax_node_to_token_tree(&parsed.syntax_node()); + let original_as_tt = + mbe::syntax_node_to_token_tree(&parsed.syntax_node(), span_map.as_ref()); assert!( check_subtree_eq(&tt, &original_as_tt), - "different token tree: {tt:?},\n{original_as_tt:?}" + "different token tree:\n{tt:?}\n\n{original_as_tt:?}" ); } @@ -394,7 +416,7 @@ fn foo() { } "#, expect![[r#" -fn foo () {for _ in __ra_fixup {}} +fn foo () {for _ in __ra_fixup { }} "#]], ) } @@ -422,7 +444,7 @@ fn foo() { } "#, expect![[r#" -fn foo () {for bar in qux {}} +fn foo () {for bar in qux { }} "#]], ) } @@ -453,7 +475,7 @@ fn foo() { } "#, expect![[r#" -fn foo () {match __ra_fixup {}} +fn foo () {match __ra_fixup { }} "#]], ) } @@ -485,7 +507,7 @@ fn foo() { } "#, expect![[r#" -fn foo () {match __ra_fixup {}} +fn foo () {match __ra_fixup { }} "#]], ) } @@ -600,7 +622,7 @@ fn foo() { } "#, expect![[r#" -fn foo () {if a {}} +fn foo () {if a { }} "#]], ) } @@ -614,7 +636,7 @@ fn foo() { } "#, expect![[r#" -fn foo () {if __ra_fixup {}} +fn foo () {if __ra_fixup { }} "#]], ) } @@ -628,7 +650,7 @@ fn foo() { } "#, expect![[r#" -fn foo () {if __ra_fixup {} {}} +fn foo () {if __ra_fixup {} { }} "#]], ) } @@ -642,7 +664,7 @@ fn foo() { } "#, expect![[r#" -fn foo () {while __ra_fixup {}} +fn foo () {while __ra_fixup { }} "#]], ) } @@ -656,7 +678,7 @@ fn foo() { } "#, expect![[r#" -fn foo () {while foo {}} +fn foo () {while foo { }} "#]], ) } @@ -683,7 +705,7 @@ fn foo() { } "#, expect![[r#" -fn foo () {loop {}} +fn foo () {loop { }} "#]], ) } diff --git a/crates/hir-expand/src/lib.rs b/crates/hir-expand/src/lib.rs index 9027ea1c27b4..491c5e638ee1 100644 --- a/crates/hir-expand/src/lib.rs +++ b/crates/hir-expand/src/lib.rs @@ -20,7 +20,7 @@ pub mod mod_path; pub mod attrs; pub mod span; pub mod files; -// mod fixup; +mod fixup; use triomphe::Arc; @@ -42,6 +42,7 @@ use crate::{ builtin_derive_macro::BuiltinDeriveExpander, builtin_fn_macro::{BuiltinFnLikeExpander, EagerExpander}, db::TokenExpander, + fixup::SyntaxFixupUndoInfo, mod_path::ModPath, proc_macro::ProcMacroExpander, span::{ExpansionSpanMap, SpanMap}, @@ -695,8 +696,14 @@ impl ExpansionInfo { let (parse, exp_map) = db.parse_macro_expansion(macro_file).value; let expanded = InMacroFile { file_id: macro_file, value: parse.syntax_node() }; - let macro_arg = db.macro_arg(macro_file.macro_call_id).value.unwrap_or_else(|| { - Arc::new(tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: Vec::new() }) + let (macro_arg, _) = db.macro_arg(macro_file.macro_call_id).value.unwrap_or_else(|| { + ( + Arc::new(tt::Subtree { + delimiter: tt::Delimiter::UNSPECIFIED, + token_trees: Vec::new(), + }), + SyntaxFixupUndoInfo::NONE, + ) }); let def = loc.def.ast_id().left().and_then(|id| { diff --git a/crates/ide-diagnostics/src/handlers/unresolved_module.rs b/crates/ide-diagnostics/src/handlers/unresolved_module.rs index 234ca2a7671b..4f81ec051258 100644 --- a/crates/ide-diagnostics/src/handlers/unresolved_module.rs +++ b/crates/ide-diagnostics/src/handlers/unresolved_module.rs @@ -150,7 +150,7 @@ mod baz {} ], ), main_node: Some( - InFile { + InFileWrapper { file_id: FileId( 0, ), diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index fdf97ad538cb..c19112b4c540 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -36,7 +36,7 @@ pub use crate::{ syntax_bridge::{ map_from_syntax_node, parse_exprs_with_sep, parse_to_token_tree, parse_to_token_tree_static_span, syntax_node_to_token_tree, - syntax_node_to_token_tree_censored, token_tree_to_syntax_node, SpanMapper, + syntax_node_to_token_tree_modified, token_tree_to_syntax_node, SpanMapper, }, token_map::TokenMap, }; diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index c61c52628607..be5eafd014af 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -1,6 +1,7 @@ //! Conversions between [`SyntaxNode`] and [`tt::TokenTree`]. -use stdx::non_empty_vec::NonEmptyVec; +use rustc_hash::{FxHashMap, FxHashSet}; +use stdx::{never, non_empty_vec::NonEmptyVec}; use syntax::{ ast::{self, make::tokens::doc_comment}, AstToken, NodeOrToken, Parse, PreorderWithTokens, SmolStr, SyntaxElement, SyntaxKind, @@ -74,14 +75,15 @@ where Ctx: SyntaxContext, SpanMap: SpanMapper>, { - let mut c = Converter::new(node, vec![], map); + let mut c = Converter::new(node, map, Default::default(), Default::default()); convert_tokens(&mut c) } -pub fn syntax_node_to_token_tree_censored( +pub fn syntax_node_to_token_tree_modified( node: &SyntaxNode, map: SpanMap, - censored: Vec, + append: FxHashMap>>>, + remove: FxHashSet, ) -> tt::Subtree> where SpanMap: SpanMapper>, @@ -89,7 +91,7 @@ where Anchor: Copy, Ctx: SyntaxContext, { - let mut c = Converter::new(node, censored, map); + let mut c = Converter::new(node, map, append, remove); convert_tokens(&mut c) } @@ -237,102 +239,105 @@ where while let Some((token, abs_range)) = conv.bump() { let tt::Subtree { delimiter, token_trees: result } = stack.last_mut(); - let kind = token.kind(conv); - - let tt = match kind { - // Desugar doc comments into doc attributes - COMMENT => { - let span = conv.span_for(abs_range); - if let Some(tokens) = conv.convert_doc_comment(&token, span) { - result.extend(tokens); - } - continue; - } - _ if kind.is_punct() && kind != UNDERSCORE => { - let expected = match delimiter.kind { - tt::DelimiterKind::Parenthesis => Some(T![')']), - tt::DelimiterKind::Brace => Some(T!['}']), - tt::DelimiterKind::Bracket => Some(T![']']), - tt::DelimiterKind::Invisible => None, - }; - - // Current token is a closing delimiter that we expect, fix up the closing span - // and end the subtree here - if matches!(expected, Some(expected) if expected == kind) { - if let Some(mut subtree) = stack.pop() { - subtree.delimiter.close = conv.span_for(abs_range); - stack.last_mut().token_trees.push(subtree.into()); + let tt = match token.as_leaf() { + Some(leaf) => tt::TokenTree::Leaf(leaf.clone()), + None => match token.kind(conv) { + // Desugar doc comments into doc attributes + COMMENT => { + let span = conv.span_for(abs_range); + if let Some(tokens) = conv.convert_doc_comment(&token, span) { + result.extend(tokens); } continue; } + kind if kind.is_punct() && kind != UNDERSCORE => { + let expected = match delimiter.kind { + tt::DelimiterKind::Parenthesis => Some(T![')']), + tt::DelimiterKind::Brace => Some(T!['}']), + tt::DelimiterKind::Bracket => Some(T![']']), + tt::DelimiterKind::Invisible => None, + }; - let delim = match kind { - T!['('] => Some(tt::DelimiterKind::Parenthesis), - T!['{'] => Some(tt::DelimiterKind::Brace), - T!['['] => Some(tt::DelimiterKind::Bracket), - _ => None, - }; - - // Start a new subtree - if let Some(kind) = delim { - let open = conv.span_for(abs_range); - stack.push(tt::Subtree { - delimiter: tt::Delimiter { - open, - // will be overwritten on subtree close above - close: open, - kind, - }, - token_trees: vec![], - }); - continue; - } + // Current token is a closing delimiter that we expect, fix up the closing span + // and end the subtree here + if matches!(expected, Some(expected) if expected == kind) { + if let Some(mut subtree) = stack.pop() { + subtree.delimiter.close = conv.span_for(abs_range); + stack.last_mut().token_trees.push(subtree.into()); + } + continue; + } - let spacing = match conv.peek().map(|next| next.kind(conv)) { - Some(kind) if is_single_token_op(kind) => tt::Spacing::Joint, - _ => tt::Spacing::Alone, - }; - let Some(char) = token.to_char(conv) else { - panic!("Token from lexer must be single char: token = {token:#?}") - }; - tt::Leaf::from(tt::Punct { char, spacing, span: conv.span_for(abs_range) }).into() - } - _ => { - macro_rules! make_leaf { - ($i:ident) => { - tt::$i { span: conv.span_for(abs_range), text: token.to_text(conv) }.into() + let delim = match kind { + T!['('] => Some(tt::DelimiterKind::Parenthesis), + T!['{'] => Some(tt::DelimiterKind::Brace), + T!['['] => Some(tt::DelimiterKind::Bracket), + _ => None, }; - } - let leaf: tt::Leaf<_> = match kind { - T![true] | T![false] => make_leaf!(Ident), - IDENT => make_leaf!(Ident), - UNDERSCORE => make_leaf!(Ident), - k if k.is_keyword() => make_leaf!(Ident), - k if k.is_literal() => make_leaf!(Literal), - LIFETIME_IDENT => { - let apostrophe = tt::Leaf::from(tt::Punct { - char: '\'', - spacing: tt::Spacing::Joint, - span: conv - .span_for(TextRange::at(abs_range.start(), TextSize::of('\''))), - }); - result.push(apostrophe.into()); - - let ident = tt::Leaf::from(tt::Ident { - text: SmolStr::new(&token.to_text(conv)[1..]), - span: conv.span_for(TextRange::at( - abs_range.start() + TextSize::of('\''), - abs_range.end(), - )), + + // Start a new subtree + if let Some(kind) = delim { + let open = conv.span_for(abs_range); + stack.push(tt::Subtree { + delimiter: tt::Delimiter { + open, + // will be overwritten on subtree close above + close: open, + kind, + }, + token_trees: vec![], }); - result.push(ident.into()); continue; } - _ => continue, - }; - leaf.into() - } + let spacing = match conv.peek().map(|next| next.kind(conv)) { + Some(kind) if is_single_token_op(kind) => tt::Spacing::Joint, + _ => tt::Spacing::Alone, + }; + let Some(char) = token.to_char(conv) else { + panic!("Token from lexer must be single char: token = {token:#?}") + }; + tt::Leaf::from(tt::Punct { char, spacing, span: conv.span_for(abs_range) }) + .into() + } + kind => { + macro_rules! make_leaf { + ($i:ident) => { + tt::$i { span: conv.span_for(abs_range), text: token.to_text(conv) } + .into() + }; + } + let leaf: tt::Leaf<_> = match kind { + T![true] | T![false] => make_leaf!(Ident), + IDENT => make_leaf!(Ident), + UNDERSCORE => make_leaf!(Ident), + k if k.is_keyword() => make_leaf!(Ident), + k if k.is_literal() => make_leaf!(Literal), + LIFETIME_IDENT => { + let apostrophe = tt::Leaf::from(tt::Punct { + char: '\'', + spacing: tt::Spacing::Joint, + span: conv + .span_for(TextRange::at(abs_range.start(), TextSize::of('\''))), + }); + result.push(apostrophe.into()); + + let ident = tt::Leaf::from(tt::Ident { + text: SmolStr::new(&token.to_text(conv)[1..]), + span: conv.span_for(TextRange::at( + abs_range.start() + TextSize::of('\''), + abs_range.end(), + )), + }); + result.push(ident.into()); + continue; + } + _ => continue, + }; + + leaf.into() + } + }, }; result.push(tt); @@ -470,16 +475,20 @@ struct StaticRawConverter<'a, S> { span: S, } -trait SrcToken: std::fmt::Debug { +trait SrcToken: std::fmt::Debug { fn kind(&self, ctx: &Ctx) -> SyntaxKind; fn to_char(&self, ctx: &Ctx) -> Option; fn to_text(&self, ctx: &Ctx) -> SmolStr; + + fn as_leaf(&self) -> Option<&tt::Leaf> { + None + } } trait TokenConverter: Sized { - type Token: SrcToken; + type Token: SrcToken; fn convert_doc_comment(&self, token: &Self::Token, span: S) -> Option>>; @@ -490,7 +499,7 @@ trait TokenConverter: Sized { fn span_for(&self, range: TextRange) -> S; } -impl SrcToken> for usize { +impl SrcToken, S> for usize { fn kind(&self, ctx: &RawConverter<'_, Anchor>) -> SyntaxKind { ctx.lexed.kind(*self) } @@ -504,7 +513,7 @@ impl SrcToken> for usize { } } -impl SrcToken> for usize { +impl SrcToken, S> for usize { fn kind(&self, ctx: &StaticRawConverter<'_, S>) -> SyntaxKind { ctx.lexed.kind(*self) } @@ -593,32 +602,79 @@ where } } -struct Converter { +struct Converter { current: Option, + current_leafs: Vec>, preorder: PreorderWithTokens, range: TextRange, punct_offset: Option<(SyntaxToken, TextSize)>, /// Used to make the emitted text ranges in the spans relative to the span anchor. map: SpanMap, - censored: Vec, -} - -impl Converter { - fn new(node: &SyntaxNode, censored: Vec, map: SpanMap) -> Self { - let range = node.text_range(); - let mut preorder = node.preorder_with_tokens(); - let first = Self::next_token(&mut preorder, &censored); - Converter { current: first, preorder, range, punct_offset: None, censored, map } - } - - fn next_token(preorder: &mut PreorderWithTokens, censor: &[SyntaxNode]) -> Option { - while let Some(ev) = preorder.next() { + append: FxHashMap>>, + remove: FxHashSet, +} + +impl Converter { + fn new( + node: &SyntaxNode, + map: SpanMap, + append: FxHashMap>>, + remove: FxHashSet, + ) -> Self { + let mut this = Converter { + current: None, + preorder: node.preorder_with_tokens(), + range: node.text_range(), + punct_offset: None, + map, + append, + remove, + current_leafs: vec![], + }; + let first = this.next_token(); + this.current = first; + this + } + + fn next_token(&mut self) -> Option { + // while let Some(ev) = self.preorder.next() { + // match ev { + // WalkEvent::Enter(SyntaxElement::Token(t)) => { + // if let Some(leafs) = self.append.remove(&t.clone().into()) { + // self.current_leafs.extend(leafs); + // } + // return Some(t); + // } + // WalkEvent::Enter(SyntaxElement::Node(n)) if self.remove.contains(&n) => { + // self.preorder.skip_subtree(); + // if let Some(leafs) = self.append.remove(&n.into()) { + // self.current_leafs.extend(leafs); + // } + // } + // _ => (), + // } + // } + // None; + + while let Some(ev) = self.preorder.next() { match ev { WalkEvent::Enter(SyntaxElement::Token(t)) => return Some(t), - WalkEvent::Enter(SyntaxElement::Node(n)) if censor.contains(&n) => { - preorder.skip_subtree() + WalkEvent::Enter(SyntaxElement::Node(n)) if self.remove.contains(&n) => { + self.preorder.skip_subtree(); + if let Some(mut v) = self.append.remove(&n.into()) { + v.reverse(); + self.current_leafs.extend(v); + return None; + } + } + WalkEvent::Enter(SyntaxElement::Node(_)) => (), + WalkEvent::Leave(ele) => { + if let Some(mut v) = self.append.remove(&ele) { + v.reverse(); + self.current_leafs.extend(v); + return None; + } } - _ => (), } } None @@ -626,45 +682,62 @@ impl Converter { } #[derive(Debug)] -enum SynToken { +enum SynToken { Ordinary(SyntaxToken), - Punct(SyntaxToken, usize), + Punct { token: SyntaxToken, offset: usize }, + Leaf(tt::Leaf), } -impl SynToken { +impl SynToken { fn token(&self) -> &SyntaxToken { match self { - SynToken::Ordinary(it) | SynToken::Punct(it, _) => it, + SynToken::Ordinary(it) | SynToken::Punct { token: it, offset: _ } => it, + SynToken::Leaf(_) => unreachable!(), } } } -impl SrcToken> for SynToken { - fn kind(&self, ctx: &Converter) -> SyntaxKind { +impl SrcToken, S> for SynToken { + fn kind(&self, ctx: &Converter) -> SyntaxKind { match self { SynToken::Ordinary(token) => token.kind(), - SynToken::Punct(..) => SyntaxKind::from_char(self.to_char(ctx).unwrap()).unwrap(), + SynToken::Punct { .. } => SyntaxKind::from_char(self.to_char(ctx).unwrap()).unwrap(), + SynToken::Leaf(_) => { + never!(); + SyntaxKind::ERROR + } } } - fn to_char(&self, _ctx: &Converter) -> Option { + fn to_char(&self, _ctx: &Converter) -> Option { match self { SynToken::Ordinary(_) => None, - SynToken::Punct(it, i) => it.text().chars().nth(*i), + SynToken::Punct { token: it, offset: i } => it.text().chars().nth(*i), + SynToken::Leaf(_) => None, + } + } + fn to_text(&self, _ctx: &Converter) -> SmolStr { + match self { + SynToken::Ordinary(token) | SynToken::Punct { token, offset: _ } => token.text().into(), + SynToken::Leaf(_) => { + never!(); + "".into() + } } } - fn to_text(&self, _ctx: &Converter) -> SmolStr { + fn as_leaf(&self) -> Option<&tt::Leaf> { match self { - SynToken::Ordinary(token) | SynToken::Punct(token, _) => token.text().into(), + SynToken::Ordinary(_) | SynToken::Punct { .. } => None, + SynToken::Leaf(it) => Some(it), } } } -impl TokenConverter for Converter +impl TokenConverter for Converter where S: Span, SpanMap: SpanMapper, { - type Token = SynToken; + type Token = SynToken; fn convert_doc_comment(&self, token: &Self::Token, span: S) -> Option>> { convert_doc_comment(token.token(), span) } @@ -676,20 +749,31 @@ where let range = punct.text_range(); self.punct_offset = Some((punct.clone(), offset)); let range = TextRange::at(range.start() + offset, TextSize::of('.')); - return Some((SynToken::Punct(punct, u32::from(offset) as usize), range)); + return Some(( + SynToken::Punct { token: punct, offset: u32::from(offset) as usize }, + range, + )); + } + } + + if let Some(leaf) = self.current_leafs.pop() { + if self.current_leafs.is_empty() { + self.current = self.next_token(); } + return Some((SynToken::Leaf(leaf), TextRange::empty(TextSize::new(0)))); } let curr = self.current.clone()?; if !self.range.contains_range(curr.text_range()) { return None; } - self.current = Self::next_token(&mut self.preorder, &self.censored); + + self.current = self.next_token(); let token = if curr.kind().is_punct() { self.punct_offset = Some((curr.clone(), 0.into())); let range = curr.text_range(); let range = TextRange::at(range.start(), TextSize::of('.')); - (SynToken::Punct(curr, 0 as usize), range) + (SynToken::Punct { token: curr, offset: 0 as usize }, range) } else { self.punct_offset = None; let range = curr.text_range(); @@ -703,7 +787,7 @@ where if let Some((punct, mut offset)) = self.punct_offset.clone() { offset += TextSize::of('.'); if usize::from(offset) < punct.text().len() { - return Some(SynToken::Punct(punct, usize::from(offset))); + return Some(SynToken::Punct { token: punct, offset: usize::from(offset) }); } } @@ -713,7 +797,7 @@ where } let token = if curr.kind().is_punct() { - SynToken::Punct(curr, 0 as usize) + SynToken::Punct { token: curr, offset: 0 as usize } } else { SynToken::Ordinary(curr) }; From 0003e568cacaa78d043cde29610182b931100109 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Fri, 1 Dec 2023 13:56:25 +0100 Subject: [PATCH 17/24] Pass calling span through to builtin macro expansions --- crates/hir-def/src/data.rs | 2 +- crates/hir-def/src/lib.rs | 8 +- .../hir-def/src/macro_expansion_tests/mbe.rs | 2 +- crates/hir-def/src/nameres/collector.rs | 28 +- crates/hir-expand/src/attrs.rs | 4 +- crates/hir-expand/src/builtin_attr_macro.rs | 30 +- crates/hir-expand/src/builtin_derive_macro.rs | 363 ++++++++++-------- crates/hir-expand/src/builtin_fn_macro.rs | 158 +++++--- crates/hir-expand/src/db.rs | 35 +- crates/hir-expand/src/eager.rs | 13 +- crates/hir-expand/src/lib.rs | 5 +- crates/hir-expand/src/proc_macro.rs | 18 +- crates/hir-expand/src/quote.rs | 150 ++++---- crates/load-cargo/src/lib.rs | 5 +- crates/mbe/src/expander.rs | 2 +- crates/mbe/src/expander/matcher.rs | 11 +- crates/mbe/src/expander/transcriber.rs | 22 +- crates/mbe/src/lib.rs | 6 +- crates/mbe/src/syntax_bridge.rs | 45 +-- crates/mbe/src/tt_iter.rs | 2 +- crates/proc-macro-api/src/msg.rs | 4 +- crates/proc-macro-api/src/msg/flat.rs | 10 - crates/proc-macro-srv/src/server.rs | 24 +- crates/proc-macro-srv/src/tests/mod.rs | 98 ++--- crates/proc-macro-srv/src/tests/utils.rs | 11 +- crates/tt/src/lib.rs | 61 ++- crates/vfs/src/lib.rs | 5 + 27 files changed, 624 insertions(+), 498 deletions(-) diff --git a/crates/hir-def/src/data.rs b/crates/hir-def/src/data.rs index 9986870d9dce..2af4622a0724 100644 --- a/crates/hir-def/src/data.rs +++ b/crates/hir-def/src/data.rs @@ -663,7 +663,7 @@ impl<'a> AssocItemCollector<'a> { self.module_id.local_id, MacroCallKind::Attr { ast_id, - attr_args: Arc::new(tt::Subtree::empty()), + attr_args: None, invoc_attr_index: attr.id, }, attr.path().clone(), diff --git a/crates/hir-def/src/lib.rs b/crates/hir-def/src/lib.rs index f9374347c264..35fb10e465e0 100644 --- a/crates/hir-def/src/lib.rs +++ b/crates/hir-def/src/lib.rs @@ -1351,11 +1351,11 @@ fn attr_macro_as_call_id( let arg = match macro_attr.input.as_deref() { Some(AttrInput::TokenTree(tt)) => { let mut tt = tt.as_ref().clone(); - tt.delimiter = tt::Delimiter::UNSPECIFIED; - tt + tt.delimiter = tt::Delimiter::DUMMY_INVISIBLE; + Some(tt) } - _ => tt::Subtree::empty(), + _ => None, }; def.as_lazy_macro( @@ -1363,7 +1363,7 @@ fn attr_macro_as_call_id( krate, MacroCallKind::Attr { ast_id: item_attr.ast_id, - attr_args: Arc::new(arg), + attr_args: arg.map(Arc::new), invoc_attr_index: macro_attr.id, }, macro_attr.ctxt, diff --git a/crates/hir-def/src/macro_expansion_tests/mbe.rs b/crates/hir-def/src/macro_expansion_tests/mbe.rs index 39079685a4d9..9f0d3938b7ea 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe.rs @@ -138,7 +138,7 @@ macro_rules! identity { } fn main(foo: ()) { - builtin#FileId(0):0@0..0\0# ##FileId(0):0@0..0\0#format_args#FileId(0):0@0..0\0# (#FileId(0):3@56..57\0#"{} {} {}"#FileId(0):3@57..67\0#,#FileId(0):3@67..68\0# format_args#FileId(0):3@69..80\0#!#FileId(0):3@80..81\0#(#FileId(0):3@81..82\0#"{}"#FileId(0):3@82..86\0#,#FileId(0):3@86..87\0# 0#FileId(0):3@88..89\0#)#FileId(0):3@89..90\0#,#FileId(0):3@90..91\0# foo#FileId(0):3@92..95\0#,#FileId(0):3@95..96\0# identity#FileId(0):3@97..105\0#!#FileId(0):3@105..106\0#(#FileId(0):3@106..107\0#10#FileId(0):3@107..109\0#)#FileId(0):3@109..110\0#,#FileId(0):3@110..111\0# "bar"#FileId(0):3@112..117\0#)#FileId(0):3@117..118\0# + builtin#FileId(0):3@23..118\3# ##FileId(0):3@23..118\3#format_args#FileId(0):3@23..118\3# (#FileId(0):3@56..57\0#"{} {} {}"#FileId(0):3@57..67\0#,#FileId(0):3@67..68\0# format_args#FileId(0):3@69..80\0#!#FileId(0):3@80..81\0#(#FileId(0):3@81..82\0#"{}"#FileId(0):3@82..86\0#,#FileId(0):3@86..87\0# 0#FileId(0):3@88..89\0#)#FileId(0):3@89..90\0#,#FileId(0):3@90..91\0# foo#FileId(0):3@92..95\0#,#FileId(0):3@95..96\0# identity#FileId(0):3@97..105\0#!#FileId(0):3@105..106\0#(#FileId(0):3@106..107\0#10#FileId(0):3@107..109\0#)#FileId(0):3@109..110\0#,#FileId(0):3@110..111\0# "bar"#FileId(0):3@112..117\0#)#FileId(0):3@117..118\0# } "##]], diff --git a/crates/hir-def/src/nameres/collector.rs b/crates/hir-def/src/nameres/collector.rs index 599010e542ca..b3a10a3869a4 100644 --- a/crates/hir-def/src/nameres/collector.rs +++ b/crates/hir-def/src/nameres/collector.rs @@ -5,7 +5,6 @@ use std::{cmp::Ordering, iter, mem}; -use ::tt::Span; use base_db::{span::SyntaxContextId, CrateId, Dependency, Edition, FileId}; use cfg::{CfgExpr, CfgOptions}; use either::Either; @@ -85,7 +84,17 @@ pub(super) fn collect_defs(db: &dyn DefDatabase, def_map: DefMap, tree_id: TreeI .enumerate() .map(|(idx, it)| { // FIXME: a hacky way to create a Name from string. - let name = tt::Ident { text: it.name.clone(), span: tt::SpanData::DUMMY }; + let name = tt::Ident { + text: it.name.clone(), + span: tt::SpanData { + range: syntax::TextRange::empty(syntax::TextSize::new(0)), + anchor: base_db::span::SpanAnchor { + file_id: FileId::BOGUS, + ast_id: base_db::span::ROOT_ERASED_FILE_AST_ID, + }, + ctx: SyntaxContextId::ROOT, + }, + }; (name.as_name(), ProcMacroExpander::new(base_db::ProcMacroId(idx as u32))) }) .collect()) @@ -476,7 +485,7 @@ impl DefCollector<'_> { directive.module_id, MacroCallKind::Attr { ast_id: ast_id.ast_id, - attr_args: Arc::new(tt::Subtree::empty()), + attr_args: None, invoc_attr_index: attr.id, }, attr.path().clone(), @@ -2079,7 +2088,18 @@ impl ModCollector<'_, '_> { let name = match attrs.by_key("rustc_builtin_macro").string_value() { Some(it) => { // FIXME: a hacky way to create a Name from string. - name = tt::Ident { text: it.clone(), span: tt::SpanData::DUMMY }.as_name(); + name = tt::Ident { + text: it.clone(), + span: tt::SpanData { + range: syntax::TextRange::empty(syntax::TextSize::new(0)), + anchor: base_db::span::SpanAnchor { + file_id: FileId::BOGUS, + ast_id: base_db::span::ROOT_ERASED_FILE_AST_ID, + }, + ctx: SyntaxContextId::ROOT, + }, + } + .as_name(); &name } None => { diff --git a/crates/hir-expand/src/attrs.rs b/crates/hir-expand/src/attrs.rs index 23a8fffa8c29..edaf2f06a4e8 100644 --- a/crates/hir-expand/src/attrs.rs +++ b/crates/hir-expand/src/attrs.rs @@ -130,7 +130,7 @@ impl RawAttrs { let attrs = parts.enumerate().take(1 << AttrId::CFG_ATTR_BITS).filter_map( |(idx, attr)| { let tree = Subtree { - delimiter: tt::Delimiter::unspecified(), + delimiter: tt::Delimiter::dummy_invisible(), token_trees: attr.to_vec(), }; Attr::from_tt(db, &tree, index.with_cfg_attr(idx)) @@ -296,7 +296,7 @@ impl Attr { // FIXME: This is necessarily a hack. It'd be nice if we could avoid allocation // here or maybe just parse a mod path from a token tree directly let subtree = tt::Subtree { - delimiter: tt::Delimiter::unspecified(), + delimiter: tt::Delimiter::dummy_invisible(), token_trees: tts.to_vec(), }; let (parse, span_map) = diff --git a/crates/hir-expand/src/builtin_attr_macro.rs b/crates/hir-expand/src/builtin_attr_macro.rs index c16b881df826..de58a495fef4 100644 --- a/crates/hir-expand/src/builtin_attr_macro.rs +++ b/crates/hir-expand/src/builtin_attr_macro.rs @@ -1,18 +1,22 @@ //! Builtin attributes. -use ::tt::Span; +use base_db::{ + span::{SyntaxContextId, ROOT_ERASED_FILE_AST_ID}, + FileId, +}; +use syntax::{TextRange, TextSize}; use crate::{db::ExpandDatabase, name, tt, ExpandResult, MacroCallId, MacroCallKind}; macro_rules! register_builtin { - ( $(($name:ident, $variant:ident) => $expand:ident),* ) => { + ($expand_fn:ident: $(($name:ident, $variant:ident) => $expand:ident),* ) => { #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum BuiltinAttrExpander { $($variant),* } impl BuiltinAttrExpander { - pub fn expand( + pub fn $expand_fn( &self, db: &dyn ExpandDatabase, id: MacroCallId, @@ -47,7 +51,7 @@ impl BuiltinAttrExpander { } } -register_builtin! { +register_builtin! { expand: (bench, Bench) => dummy_attr_expand, (cfg_accessible, CfgAccessible) => dummy_attr_expand, (cfg_eval, CfgEval) => dummy_attr_expand, @@ -99,21 +103,31 @@ fn derive_attr_expand( ) -> ExpandResult { let loc = db.lookup_intern_macro_call(id); let derives = match &loc.kind { - MacroCallKind::Attr { attr_args, .. } if loc.def.is_attribute_derive() => attr_args, - _ => return ExpandResult::ok(tt::Subtree::empty()), + MacroCallKind::Attr { attr_args: Some(attr_args), .. } if loc.def.is_attribute_derive() => { + attr_args + } + _ => return ExpandResult::ok(tt::Subtree::empty(tt::DelimSpan::DUMMY)), }; - pseudo_derive_attr_expansion(tt, derives) + pseudo_derive_attr_expansion(tt, derives, loc.call_site) } pub fn pseudo_derive_attr_expansion( tt: &tt::Subtree, args: &tt::Subtree, + call_site: SyntaxContextId, ) -> ExpandResult { let mk_leaf = |char| { tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char, spacing: tt::Spacing::Alone, - span: tt::SpanData::DUMMY, + span: tt::SpanData { + range: TextRange::empty(TextSize::new(0)), + anchor: base_db::span::SpanAnchor { + file_id: FileId::BOGUS, + ast_id: ROOT_ERASED_FILE_AST_ID, + }, + ctx: call_site, + }, })) }; diff --git a/crates/hir-expand/src/builtin_derive_macro.rs b/crates/hir-expand/src/builtin_derive_macro.rs index e9d137d99007..410aa4d289eb 100644 --- a/crates/hir-expand/src/builtin_derive_macro.rs +++ b/crates/hir-expand/src/builtin_derive_macro.rs @@ -1,6 +1,5 @@ //! Builtin derives. -use ::tt::Span; use base_db::{span::SpanData, CrateOrigin, LangCrateOrigin}; use itertools::izip; use rustc_hash::FxHashSet; @@ -74,19 +73,19 @@ enum VariantShape { Unit, } -fn tuple_field_iterator(n: usize) -> impl Iterator { - (0..n).map(|it| tt::Ident::new(format!("f{it}"), tt::SpanData::DUMMY)) +fn tuple_field_iterator(span: SpanData, n: usize) -> impl Iterator { + (0..n).map(move |it| tt::Ident::new(format!("f{it}"), span)) } impl VariantShape { - fn as_pattern(&self, path: tt::Subtree) -> tt::Subtree { - self.as_pattern_map(path, |it| quote!(#it)) + fn as_pattern(&self, path: tt::Subtree, span: SpanData) -> tt::Subtree { + self.as_pattern_map(path, span, |it| quote!(span => #it)) } - fn field_names(&self) -> Vec { + fn field_names(&self, span: SpanData) -> Vec { match self { VariantShape::Struct(s) => s.clone(), - VariantShape::Tuple(n) => tuple_field_iterator(*n).collect(), + VariantShape::Tuple(n) => tuple_field_iterator(span, *n).collect(), VariantShape::Unit => vec![], } } @@ -94,26 +93,27 @@ impl VariantShape { fn as_pattern_map( &self, path: tt::Subtree, + span: SpanData, field_map: impl Fn(&tt::Ident) -> tt::Subtree, ) -> tt::Subtree { match self { VariantShape::Struct(fields) => { let fields = fields.iter().map(|it| { let mapped = field_map(it); - quote! { #it : #mapped , } + quote! {span => #it : #mapped , } }); - quote! { + quote! {span => #path { ##fields } } } &VariantShape::Tuple(n) => { - let fields = tuple_field_iterator(n).map(|it| { + let fields = tuple_field_iterator(span, n).map(|it| { let mapped = field_map(&it); - quote! { + quote! {span => #mapped , } }); - quote! { + quote! {span => #path ( ##fields ) } } @@ -143,17 +143,17 @@ enum AdtShape { } impl AdtShape { - fn as_pattern(&self, name: &tt::Ident) -> Vec { - self.as_pattern_map(name, |it| quote!(#it)) + fn as_pattern(&self, span: SpanData, name: &tt::Ident) -> Vec { + self.as_pattern_map(name, |it| quote!(span =>#it), span) } - fn field_names(&self) -> Vec> { + fn field_names(&self, span: SpanData) -> Vec> { match self { AdtShape::Struct(s) => { - vec![s.field_names()] + vec![s.field_names(span)] } AdtShape::Enum { variants, .. } => { - variants.iter().map(|(_, fields)| fields.field_names()).collect() + variants.iter().map(|(_, fields)| fields.field_names(span)).collect() } AdtShape::Union => { never!("using fields of union in derive is always wrong"); @@ -166,18 +166,21 @@ impl AdtShape { &self, name: &tt::Ident, field_map: impl Fn(&tt::Ident) -> tt::Subtree, + span: SpanData, ) -> Vec { match self { AdtShape::Struct(s) => { - vec![s.as_pattern_map(quote! { #name }, field_map)] + vec![s.as_pattern_map(quote! {span => #name }, span, field_map)] } AdtShape::Enum { variants, .. } => variants .iter() - .map(|(v, fields)| fields.as_pattern_map(quote! { #name :: #v }, &field_map)) + .map(|(v, fields)| { + fields.as_pattern_map(quote! {span => #name :: #v }, span, &field_map) + }) .collect(), AdtShape::Union => { never!("pattern matching on union is always wrong"); - vec![quote! { un }] + vec![quote! {span => un }] } } } @@ -193,7 +196,11 @@ struct BasicAdtInfo { associated_types: Vec, } -fn parse_adt(tm: SpanMapRef<'_>, adt: &ast::Adt) -> Result { +fn parse_adt( + tm: SpanMapRef<'_>, + adt: &ast::Adt, + call_site: SpanData, +) -> Result { let (name, generic_param_list, shape) = match adt { ast::Adt::Struct(it) => ( it.name(), @@ -240,7 +247,9 @@ fn parse_adt(tm: SpanMapRef<'_>, adt: &ast::Adt) -> Result tt::Subtree::empty(), + None => { + tt::Subtree::empty(::tt::DelimSpan { open: call_site, close: call_site }) + } } }; let bounds = match ¶m { @@ -253,7 +262,9 @@ fn parse_adt(tm: SpanMapRef<'_>, adt: &ast::Adt) -> Result, trait_path: tt::Subtree, make_trait_body: impl FnOnce(&BasicAdtInfo) -> tt::Subtree, ) -> ExpandResult { - let info = match parse_adt(tm, tt) { + let info = match parse_adt(tm, tt, invoc_span) { Ok(info) => info, - Err(e) => return ExpandResult::new(tt::Subtree::empty(), e), + Err(e) => { + return ExpandResult::new( + tt::Subtree::empty(tt::DelimSpan { open: invoc_span, close: invoc_span }), + e, + ) + } }; let trait_body = make_trait_body(&info); let mut where_block = vec![]; @@ -357,13 +373,13 @@ fn expand_simple_derive( let ident_ = ident.clone(); if let Some(b) = bound { let ident = ident.clone(); - where_block.push(quote! { #ident : #b , }); + where_block.push(quote! {invoc_span => #ident : #b , }); } if let Some(ty) = param_ty { - (quote! { const #ident : #ty , }, quote! { #ident_ , }) + (quote! {invoc_span => const #ident : #ty , }, quote! {invoc_span => #ident_ , }) } else { let bound = trait_path.clone(); - (quote! { #ident : #bound , }, quote! { #ident_ , }) + (quote! {invoc_span => #ident : #bound , }, quote! {invoc_span => #ident_ , }) } }) .unzip(); @@ -371,17 +387,17 @@ fn expand_simple_derive( where_block.extend(info.associated_types.iter().map(|it| { let it = it.clone(); let bound = trait_path.clone(); - quote! { #it : #bound , } + quote! {invoc_span => #it : #bound , } })); let name = info.name; - let expanded = quote! { + let expanded = quote! {invoc_span => impl < ##params > #trait_path for #name < ##args > where ##where_block { #trait_body } }; ExpandResult::ok(expanded) } -fn find_builtin_crate(db: &dyn ExpandDatabase, id: MacroCallId) -> tt::TokenTree { +fn find_builtin_crate(db: &dyn ExpandDatabase, id: MacroCallId, span: SpanData) -> tt::TokenTree { // FIXME: make hygiene works for builtin derive macro // such that $crate can be used here. let cg = db.crate_graph(); @@ -389,9 +405,9 @@ fn find_builtin_crate(db: &dyn ExpandDatabase, id: MacroCallId) -> tt::TokenTree let tt = if matches!(cg[krate].origin, CrateOrigin::Lang(LangCrateOrigin::Core)) { cov_mark::hit!(test_copy_expand_in_core); - quote! { crate } + quote! {span => crate } } else { - quote! { core } + quote! {span => core } }; tt.token_trees[0].clone() @@ -404,8 +420,8 @@ fn copy_expand( tt: &ast::Adt, tm: SpanMapRef<'_>, ) -> ExpandResult { - let krate = find_builtin_crate(db, id); - expand_simple_derive(span, tt, tm, quote! { #krate::marker::Copy }, |_| quote! {}) + let krate = find_builtin_crate(db, id, span); + expand_simple_derive(span, tt, tm, quote! {span => #krate::marker::Copy }, |_| quote! {span =>}) } fn clone_expand( @@ -415,37 +431,35 @@ fn clone_expand( tt: &ast::Adt, tm: SpanMapRef<'_>, ) -> ExpandResult { - let krate = find_builtin_crate(db, id); - expand_simple_derive(span, tt, tm, quote! { #krate::clone::Clone }, |adt| { + let krate = find_builtin_crate(db, id, span); + expand_simple_derive(span, tt, tm, quote! {span => #krate::clone::Clone }, |adt| { if matches!(adt.shape, AdtShape::Union) { - let star = - tt::Punct { char: '*', spacing: ::tt::Spacing::Alone, span: tt::SpanData::DUMMY }; - return quote! { + let star = tt::Punct { char: '*', spacing: ::tt::Spacing::Alone, span }; + return quote! {span => fn clone(&self) -> Self { #star self } }; } if matches!(&adt.shape, AdtShape::Enum { variants, .. } if variants.is_empty()) { - let star = - tt::Punct { char: '*', spacing: ::tt::Spacing::Alone, span: tt::SpanData::DUMMY }; - return quote! { + let star = tt::Punct { char: '*', spacing: ::tt::Spacing::Alone, span }; + return quote! {span => fn clone(&self) -> Self { match #star self {} } }; } let name = &adt.name; - let patterns = adt.shape.as_pattern(name); - let exprs = adt.shape.as_pattern_map(name, |it| quote! { #it .clone() }); + let patterns = adt.shape.as_pattern(span, name); + let exprs = adt.shape.as_pattern_map(name, |it| quote! {span => #it .clone() }, span); let arms = patterns.into_iter().zip(exprs.into_iter()).map(|(pat, expr)| { - let fat_arrow = fat_arrow(); - quote! { + let fat_arrow = fat_arrow(span); + quote! {span => #pat #fat_arrow #expr, } }); - quote! { + quote! {span => fn clone(&self) -> Self { match self { ##arms @@ -455,16 +469,16 @@ fn clone_expand( }) } -/// This function exists since `quote! { => }` doesn't work. -fn fat_arrow() -> tt::Subtree { - let eq = tt::Punct { char: '=', spacing: ::tt::Spacing::Joint, span: tt::SpanData::DUMMY }; - quote! { #eq> } +/// This function exists since `quote! {span => => }` doesn't work. +fn fat_arrow(span: SpanData) -> tt::Subtree { + let eq = tt::Punct { char: '=', spacing: ::tt::Spacing::Joint, span }; + quote! {span => #eq> } } -/// This function exists since `quote! { && }` doesn't work. -fn and_and() -> tt::Subtree { - let and = tt::Punct { char: '&', spacing: ::tt::Spacing::Joint, span: tt::SpanData::DUMMY }; - quote! { #and& } +/// This function exists since `quote! {span => && }` doesn't work. +fn and_and(span: SpanData) -> tt::Subtree { + let and = tt::Punct { char: '&', spacing: ::tt::Spacing::Joint, span }; + quote! {span => #and& } } fn default_expand( @@ -474,33 +488,37 @@ fn default_expand( tt: &ast::Adt, tm: SpanMapRef<'_>, ) -> ExpandResult { - let krate = &find_builtin_crate(db, id); - expand_simple_derive(span, tt, tm, quote! { #krate::default::Default }, |adt| { + let krate = &find_builtin_crate(db, id, span); + expand_simple_derive(span, tt, tm, quote! {span => #krate::default::Default }, |adt| { let body = match &adt.shape { AdtShape::Struct(fields) => { let name = &adt.name; - fields - .as_pattern_map(quote!(#name), |_| quote!(#krate::default::Default::default())) + fields.as_pattern_map( + quote!(span =>#name), + span, + |_| quote!(span =>#krate::default::Default::default()), + ) } AdtShape::Enum { default_variant, variants } => { if let Some(d) = default_variant { let (name, fields) = &variants[*d]; let adt_name = &adt.name; fields.as_pattern_map( - quote!(#adt_name :: #name), - |_| quote!(#krate::default::Default::default()), + quote!(span =>#adt_name :: #name), + span, + |_| quote!(span =>#krate::default::Default::default()), ) } else { // FIXME: Return expand error here - quote!() + quote!(span =>) } } AdtShape::Union => { // FIXME: Return expand error here - quote!() + quote!(span =>) } }; - quote! { + quote! {span => fn default() -> Self { #body } @@ -515,38 +533,37 @@ fn debug_expand( tt: &ast::Adt, tm: SpanMapRef<'_>, ) -> ExpandResult { - let krate = &find_builtin_crate(db, id); - expand_simple_derive(span, tt, tm, quote! { #krate::fmt::Debug }, |adt| { + let krate = &find_builtin_crate(db, id, span); + expand_simple_derive(span, tt, tm, quote! {span => #krate::fmt::Debug }, |adt| { let for_variant = |name: String, v: &VariantShape| match v { VariantShape::Struct(fields) => { let for_fields = fields.iter().map(|it| { let x_string = it.to_string(); - quote! { + quote! {span => .field(#x_string, & #it) } }); - quote! { + quote! {span => f.debug_struct(#name) ##for_fields .finish() } } VariantShape::Tuple(n) => { - let for_fields = tuple_field_iterator(*n).map(|it| { - quote! { + let for_fields = tuple_field_iterator(span, *n).map(|it| { + quote! {span => .field( & #it) } }); - quote! { + quote! {span => f.debug_tuple(#name) ##for_fields .finish() } } - VariantShape::Unit => quote! { + VariantShape::Unit => quote! {span => f.write_str(#name) }, }; if matches!(&adt.shape, AdtShape::Enum { variants, .. } if variants.is_empty()) { - let star = - tt::Punct { char: '*', spacing: ::tt::Spacing::Alone, span: tt::SpanData::DUMMY }; - return quote! { + let star = tt::Punct { char: '*', spacing: ::tt::Spacing::Alone, span }; + return quote! {span => fn fmt(&self, f: &mut #krate::fmt::Formatter) -> #krate::fmt::Result { match #star self {} } @@ -554,20 +571,20 @@ fn debug_expand( } let arms = match &adt.shape { AdtShape::Struct(fields) => { - let fat_arrow = fat_arrow(); + let fat_arrow = fat_arrow(span); let name = &adt.name; - let pat = fields.as_pattern(quote!(#name)); + let pat = fields.as_pattern(quote!(span =>#name), span); let expr = for_variant(name.to_string(), fields); - vec![quote! { #pat #fat_arrow #expr }] + vec![quote! {span => #pat #fat_arrow #expr }] } AdtShape::Enum { variants, .. } => variants .iter() .map(|(name, v)| { - let fat_arrow = fat_arrow(); + let fat_arrow = fat_arrow(span); let adt_name = &adt.name; - let pat = v.as_pattern(quote!(#adt_name :: #name)); + let pat = v.as_pattern(quote!(span =>#adt_name :: #name), span); let expr = for_variant(name.to_string(), v); - quote! { + quote! {span => #pat #fat_arrow #expr , } }) @@ -577,7 +594,7 @@ fn debug_expand( vec![] } }; - quote! { + quote! {span => fn fmt(&self, f: &mut #krate::fmt::Formatter) -> #krate::fmt::Result { match self { ##arms @@ -594,41 +611,42 @@ fn hash_expand( tt: &ast::Adt, tm: SpanMapRef<'_>, ) -> ExpandResult { - let krate = &find_builtin_crate(db, id); - expand_simple_derive(span, tt, tm, quote! { #krate::hash::Hash }, |adt| { + let krate = &find_builtin_crate(db, id, span); + expand_simple_derive(span, tt, tm, quote! {span => #krate::hash::Hash }, |adt| { if matches!(adt.shape, AdtShape::Union) { // FIXME: Return expand error here - return quote! {}; + return quote! {span =>}; } if matches!(&adt.shape, AdtShape::Enum { variants, .. } if variants.is_empty()) { - let star = - tt::Punct { char: '*', spacing: ::tt::Spacing::Alone, span: tt::SpanData::DUMMY }; - return quote! { + let star = tt::Punct { char: '*', spacing: ::tt::Spacing::Alone, span }; + return quote! {span => fn hash(&self, ra_expand_state: &mut H) { match #star self {} } }; } - let arms = adt.shape.as_pattern(&adt.name).into_iter().zip(adt.shape.field_names()).map( - |(pat, names)| { - let expr = { - let it = names.iter().map(|it| quote! { #it . hash(ra_expand_state); }); - quote! { { - ##it - } } - }; - let fat_arrow = fat_arrow(); - quote! { - #pat #fat_arrow #expr , - } - }, - ); + let arms = + adt.shape.as_pattern(span, &adt.name).into_iter().zip(adt.shape.field_names(span)).map( + |(pat, names)| { + let expr = { + let it = + names.iter().map(|it| quote! {span => #it . hash(ra_expand_state); }); + quote! {span => { + ##it + } } + }; + let fat_arrow = fat_arrow(span); + quote! {span => + #pat #fat_arrow #expr , + } + }, + ); let check_discriminant = if matches!(&adt.shape, AdtShape::Enum { .. }) { - quote! { #krate::mem::discriminant(self).hash(ra_expand_state); } + quote! {span => #krate::mem::discriminant(self).hash(ra_expand_state); } } else { - quote! {} + quote! {span =>} }; - quote! { + quote! {span => fn hash(&self, ra_expand_state: &mut H) { #check_discriminant match self { @@ -646,8 +664,8 @@ fn eq_expand( tt: &ast::Adt, tm: SpanMapRef<'_>, ) -> ExpandResult { - let krate = find_builtin_crate(db, id); - expand_simple_derive(span, tt, tm, quote! { #krate::cmp::Eq }, |_| quote! {}) + let krate = find_builtin_crate(db, id, span); + expand_simple_derive(span, tt, tm, quote! {span => #krate::cmp::Eq }, |_| quote! {span =>}) } fn partial_eq_expand( @@ -657,43 +675,43 @@ fn partial_eq_expand( tt: &ast::Adt, tm: SpanMapRef<'_>, ) -> ExpandResult { - let krate = find_builtin_crate(db, id); - expand_simple_derive(span, tt, tm, quote! { #krate::cmp::PartialEq }, |adt| { + let krate = find_builtin_crate(db, id, span); + expand_simple_derive(span, tt, tm, quote! {span => #krate::cmp::PartialEq }, |adt| { if matches!(adt.shape, AdtShape::Union) { // FIXME: Return expand error here - return quote! {}; + return quote! {span =>}; } let name = &adt.name; - let (self_patterns, other_patterns) = self_and_other_patterns(adt, name); - let arms = izip!(self_patterns, other_patterns, adt.shape.field_names()).map( + let (self_patterns, other_patterns) = self_and_other_patterns(adt, name, span); + let arms = izip!(self_patterns, other_patterns, adt.shape.field_names(span)).map( |(pat1, pat2, names)| { - let fat_arrow = fat_arrow(); + let fat_arrow = fat_arrow(span); let body = match &*names { [] => { - quote!(true) + quote!(span =>true) } [first, rest @ ..] => { let rest = rest.iter().map(|it| { let t1 = tt::Ident::new(format!("{}_self", it.text), it.span); let t2 = tt::Ident::new(format!("{}_other", it.text), it.span); - let and_and = and_and(); - quote!(#and_and #t1 .eq( #t2 )) + let and_and = and_and(span); + quote!(span =>#and_and #t1 .eq( #t2 )) }); let first = { let t1 = tt::Ident::new(format!("{}_self", first.text), first.span); let t2 = tt::Ident::new(format!("{}_other", first.text), first.span); - quote!(#t1 .eq( #t2 )) + quote!(span =>#t1 .eq( #t2 )) }; - quote!(#first ##rest) + quote!(span =>#first ##rest) } }; - quote! { ( #pat1 , #pat2 ) #fat_arrow #body , } + quote! {span => ( #pat1 , #pat2 ) #fat_arrow #body , } }, ); - let fat_arrow = fat_arrow(); - quote! { + let fat_arrow = fat_arrow(span); + quote! {span => fn eq(&self, other: &Self) -> bool { match (self, other) { ##arms @@ -707,15 +725,24 @@ fn partial_eq_expand( fn self_and_other_patterns( adt: &BasicAdtInfo, name: &tt::Ident, + span: SpanData, ) -> (Vec, Vec) { - let self_patterns = adt.shape.as_pattern_map(name, |it| { - let t = tt::Ident::new(format!("{}_self", it.text), it.span); - quote!(#t) - }); - let other_patterns = adt.shape.as_pattern_map(name, |it| { - let t = tt::Ident::new(format!("{}_other", it.text), it.span); - quote!(#t) - }); + let self_patterns = adt.shape.as_pattern_map( + name, + |it| { + let t = tt::Ident::new(format!("{}_self", it.text), it.span); + quote!(span =>#t) + }, + span, + ); + let other_patterns = adt.shape.as_pattern_map( + name, + |it| { + let t = tt::Ident::new(format!("{}_other", it.text), it.span); + quote!(span =>#t) + }, + span, + ); (self_patterns, other_patterns) } @@ -726,17 +753,18 @@ fn ord_expand( tt: &ast::Adt, tm: SpanMapRef<'_>, ) -> ExpandResult { - let krate = &find_builtin_crate(db, id); - expand_simple_derive(span, tt, tm, quote! { #krate::cmp::Ord }, |adt| { + let krate = &find_builtin_crate(db, id, span); + expand_simple_derive(span, tt, tm, quote! {span => #krate::cmp::Ord }, |adt| { fn compare( krate: &tt::TokenTree, left: tt::Subtree, right: tt::Subtree, rest: tt::Subtree, + span: SpanData, ) -> tt::Subtree { - let fat_arrow1 = fat_arrow(); - let fat_arrow2 = fat_arrow(); - quote! { + let fat_arrow1 = fat_arrow(span); + let fat_arrow2 = fat_arrow(span); + quote! {span => match #left.cmp(&#right) { #krate::cmp::Ordering::Equal #fat_arrow1 { #rest @@ -747,34 +775,34 @@ fn ord_expand( } if matches!(adt.shape, AdtShape::Union) { // FIXME: Return expand error here - return quote!(); + return quote!(span =>); } - let (self_patterns, other_patterns) = self_and_other_patterns(adt, &adt.name); - let arms = izip!(self_patterns, other_patterns, adt.shape.field_names()).map( + let (self_patterns, other_patterns) = self_and_other_patterns(adt, &adt.name, span); + let arms = izip!(self_patterns, other_patterns, adt.shape.field_names(span)).map( |(pat1, pat2, fields)| { - let mut body = quote!(#krate::cmp::Ordering::Equal); + let mut body = quote!(span =>#krate::cmp::Ordering::Equal); for f in fields.into_iter().rev() { let t1 = tt::Ident::new(format!("{}_self", f.text), f.span); let t2 = tt::Ident::new(format!("{}_other", f.text), f.span); - body = compare(krate, quote!(#t1), quote!(#t2), body); + body = compare(krate, quote!(span =>#t1), quote!(span =>#t2), body, span); } - let fat_arrow = fat_arrow(); - quote! { ( #pat1 , #pat2 ) #fat_arrow #body , } + let fat_arrow = fat_arrow(span); + quote! {span => ( #pat1 , #pat2 ) #fat_arrow #body , } }, ); - let fat_arrow = fat_arrow(); - let mut body = quote! { + let fat_arrow = fat_arrow(span); + let mut body = quote! {span => match (self, other) { ##arms _unused #fat_arrow #krate::cmp::Ordering::Equal } }; if matches!(&adt.shape, AdtShape::Enum { .. }) { - let left = quote!(#krate::intrinsics::discriminant_value(self)); - let right = quote!(#krate::intrinsics::discriminant_value(other)); - body = compare(krate, left, right, body); + let left = quote!(span =>#krate::intrinsics::discriminant_value(self)); + let right = quote!(span =>#krate::intrinsics::discriminant_value(other)); + body = compare(krate, left, right, body, span); } - quote! { + quote! {span => fn cmp(&self, other: &Self) -> #krate::cmp::Ordering { #body } @@ -789,17 +817,18 @@ fn partial_ord_expand( tt: &ast::Adt, tm: SpanMapRef<'_>, ) -> ExpandResult { - let krate = &find_builtin_crate(db, id); - expand_simple_derive(span, tt, tm, quote! { #krate::cmp::PartialOrd }, |adt| { + let krate = &find_builtin_crate(db, id, span); + expand_simple_derive(span, tt, tm, quote! {span => #krate::cmp::PartialOrd }, |adt| { fn compare( krate: &tt::TokenTree, left: tt::Subtree, right: tt::Subtree, rest: tt::Subtree, + span: SpanData, ) -> tt::Subtree { - let fat_arrow1 = fat_arrow(); - let fat_arrow2 = fat_arrow(); - quote! { + let fat_arrow1 = fat_arrow(span); + let fat_arrow2 = fat_arrow(span); + quote! {span => match #left.partial_cmp(&#right) { #krate::option::Option::Some(#krate::cmp::Ordering::Equal) #fat_arrow1 { #rest @@ -810,37 +839,39 @@ fn partial_ord_expand( } if matches!(adt.shape, AdtShape::Union) { // FIXME: Return expand error here - return quote!(); + return quote!(span =>); } - let left = quote!(#krate::intrinsics::discriminant_value(self)); - let right = quote!(#krate::intrinsics::discriminant_value(other)); + let left = quote!(span =>#krate::intrinsics::discriminant_value(self)); + let right = quote!(span =>#krate::intrinsics::discriminant_value(other)); - let (self_patterns, other_patterns) = self_and_other_patterns(adt, &adt.name); - let arms = izip!(self_patterns, other_patterns, adt.shape.field_names()).map( + let (self_patterns, other_patterns) = self_and_other_patterns(adt, &adt.name, span); + let arms = izip!(self_patterns, other_patterns, adt.shape.field_names(span)).map( |(pat1, pat2, fields)| { - let mut body = quote!(#krate::option::Option::Some(#krate::cmp::Ordering::Equal)); + let mut body = + quote!(span =>#krate::option::Option::Some(#krate::cmp::Ordering::Equal)); for f in fields.into_iter().rev() { let t1 = tt::Ident::new(format!("{}_self", f.text), f.span); let t2 = tt::Ident::new(format!("{}_other", f.text), f.span); - body = compare(krate, quote!(#t1), quote!(#t2), body); + body = compare(krate, quote!(span =>#t1), quote!(span =>#t2), body, span); } - let fat_arrow = fat_arrow(); - quote! { ( #pat1 , #pat2 ) #fat_arrow #body , } + let fat_arrow = fat_arrow(span); + quote! {span => ( #pat1 , #pat2 ) #fat_arrow #body , } }, ); - let fat_arrow = fat_arrow(); + let fat_arrow = fat_arrow(span); let body = compare( krate, left, right, - quote! { + quote! {span => match (self, other) { ##arms _unused #fat_arrow #krate::option::Option::Some(#krate::cmp::Ordering::Equal) } }, + span, ); - quote! { + quote! {span => fn partial_cmp(&self, other: &Self) -> #krate::option::Option::Option<#krate::cmp::Ordering> { #body } diff --git a/crates/hir-expand/src/builtin_fn_macro.rs b/crates/hir-expand/src/builtin_fn_macro.rs index 459d1c868d80..726b9835369f 100644 --- a/crates/hir-expand/src/builtin_fn_macro.rs +++ b/crates/hir-expand/src/builtin_fn_macro.rs @@ -1,8 +1,7 @@ //! Builtin macro -use ::tt::Span; use base_db::{ - span::{SpanAnchor, ROOT_ERASED_FILE_AST_ID}, + span::{SpanAnchor, SpanData, SyntaxContextId, ROOT_ERASED_FILE_AST_ID}, AnchoredPath, Edition, FileId, }; use cfg::CfgExpr; @@ -15,8 +14,9 @@ use syntax::{ use crate::{ db::ExpandDatabase, + hygiene::span_with_def_site_ctxt, name, quote, - tt::{self}, + tt::{self, DelimSpan}, EagerCallInfo, ExpandError, ExpandResult, HirFileIdExt, MacroCallId, MacroCallLoc, }; @@ -42,7 +42,10 @@ macro_rules! register_builtin { let expander = match *self { $( BuiltinFnLikeExpander::$kind => $expand, )* }; - expander(db, id, tt) + + let span = db.lookup_intern_macro_call(id).span(db); + let span = span_with_def_site_ctxt(db, span, id); + expander(db, id, tt, span) } } @@ -50,13 +53,16 @@ macro_rules! register_builtin { pub fn expand( &self, db: &dyn ExpandDatabase, - arg_id: MacroCallId, + id: MacroCallId, tt: &tt::Subtree, ) -> ExpandResult { let expander = match *self { $( EagerExpander::$e_kind => $e_expand, )* }; - expander(db, arg_id, tt) + + let span = db.lookup_intern_macro_call(id).span(db); + let span = span_with_def_site_ctxt(db, span, id); + expander(db, id, tt, span) } } @@ -115,29 +121,42 @@ register_builtin! { (option_env, OptionEnv) => option_env_expand } -const DOLLAR_CRATE: tt::Ident = - tt::Ident { text: SmolStr::new_inline("$crate"), span: tt::SpanData::DUMMY }; +fn mk_pound(span: SpanData) -> tt::Subtree { + crate::quote::IntoTt::to_subtree( + vec![crate::tt::Leaf::Punct(crate::tt::Punct { + char: '#', + spacing: crate::tt::Spacing::Alone, + span: span, + }) + .into()], + span, + ) +} fn module_path_expand( _db: &dyn ExpandDatabase, _id: MacroCallId, _tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { // Just return a dummy result. - ExpandResult::ok(quote! { "module::path" }) + ExpandResult::ok(quote! {span => + "module::path" + }) } fn line_expand( _db: &dyn ExpandDatabase, _id: MacroCallId, _tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { // dummy implementation for type-checking purposes ExpandResult::ok(tt::Subtree { - delimiter: tt::Delimiter::unspecified(), + delimiter: tt::Delimiter::dummy_invisible(), token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { text: "0u32".into(), - span: tt::SpanData::DUMMY, + span, }))], }) } @@ -146,26 +165,29 @@ fn log_syntax_expand( _db: &dyn ExpandDatabase, _id: MacroCallId, _tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { - ExpandResult::ok(quote! {}) + ExpandResult::ok(quote! {span =>}) } fn trace_macros_expand( _db: &dyn ExpandDatabase, _id: MacroCallId, _tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { - ExpandResult::ok(quote! {}) + ExpandResult::ok(quote! {span =>}) } fn stringify_expand( _db: &dyn ExpandDatabase, _id: MacroCallId, tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { let pretty = ::tt::pretty(&tt.token_trees); - let expanded = quote! { + let expanded = quote! {span => #pretty }; @@ -176,27 +198,29 @@ fn assert_expand( _db: &dyn ExpandDatabase, _id: MacroCallId, tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { let args = parse_exprs_with_sep(tt, ','); + let dollar_crate = tt::Ident { text: SmolStr::new_inline("$crate"), span }; let expanded = match &*args { [cond, panic_args @ ..] => { let comma = tt::Subtree { - delimiter: tt::Delimiter::unspecified(), + delimiter: tt::Delimiter::dummy_invisible(), token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', spacing: tt::Spacing::Alone, - span: tt::SpanData::DUMMY, + span, }))], }; let cond = cond.clone(); let panic_args = itertools::Itertools::intersperse(panic_args.iter().cloned(), comma); - quote! {{ + quote! {span =>{ if !(#cond) { - #DOLLAR_CRATE::panic!(##panic_args); + #dollar_crate::panic!(##panic_args); } }} } - [] => quote! {{}}, + [] => quote! {span =>{}}, }; ExpandResult::ok(expanded) @@ -206,12 +230,13 @@ fn file_expand( _db: &dyn ExpandDatabase, _id: MacroCallId, _tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { // FIXME: RA purposefully lacks knowledge of absolute file names // so just return "". let file_name = ""; - let expanded = quote! { + let expanded = quote! {span => #file_name }; @@ -222,16 +247,18 @@ fn format_args_expand( db: &dyn ExpandDatabase, id: MacroCallId, tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { - format_args_expand_general(db, id, tt, "") + format_args_expand_general(db, id, tt, "", span) } fn format_args_nl_expand( db: &dyn ExpandDatabase, id: MacroCallId, tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { - format_args_expand_general(db, id, tt, "\\n") + format_args_expand_general(db, id, tt, "\\n", span) } fn format_args_expand_general( @@ -240,11 +267,12 @@ fn format_args_expand_general( tt: &tt::Subtree, // FIXME: Make use of this so that mir interpretation works properly _end_string: &str, + span: SpanData, ) -> ExpandResult { - let pound = quote! {@PUNCT '#'}; + let pound = mk_pound(span); let mut tt = tt.clone(); tt.delimiter.kind = tt::DelimiterKind::Parenthesis; - return ExpandResult::ok(quote! { + return ExpandResult::ok(quote! {span => builtin #pound format_args #tt }); } @@ -253,25 +281,25 @@ fn asm_expand( _db: &dyn ExpandDatabase, _id: MacroCallId, tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { // We expand all assembly snippets to `format_args!` invocations to get format syntax // highlighting for them. - let mut literals = Vec::new(); for tt in tt.token_trees.chunks(2) { match tt { [tt::TokenTree::Leaf(tt::Leaf::Literal(lit))] | [tt::TokenTree::Leaf(tt::Leaf::Literal(lit)), tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', span: _, spacing: _ }))] => { - let krate = DOLLAR_CRATE.clone(); - literals.push(quote!(#krate::format_args!(#lit);)); + let dollar_krate = tt::Ident { text: SmolStr::new_inline("$crate"), span }; + literals.push(quote!(span=>#dollar_krate::format_args!(#lit);)); } _ => break, } } - let pound = quote! {@PUNCT '#'}; - let expanded = quote! { + let pound = mk_pound(span); + let expanded = quote! {span => builtin #pound asm ( {##literals} ) @@ -283,20 +311,22 @@ fn global_asm_expand( _db: &dyn ExpandDatabase, _id: MacroCallId, _tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { // Expand to nothing (at item-level) - ExpandResult::ok(quote! {}) + ExpandResult::ok(quote! {span =>}) } fn cfg_expand( db: &dyn ExpandDatabase, id: MacroCallId, tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { let loc = db.lookup_intern_macro_call(id); let expr = CfgExpr::parse(tt); let enabled = db.crate_graph()[loc.krate].cfg_options.check(&expr) != Some(false); - let expanded = if enabled { quote!(true) } else { quote!(false) }; + let expanded = if enabled { quote!(span=>true) } else { quote!(span=>false) }; ExpandResult::ok(expanded) } @@ -304,13 +334,15 @@ fn panic_expand( db: &dyn ExpandDatabase, id: MacroCallId, tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { let loc: MacroCallLoc = db.lookup_intern_macro_call(id); + let dollar_crate = tt::Ident { text: SmolStr::new_inline("$crate"), span }; // Expand to a macro call `$crate::panic::panic_{edition}` let mut call = if db.crate_graph()[loc.krate].edition >= Edition::Edition2021 { - quote!(#DOLLAR_CRATE::panic::panic_2021!) + quote!(span =>#dollar_crate::panic::panic_2021!) } else { - quote!(#DOLLAR_CRATE::panic::panic_2015!) + quote!(span =>#dollar_crate::panic::panic_2015!) }; // Pass the original arguments @@ -322,13 +354,15 @@ fn unreachable_expand( db: &dyn ExpandDatabase, id: MacroCallId, tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { let loc: MacroCallLoc = db.lookup_intern_macro_call(id); // Expand to a macro call `$crate::panic::unreachable_{edition}` + let dollar_crate = tt::Ident { text: SmolStr::new_inline("$crate"), span }; let mut call = if db.crate_graph()[loc.krate].edition >= Edition::Edition2021 { - quote!(#DOLLAR_CRATE::panic::unreachable_2021!) + quote!(span =>#dollar_crate::panic::unreachable_2021!) } else { - quote!(#DOLLAR_CRATE::panic::unreachable_2015!) + quote!(span =>#dollar_crate::panic::unreachable_2015!) }; // Pass the original arguments @@ -358,6 +392,7 @@ fn compile_error_expand( _db: &dyn ExpandDatabase, _id: MacroCallId, tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { let err = match &*tt.token_trees { [tt::TokenTree::Leaf(tt::Leaf::Literal(it))] => match unquote_str(it) { @@ -367,13 +402,14 @@ fn compile_error_expand( _ => ExpandError::other("`compile_error!` argument must be a string"), }; - ExpandResult { value: quote! {}, err: Some(err) } + ExpandResult { value: quote! {span =>}, err: Some(err) } } fn concat_expand( _db: &dyn ExpandDatabase, _arg_id: MacroCallId, tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { let mut err = None; let mut text = String::new(); @@ -413,13 +449,14 @@ fn concat_expand( } } } - ExpandResult { value: quote!(#text), err } + ExpandResult { value: quote!(span =>#text), err } } fn concat_bytes_expand( _db: &dyn ExpandDatabase, _arg_id: MacroCallId, tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { let mut bytes = Vec::new(); let mut err = None; @@ -452,8 +489,8 @@ fn concat_bytes_expand( } } } - let ident = tt::Ident { text: bytes.join(", ").into(), span: tt::SpanData::DUMMY }; - ExpandResult { value: quote!([#ident]), err } + let ident = tt::Ident { text: bytes.join(", ").into(), span }; + ExpandResult { value: quote!(span =>[#ident]), err } } fn concat_bytes_expand_subtree( @@ -486,6 +523,7 @@ fn concat_idents_expand( _db: &dyn ExpandDatabase, _arg_id: MacroCallId, tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { let mut err = None; let mut ident = String::new(); @@ -500,8 +538,9 @@ fn concat_idents_expand( } } } - let ident = tt::Ident { text: ident.into(), span: tt::SpanData::DUMMY }; - ExpandResult { value: quote!(#ident), err } + // FIXME merge spans + let ident = tt::Ident { text: ident.into(), span }; + ExpandResult { value: quote!(span =>#ident), err } } fn relative_file( @@ -537,10 +576,11 @@ fn include_expand( db: &dyn ExpandDatabase, arg_id: MacroCallId, _tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { match db.include_expand(arg_id) { Ok((res, _)) => ExpandResult::ok(res.as_ref().clone()), - Err(e) => ExpandResult::new(tt::Subtree::empty(), e), + Err(e) => ExpandResult::new(tt::Subtree::empty(DelimSpan { open: span, close: span }), e), } } @@ -559,6 +599,8 @@ pub(crate) fn include_arg_to_tt( // why are we not going through a SyntaxNode here? let subtree = parse_to_token_tree( SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, + // FIXME + SyntaxContextId::ROOT, &db.file_text(file_id), ) .ok_or(mbe::ExpandError::ConversionError)?; @@ -569,17 +611,18 @@ fn include_bytes_expand( _db: &dyn ExpandDatabase, _arg_id: MacroCallId, tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { if let Err(e) = parse_string(tt) { - return ExpandResult::new(tt::Subtree::empty(), e); + return ExpandResult::new(tt::Subtree::empty(DelimSpan { open: span, close: span }), e); } // FIXME: actually read the file here if the user asked for macro expansion let res = tt::Subtree { - delimiter: tt::Delimiter::unspecified(), + delimiter: tt::Delimiter::dummy_invisible(), token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { text: r#"b"""#.into(), - span: tt::SpanData::DUMMY, + span, }))], }; ExpandResult::ok(res) @@ -589,10 +632,13 @@ fn include_str_expand( db: &dyn ExpandDatabase, arg_id: MacroCallId, tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { let path = match parse_string(tt) { Ok(it) => it, - Err(e) => return ExpandResult::new(tt::Subtree::empty(), e), + Err(e) => { + return ExpandResult::new(tt::Subtree::empty(DelimSpan { open: span, close: span }), e) + } }; // FIXME: we're not able to read excluded files (which is most of them because @@ -602,14 +648,14 @@ fn include_str_expand( let file_id = match relative_file(db, arg_id, &path, true) { Ok(file_id) => file_id, Err(_) => { - return ExpandResult::ok(quote!("")); + return ExpandResult::ok(quote!(span =>"")); } }; let text = db.file_text(file_id); let text = &*text; - ExpandResult::ok(quote!(#text)) + ExpandResult::ok(quote!(span =>#text)) } fn get_env_inner(db: &dyn ExpandDatabase, arg_id: MacroCallId, key: &str) -> Option { @@ -621,10 +667,13 @@ fn env_expand( db: &dyn ExpandDatabase, arg_id: MacroCallId, tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { let key = match parse_string(tt) { Ok(it) => it, - Err(e) => return ExpandResult::new(tt::Subtree::empty(), e), + Err(e) => { + return ExpandResult::new(tt::Subtree::empty(DelimSpan { open: span, close: span }), e) + } }; let mut err = None; @@ -641,7 +690,7 @@ fn env_expand( // `include!("foo.rs"), which might go to infinite loop "UNRESOLVED_ENV_VAR".to_string() }); - let expanded = quote! { #s }; + let expanded = quote! {span => #s }; ExpandResult { value: expanded, err } } @@ -650,15 +699,18 @@ fn option_env_expand( db: &dyn ExpandDatabase, arg_id: MacroCallId, tt: &tt::Subtree, + span: SpanData, ) -> ExpandResult { let key = match parse_string(tt) { Ok(it) => it, - Err(e) => return ExpandResult::new(tt::Subtree::empty(), e), + Err(e) => { + return ExpandResult::new(tt::Subtree::empty(DelimSpan { open: span, close: span }), e) + } }; // FIXME: Use `DOLLAR_CRATE` when that works in eager macros. let expanded = match get_env_inner(db, arg_id, &key) { - None => quote! { ::core::option::Option::None::<&str> }, - Some(s) => quote! { ::core::option::Option::Some(#s) }, + None => quote! {span => ::core::option::Option::None::<&str> }, + Some(s) => quote! {span => ::core::option::Option::Some(#s) }, }; ExpandResult::ok(expanded) diff --git a/crates/hir-expand/src/db.rs b/crates/hir-expand/src/db.rs index 0135d97f1cd1..7f77a2f51bab 100644 --- a/crates/hir-expand/src/db.rs +++ b/crates/hir-expand/src/db.rs @@ -1,11 +1,6 @@ //! Defines database & queries for macro expansion. -use ::tt::{SpanAnchor as _, SyntaxContext}; -use base_db::{ - salsa, - span::{SpanAnchor, SyntaxContextId}, - CrateId, Edition, FileId, SourceDatabase, -}; +use base_db::{salsa, span::SyntaxContextId, CrateId, Edition, FileId, SourceDatabase}; use either::Either; use limit::Limit; use mbe::{syntax_node_to_token_tree, ValueResult}; @@ -53,7 +48,7 @@ impl DeclarativeMacroExpander { ) -> ExpandResult { match self.mac.err() { Some(e) => ExpandResult::new( - tt::Subtree::empty(), + tt::Subtree::empty(tt::DelimSpan::DUMMY), ExpandError::other(format!("invalid macro definition: {e}")), ), None => self @@ -66,7 +61,7 @@ impl DeclarativeMacroExpander { pub fn expand_unhygienic(&self, tt: tt::Subtree) -> ExpandResult { match self.mac.err() { Some(e) => ExpandResult::new( - tt::Subtree::empty(), + tt::Subtree::empty(tt::DelimSpan::DUMMY), ExpandError::other(format!("invalid macro definition: {e}")), ), None => self.mac.expand(&tt, |_| ()).map_err(Into::into), @@ -191,7 +186,7 @@ pub fn expand_speculative( ) -> Option<(SyntaxNode, SyntaxToken)> { let loc = db.lookup_intern_macro_call(actual_macro_call); - let span_map = RealSpanMap::absolute(SpanAnchor::DUMMY.file_id); + let span_map = RealSpanMap::absolute(FileId::BOGUS); let span_map = SpanMapRef::RealSpanMap(&span_map); // Build the subtree and token mapping for the speculative args @@ -235,7 +230,7 @@ pub fn expand_speculative( match attr.token_tree() { Some(token_tree) => { let mut tree = syntax_node_to_token_tree(token_tree.syntax(), span_map); - tree.delimiter = tt::Delimiter::UNSPECIFIED; + tree.delimiter = tt::Delimiter::DUMMY_INVISIBLE; Some(tree) } @@ -249,7 +244,7 @@ pub fn expand_speculative( // Otherwise the expand query will fetch the non speculative attribute args and pass those instead. let mut speculative_expansion = match loc.def.kind { MacroDefKind::ProcMacro(expander, ..) => { - tt.delimiter = tt::Delimiter::UNSPECIFIED; + tt.delimiter = tt::Delimiter::DUMMY_INVISIBLE; let call_site = loc.span(db); expander.expand( db, @@ -263,7 +258,7 @@ pub fn expand_speculative( ) } MacroDefKind::BuiltInAttr(BuiltinAttrExpander::Derive, _) => { - pseudo_derive_attr_expansion(&tt, attr_arg.as_ref()?) + pseudo_derive_attr_expansion(&tt, attr_arg.as_ref()?, loc.call_site) } MacroDefKind::BuiltInDerive(expander, ..) => { // this cast is a bit sus, can we avoid losing the typedness here? @@ -286,11 +281,7 @@ pub fn expand_speculative( let syntax_node = node.syntax_node(); let token = rev_tmap - .ranges_with_span(tt::SpanData { - range: token_to_map.text_range(), - anchor: SpanAnchor::DUMMY, - ctx: SyntaxContextId::DUMMY, - }) + .ranges_with_span(span_map.span_for_range(token_to_map.text_range())) .filter_map(|range| syntax_node.covering_element(range).into_token()) .min_by_key(|t| { // prefer tokens of the same kind and text @@ -453,7 +444,7 @@ fn macro_arg( if loc.def.is_proc_macro() { // proc macros expect their inputs without parentheses, MBEs expect it with them included - tt.delimiter = tt::Delimiter::UNSPECIFIED; + tt.delimiter = tt::Delimiter::DUMMY_INVISIBLE; } if matches!(loc.def.kind, MacroDefKind::BuiltInEager(..)) { @@ -611,7 +602,7 @@ fn macro_expand( let Some((macro_arg, undo_info)) = value else { return ExpandResult { value: Arc::new(tt::Subtree { - delimiter: tt::Delimiter::UNSPECIFIED, + delimiter: tt::Delimiter::DUMMY_INVISIBLE, token_trees: Vec::new(), }), // FIXME: We should make sure to enforce an invariant that invalid macro @@ -683,7 +674,7 @@ fn expand_proc_macro(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult ExpandResult Some(&**attr_args), + MacroCallKind::Attr { attr_args: Some(attr_args), .. } => Some(&**attr_args), _ => None, }; @@ -749,7 +740,7 @@ fn check_tt_count(tt: &tt::Subtree) -> Result<(), ExpandResult> if TOKEN_LIMIT.check(count).is_err() { Err(ExpandResult { value: Arc::new(tt::Subtree { - delimiter: tt::Delimiter::UNSPECIFIED, + delimiter: tt::Delimiter::DUMMY_INVISIBLE, token_trees: vec![], }), err: Some(ExpandError::other(format!( diff --git a/crates/hir-expand/src/eager.rs b/crates/hir-expand/src/eager.rs index deea59b93b62..aa537af14393 100644 --- a/crates/hir-expand/src/eager.rs +++ b/crates/hir-expand/src/eager.rs @@ -18,10 +18,7 @@ //! //! //! See the full discussion : -use base_db::{ - span::{SpanAnchor, SyntaxContextId}, - CrateId, -}; +use base_db::{span::SyntaxContextId, CrateId, FileId}; use rustc_hash::FxHashMap; use syntax::{ted, Parse, SyntaxNode, TextRange, TextSize, WalkEvent}; use triomphe::Arc; @@ -79,12 +76,10 @@ pub fn expand_eager_macro_input( }; // FIXME: Spans! - let mut subtree = mbe::syntax_node_to_token_tree( - &expanded_eager_input, - RealSpanMap::absolute(::DUMMY.file_id), - ); + let mut subtree = + mbe::syntax_node_to_token_tree(&expanded_eager_input, RealSpanMap::absolute(FileId::BOGUS)); - subtree.delimiter = crate::tt::Delimiter::UNSPECIFIED; + subtree.delimiter = crate::tt::Delimiter::DUMMY_INVISIBLE; let loc = MacroCallLoc { def, diff --git a/crates/hir-expand/src/lib.rs b/crates/hir-expand/src/lib.rs index 491c5e638ee1..5915fe6e3446 100644 --- a/crates/hir-expand/src/lib.rs +++ b/crates/hir-expand/src/lib.rs @@ -61,6 +61,7 @@ pub mod tt { pub use tt::{DelimiterKind, Spacing, Span, SpanAnchor}; pub type Delimiter = ::tt::Delimiter; + pub type DelimSpan = ::tt::DelimSpan; pub type Subtree = ::tt::Subtree; pub type Leaf = ::tt::Leaf; pub type Literal = ::tt::Literal; @@ -160,7 +161,7 @@ pub enum MacroCallKind { }, Attr { ast_id: AstId, - attr_args: Arc, + attr_args: Option>, /// Syntactical index of the invoking `#[attribute]`. /// /// Outer attributes are counted first, then inner attributes. This does not support @@ -699,7 +700,7 @@ impl ExpansionInfo { let (macro_arg, _) = db.macro_arg(macro_file.macro_call_id).value.unwrap_or_else(|| { ( Arc::new(tt::Subtree { - delimiter: tt::Delimiter::UNSPECIFIED, + delimiter: tt::Delimiter::DUMMY_INVISIBLE, token_trees: Vec::new(), }), SyntaxFixupUndoInfo::NONE, diff --git a/crates/hir-expand/src/proc_macro.rs b/crates/hir-expand/src/proc_macro.rs index 04b5b7b0b6a3..ccae4c288e65 100644 --- a/crates/hir-expand/src/proc_macro.rs +++ b/crates/hir-expand/src/proc_macro.rs @@ -38,9 +38,10 @@ impl ProcMacroExpander { mixed_site: SpanData, ) -> ExpandResult { match self.proc_macro_id { - ProcMacroId(DUMMY_ID) => { - ExpandResult::new(tt::Subtree::empty(), ExpandError::UnresolvedProcMacro(def_crate)) - } + ProcMacroId(DUMMY_ID) => ExpandResult::new( + tt::Subtree::empty(tt::DelimSpan { open: call_site, close: call_site }), + ExpandError::UnresolvedProcMacro(def_crate), + ), ProcMacroId(id) => { let proc_macros = db.proc_macros(); let proc_macros = match proc_macros.get(&def_crate) { @@ -48,7 +49,7 @@ impl ProcMacroExpander { Some(Err(_)) | None => { never!("Non-dummy expander even though there are no proc macros"); return ExpandResult::new( - tt::Subtree::empty(), + tt::Subtree::empty(tt::DelimSpan { open: call_site, close: call_site }), ExpandError::other("Internal error"), ); } @@ -62,7 +63,7 @@ impl ProcMacroExpander { id ); return ExpandResult::new( - tt::Subtree::empty(), + tt::Subtree::empty(tt::DelimSpan { open: call_site, close: call_site }), ExpandError::other("Internal error"), ); } @@ -82,9 +83,10 @@ impl ProcMacroExpander { ExpandResult { value: tt.clone(), err: Some(ExpandError::other(text)) } } ProcMacroExpansionError::System(text) - | ProcMacroExpansionError::Panic(text) => { - ExpandResult::new(tt::Subtree::empty(), ExpandError::other(text)) - } + | ProcMacroExpansionError::Panic(text) => ExpandResult::new( + tt::Subtree::empty(tt::DelimSpan { open: call_site, close: call_site }), + ExpandError::other(text), + ), }, } } diff --git a/crates/hir-expand/src/quote.rs b/crates/hir-expand/src/quote.rs index 44f20cbd92d2..069bcc3bd8e2 100644 --- a/crates/hir-expand/src/quote.rs +++ b/crates/hir-expand/src/quote.rs @@ -1,5 +1,7 @@ //! A simplified version of quote-crate like quasi quote macro +use base_db::span::SpanData; + // A helper macro quote macro // FIXME: // 1. Not all puncts are handled @@ -8,109 +10,109 @@ #[doc(hidden)] #[macro_export] macro_rules! __quote { - () => { + ($span:ident) => { Vec::::new() }; - ( @SUBTREE $delim:ident $($tt:tt)* ) => { + ( @SUBTREE($span:ident) $delim:ident $($tt:tt)* ) => { { - let children = $crate::__quote!($($tt)*); + let children = $crate::__quote!($span $($tt)*); crate::tt::Subtree { delimiter: crate::tt::Delimiter { kind: crate::tt::DelimiterKind::$delim, - open: ::DUMMY, - close: ::DUMMY, + open: $span, + close: $span, }, token_trees: $crate::quote::IntoTt::to_tokens(children), } } }; - ( @PUNCT $first:literal ) => { + ( @PUNCT($span:ident) $first:literal ) => { { vec![ crate::tt::Leaf::Punct(crate::tt::Punct { char: $first, spacing: crate::tt::Spacing::Alone, - span: ::DUMMY, + span: $span, }).into() ] } }; - ( @PUNCT $first:literal, $sec:literal ) => { + ( @PUNCT($span:ident) $first:literal, $sec:literal ) => { { vec![ crate::tt::Leaf::Punct(crate::tt::Punct { char: $first, spacing: crate::tt::Spacing::Joint, - span: ::DUMMY, + span: $span, }).into(), crate::tt::Leaf::Punct(crate::tt::Punct { char: $sec, spacing: crate::tt::Spacing::Alone, - span: ::DUMMY, + span: $span, }).into() ] } }; // hash variable - ( # $first:ident $($tail:tt)* ) => { + ($span:ident # $first:ident $($tail:tt)* ) => { { - let token = $crate::quote::ToTokenTree::to_token($first); + let token = $crate::quote::ToTokenTree::to_token($first, $span); let mut tokens = vec![token.into()]; - let mut tail_tokens = $crate::quote::IntoTt::to_tokens($crate::__quote!($($tail)*)); + let mut tail_tokens = $crate::quote::IntoTt::to_tokens($crate::__quote!($span $($tail)*)); tokens.append(&mut tail_tokens); tokens } }; - ( ## $first:ident $($tail:tt)* ) => { + ($span:ident ## $first:ident $($tail:tt)* ) => { { - let mut tokens = $first.into_iter().map($crate::quote::ToTokenTree::to_token).collect::>(); - let mut tail_tokens = $crate::quote::IntoTt::to_tokens($crate::__quote!($($tail)*)); + let mut tokens = $first.into_iter().map(|it| $crate::quote::ToTokenTree::to_token(it, $span)).collect::>(); + let mut tail_tokens = $crate::quote::IntoTt::to_tokens($crate::__quote!($span $($tail)*)); tokens.append(&mut tail_tokens); tokens } }; // Brace - ( { $($tt:tt)* } ) => { $crate::__quote!(@SUBTREE Brace $($tt)*) }; + ($span:ident { $($tt:tt)* } ) => { $crate::__quote!(@SUBTREE($span) Brace $($tt)*) }; // Bracket - ( [ $($tt:tt)* ] ) => { $crate::__quote!(@SUBTREE Bracket $($tt)*) }; + ($span:ident [ $($tt:tt)* ] ) => { $crate::__quote!(@SUBTREE($span) Bracket $($tt)*) }; // Parenthesis - ( ( $($tt:tt)* ) ) => { $crate::__quote!(@SUBTREE Parenthesis $($tt)*) }; + ($span:ident ( $($tt:tt)* ) ) => { $crate::__quote!(@SUBTREE($span) Parenthesis $($tt)*) }; // Literal - ( $tt:literal ) => { vec![$crate::quote::ToTokenTree::to_token($tt).into()] }; + ($span:ident $tt:literal ) => { vec![$crate::quote::ToTokenTree::to_token($tt, $span).into()] }; // Ident - ( $tt:ident ) => { + ($span:ident $tt:ident ) => { vec![ { crate::tt::Leaf::Ident(crate::tt::Ident { text: stringify!($tt).into(), - span: ::DUMMY, + span: $span, }).into() }] }; // Puncts // FIXME: Not all puncts are handled - ( -> ) => {$crate::__quote!(@PUNCT '-', '>')}; - ( & ) => {$crate::__quote!(@PUNCT '&')}; - ( , ) => {$crate::__quote!(@PUNCT ',')}; - ( : ) => {$crate::__quote!(@PUNCT ':')}; - ( ; ) => {$crate::__quote!(@PUNCT ';')}; - ( :: ) => {$crate::__quote!(@PUNCT ':', ':')}; - ( . ) => {$crate::__quote!(@PUNCT '.')}; - ( < ) => {$crate::__quote!(@PUNCT '<')}; - ( > ) => {$crate::__quote!(@PUNCT '>')}; - ( ! ) => {$crate::__quote!(@PUNCT '!')}; - - ( $first:tt $($tail:tt)+ ) => { + ($span:ident -> ) => {$crate::__quote!(@PUNCT($span) '-', '>')}; + ($span:ident & ) => {$crate::__quote!(@PUNCT($span) '&')}; + ($span:ident , ) => {$crate::__quote!(@PUNCT($span) ',')}; + ($span:ident : ) => {$crate::__quote!(@PUNCT($span) ':')}; + ($span:ident ; ) => {$crate::__quote!(@PUNCT($span) ';')}; + ($span:ident :: ) => {$crate::__quote!(@PUNCT($span) ':', ':')}; + ($span:ident . ) => {$crate::__quote!(@PUNCT($span) '.')}; + ($span:ident < ) => {$crate::__quote!(@PUNCT($span) '<')}; + ($span:ident > ) => {$crate::__quote!(@PUNCT($span) '>')}; + ($span:ident ! ) => {$crate::__quote!(@PUNCT($span) '!')}; + + ($span:ident $first:tt $($tail:tt)+ ) => { { - let mut tokens = $crate::quote::IntoTt::to_tokens($crate::__quote!($first)); - let mut tail_tokens = $crate::quote::IntoTt::to_tokens($crate::__quote!($($tail)*)); + let mut tokens = $crate::quote::IntoTt::to_tokens($crate::__quote!($span $first )); + let mut tail_tokens = $crate::quote::IntoTt::to_tokens($crate::__quote!($span $($tail)*)); tokens.append(&mut tail_tokens); tokens @@ -122,19 +124,22 @@ macro_rules! __quote { /// It probably should implement in proc-macro #[macro_export] macro_rules! quote { - ( $($tt:tt)* ) => { - $crate::quote::IntoTt::to_subtree($crate::__quote!($($tt)*)) + ($span:ident=> $($tt:tt)* ) => { + $crate::quote::IntoTt::to_subtree($crate::__quote!($span $($tt)*), $span) } } pub(crate) trait IntoTt { - fn to_subtree(self) -> crate::tt::Subtree; + fn to_subtree(self, span: SpanData) -> crate::tt::Subtree; fn to_tokens(self) -> Vec; } impl IntoTt for Vec { - fn to_subtree(self) -> crate::tt::Subtree { - crate::tt::Subtree { delimiter: crate::tt::Delimiter::unspecified(), token_trees: self } + fn to_subtree(self, span: SpanData) -> crate::tt::Subtree { + crate::tt::Subtree { + delimiter: crate::tt::Delimiter::invisible_spanned(span), + token_trees: self, + } } fn to_tokens(self) -> Vec { @@ -143,7 +148,7 @@ impl IntoTt for Vec { } impl IntoTt for crate::tt::Subtree { - fn to_subtree(self) -> crate::tt::Subtree { + fn to_subtree(self, _: SpanData) -> crate::tt::Subtree { self } @@ -153,39 +158,39 @@ impl IntoTt for crate::tt::Subtree { } pub(crate) trait ToTokenTree { - fn to_token(self) -> crate::tt::TokenTree; + fn to_token(self, span: SpanData) -> crate::tt::TokenTree; } impl ToTokenTree for crate::tt::TokenTree { - fn to_token(self) -> crate::tt::TokenTree { + fn to_token(self, _: SpanData) -> crate::tt::TokenTree { self } } impl ToTokenTree for &crate::tt::TokenTree { - fn to_token(self) -> crate::tt::TokenTree { + fn to_token(self, _: SpanData) -> crate::tt::TokenTree { self.clone() } } impl ToTokenTree for crate::tt::Subtree { - fn to_token(self) -> crate::tt::TokenTree { + fn to_token(self, _: SpanData) -> crate::tt::TokenTree { self.into() } } macro_rules! impl_to_to_tokentrees { - ($($ty:ty => $this:ident $im:block);*) => { + ($($span:ident: $ty:ty => $this:ident $im:block);*) => { $( impl ToTokenTree for $ty { - fn to_token($this) -> crate::tt::TokenTree { + fn to_token($this, $span: SpanData) -> crate::tt::TokenTree { let leaf: crate::tt::Leaf = $im.into(); leaf.into() } } impl ToTokenTree for &$ty { - fn to_token($this) -> crate::tt::TokenTree { + fn to_token($this, $span: SpanData) -> crate::tt::TokenTree { let leaf: crate::tt::Leaf = $im.clone().into(); leaf.into() } @@ -195,41 +200,45 @@ macro_rules! impl_to_to_tokentrees { } impl_to_to_tokentrees! { - u32 => self { crate::tt::Literal{text: self.to_string().into(), span: ::DUMMY} }; - usize => self { crate::tt::Literal{text: self.to_string().into(), span: ::DUMMY} }; - i32 => self { crate::tt::Literal{text: self.to_string().into(), span: ::DUMMY} }; - bool => self { crate::tt::Ident{text: self.to_string().into(), span: ::DUMMY} }; - crate::tt::Leaf => self { self }; - crate::tt::Literal => self { self }; - crate::tt::Ident => self { self }; - crate::tt::Punct => self { self }; - &str => self { crate::tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), span: ::DUMMY}}; - String => self { crate::tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), span: ::DUMMY}} + span: u32 => self { crate::tt::Literal{text: self.to_string().into(), span} }; + span: usize => self { crate::tt::Literal{text: self.to_string().into(), span} }; + span: i32 => self { crate::tt::Literal{text: self.to_string().into(), span} }; + span: bool => self { crate::tt::Ident{text: self.to_string().into(), span} }; + _span: crate::tt::Leaf => self { self }; + _span: crate::tt::Literal => self { self }; + _span: crate::tt::Ident => self { self }; + _span: crate::tt::Punct => self { self }; + span: &str => self { crate::tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), span}}; + span: String => self { crate::tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), span}} } #[cfg(test)] mod tests { + use crate::tt; + use ::tt::Span; use expect_test::expect; + const DUMMY: tt::SpanData = tt::SpanData::DUMMY; + #[test] fn test_quote_delimiters() { - assert_eq!(quote!({}).to_string(), "{}"); - assert_eq!(quote!(()).to_string(), "()"); - assert_eq!(quote!([]).to_string(), "[]"); + assert_eq!(quote!(DUMMY =>{}).to_string(), "{}"); + assert_eq!(quote!(DUMMY =>()).to_string(), "()"); + assert_eq!(quote!(DUMMY =>[]).to_string(), "[]"); } #[test] fn test_quote_idents() { - assert_eq!(quote!(32).to_string(), "32"); - assert_eq!(quote!(struct).to_string(), "struct"); + assert_eq!(quote!(DUMMY =>32).to_string(), "32"); + assert_eq!(quote!(DUMMY =>struct).to_string(), "struct"); } #[test] fn test_quote_hash_simple_literal() { let a = 20; - assert_eq!(quote!(#a).to_string(), "20"); + assert_eq!(quote!(DUMMY =>#a).to_string(), "20"); let s: String = "hello".into(); - assert_eq!(quote!(#s).to_string(), "\"hello\""); + assert_eq!(quote!(DUMMY =>#s).to_string(), "\"hello\""); } fn mk_ident(name: &str) -> crate::tt::Ident { @@ -243,7 +252,7 @@ mod tests { fn test_quote_hash_token_tree() { let a = mk_ident("hello"); - let quoted = quote!(#a); + let quoted = quote!(DUMMY =>#a); assert_eq!(quoted.to_string(), "hello"); let t = format!("{quoted:?}"); expect![[r#" @@ -255,7 +264,7 @@ mod tests { fn test_quote_simple_derive_copy() { let name = mk_ident("Foo"); - let quoted = quote! { + let quoted = quote! {DUMMY => impl Clone for #name { fn clone(&self) -> Self { Self {} @@ -275,7 +284,8 @@ mod tests { // } let struct_name = mk_ident("Foo"); let fields = [mk_ident("name"), mk_ident("id")]; - let fields = fields.iter().flat_map(|it| quote!(#it: self.#it.clone(), ).token_trees); + let fields = + fields.iter().flat_map(|it| quote!(DUMMY =>#it: self.#it.clone(), ).token_trees); let list = crate::tt::Subtree { delimiter: crate::tt::Delimiter { @@ -286,7 +296,7 @@ mod tests { token_trees: fields.collect(), }; - let quoted = quote! { + let quoted = quote! {DUMMY => impl Clone for #struct_name { fn clone(&self) -> Self { Self #list diff --git a/crates/load-cargo/src/lib.rs b/crates/load-cargo/src/lib.rs index ed4175c45830..db9654220dd7 100644 --- a/crates/load-cargo/src/lib.rs +++ b/crates/load-cargo/src/lib.rs @@ -16,6 +16,7 @@ use ide_db::{ use itertools::Itertools; use proc_macro_api::{MacroDylib, ProcMacroServer}; use project_model::{CargoConfig, PackageRoot, ProjectManifest, ProjectWorkspace}; +use tt::DelimSpan; use vfs::{file_set::FileSetConfig, loader::Handle, AbsPath, AbsPathBuf, VfsPath}; pub struct LoadCargoConfig { @@ -417,11 +418,11 @@ impl ProcMacroExpander for EmptyExpander { _: &tt::Subtree, _: Option<&tt::Subtree>, _: &Env, - _: SpanData, + call_site: SpanData, _: SpanData, _: SpanData, ) -> Result, ProcMacroExpansionError> { - Ok(tt::Subtree::empty()) + Ok(tt::Subtree::empty(DelimSpan { open: call_site, close: call_site })) } } diff --git a/crates/mbe/src/expander.rs b/crates/mbe/src/expander.rs index 487e8b35980d..0e755f69bf7d 100644 --- a/crates/mbe/src/expander.rs +++ b/crates/mbe/src/expander.rs @@ -49,7 +49,7 @@ pub(crate) fn expand_rules( ExpandResult { value, err: match_.err.or(transcribe_err) } } else { ExpandResult::new( - tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: vec![] }, + tt::Subtree { delimiter: tt::Delimiter::DUMMY_INVISIBLE, token_trees: vec![] }, ExpandError::NoMatchingRule, ) } diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs index 14b32599091e..5e1ceacf12c3 100644 --- a/crates/mbe/src/expander/matcher.rs +++ b/crates/mbe/src/expander/matcher.rs @@ -76,7 +76,8 @@ impl Bindings { fn push_optional(&mut self, name: &SmolStr) { // FIXME: Do we have a better way to represent an empty token ? // Insert an empty subtree for empty token - let tt = tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: vec![] }.into(); + let tt = + tt::Subtree { delimiter: tt::Delimiter::DUMMY_INVISIBLE, token_trees: vec![] }.into(); self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt))); } @@ -816,7 +817,7 @@ fn match_meta_var( match neg { None => lit.into(), Some(neg) => tt::TokenTree::Subtree(tt::Subtree { - delimiter: tt::Delimiter::unspecified(), + delimiter: tt::Delimiter::dummy_invisible(), token_trees: vec![neg, lit.into()], }), } @@ -849,7 +850,7 @@ impl MetaTemplate { OpDelimitedIter { inner: &self.0, idx: 0, - delimited: delimited.unwrap_or(tt::Delimiter::UNSPECIFIED), + delimited: delimited.unwrap_or(tt::Delimiter::DUMMY_INVISIBLE), } } } @@ -947,7 +948,7 @@ impl TtIter<'_, S> { let puncts = self.expect_glued_punct()?; let token_trees = puncts.into_iter().map(|p| tt::Leaf::Punct(p).into()).collect(); Ok(tt::TokenTree::Subtree(tt::Subtree { - delimiter: tt::Delimiter::unspecified(), + delimiter: tt::Delimiter::dummy_invisible(), token_trees, })) } @@ -964,7 +965,7 @@ impl TtIter<'_, S> { let ident = self.expect_ident_or_underscore()?; Ok(tt::Subtree { - delimiter: tt::Delimiter::unspecified(), + delimiter: tt::Delimiter::dummy_invisible(), token_trees: vec![ tt::Leaf::Punct(*punct).into(), tt::Leaf::Ident(ident.clone()).into(), diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs index 8aedd7314063..40f683d84626 100644 --- a/crates/mbe/src/expander/transcriber.rs +++ b/crates/mbe/src/expander/transcriber.rs @@ -88,7 +88,7 @@ impl Bindings { // FIXME: Meta and Item should get proper defaults MetaVarKind::Meta | MetaVarKind::Item | MetaVarKind::Tt | MetaVarKind::Vis => { Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { - delimiter: tt::Delimiter::UNSPECIFIED, + delimiter: tt::Delimiter::DUMMY_INVISIBLE, token_trees: vec![], })) } @@ -292,7 +292,7 @@ fn expand_subtree( let tts = arena.drain(start_elements..).collect(); ExpandResult { value: tt::Subtree { - delimiter: delimiter.unwrap_or_else(tt::Delimiter::unspecified), + delimiter: delimiter.unwrap_or_else(tt::Delimiter::dummy_invisible), token_trees: tts, }, err, @@ -325,7 +325,7 @@ fn expand_var( // ``` // We just treat it a normal tokens let tt = tt::Subtree { - delimiter: tt::Delimiter::UNSPECIFIED, + delimiter: tt::Delimiter::DUMMY_INVISIBLE, token_trees: vec![ tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, span: id }) .into(), @@ -336,7 +336,10 @@ fn expand_var( ExpandResult::ok(Fragment::Tokens(tt)) } Err(e) => ExpandResult { - value: Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree::empty())), + value: Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree::empty(tt::DelimSpan { + open: S::DUMMY, + close: S::DUMMY, + }))), err: Some(e), }, } @@ -378,8 +381,11 @@ fn expand_repeat( ); return ExpandResult { value: Fragment::Tokens( - tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] } - .into(), + tt::Subtree { + delimiter: tt::Delimiter::dummy_invisible(), + token_trees: vec![], + } + .into(), ), err: Some(ExpandError::LimitExceeded), }; @@ -390,7 +396,7 @@ fn expand_repeat( continue; } - t.delimiter = tt::Delimiter::UNSPECIFIED; + t.delimiter = tt::Delimiter::DUMMY_INVISIBLE; push_subtree(&mut buf, t); if let Some(sep) = separator { @@ -424,7 +430,7 @@ fn expand_repeat( // Check if it is a single token subtree without any delimiter // e.g {Delimiter:None> ['>'] /Delimiter:None>} - let tt = tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: buf }.into(); + let tt = tt::Subtree { delimiter: tt::Delimiter::DUMMY_INVISIBLE, token_trees: buf }.into(); if RepeatKind::OneOrMore == kind && counter == 0 { return ExpandResult { diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index c19112b4c540..10eb59bb8352 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -34,9 +34,9 @@ pub use tt::{Delimiter, DelimiterKind, Punct, SyntaxContext}; pub use crate::{ syntax_bridge::{ - map_from_syntax_node, parse_exprs_with_sep, parse_to_token_tree, - parse_to_token_tree_static_span, syntax_node_to_token_tree, - syntax_node_to_token_tree_modified, token_tree_to_syntax_node, SpanMapper, + parse_exprs_with_sep, parse_to_token_tree, parse_to_token_tree_static_span, + syntax_node_to_token_tree, syntax_node_to_token_tree_modified, token_tree_to_syntax_node, + SpanMapper, }, token_map::TokenMap, }; diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index be5eafd014af..5722a5bd8eb0 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -4,7 +4,7 @@ use rustc_hash::{FxHashMap, FxHashSet}; use stdx::{never, non_empty_vec::NonEmptyVec}; use syntax::{ ast::{self, make::tokens::doc_comment}, - AstToken, NodeOrToken, Parse, PreorderWithTokens, SmolStr, SyntaxElement, SyntaxKind, + AstToken, Parse, PreorderWithTokens, SmolStr, SyntaxElement, SyntaxKind, SyntaxKind::*, SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, WalkEvent, T, }; @@ -142,30 +142,10 @@ where tree_sink.finish() } -pub fn map_from_syntax_node( - node: &SyntaxNode, - anchor: Anchor, - anchor_offset: TextSize, -) -> TokenMap> -where - Anchor: Copy, - SpanData: Span, - Ctx: SyntaxContext, -{ - let mut map = TokenMap::empty(); - node.descendants_with_tokens().filter_map(NodeOrToken::into_token).for_each(|t| { - map.push( - t.text_range().start(), - SpanData { range: t.text_range() - anchor_offset, anchor, ctx: Ctx::DUMMY }, - ); - }); - map.finish(); - map -} - /// Convert a string to a `TokenTree` pub fn parse_to_token_tree( anchor: Anchor, + ctx: Ctx, text: &str, ) -> Option>> where @@ -177,7 +157,7 @@ where if lexed.errors().next().is_some() { return None; } - let mut conv = RawConverter { lexed, pos: 0, anchor }; + let mut conv = RawConverter { lexed, pos: 0, anchor, ctx }; Some(convert_tokens(&mut conv)) } @@ -220,7 +200,7 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec, S: Span, { - let entry = tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: vec![] }; + let entry = tt::Subtree { delimiter: tt::Delimiter::DUMMY_INVISIBLE, token_trees: vec![] }; let mut stack = NonEmptyVec::new(entry); while let Some((token, abs_range)) = conv.bump() { @@ -463,10 +443,11 @@ fn convert_doc_comment( } /// A raw token (straight from lexer) converter -struct RawConverter<'a, Anchor> { +struct RawConverter<'a, Anchor, Ctx> { lexed: parser::LexedStr<'a>, pos: usize, anchor: Anchor, + ctx: Ctx, } /// A raw token (straight from lexer) converter that gives every token the same span. struct StaticRawConverter<'a, S> { @@ -499,16 +480,16 @@ trait TokenConverter: Sized { fn span_for(&self, range: TextRange) -> S; } -impl SrcToken, S> for usize { - fn kind(&self, ctx: &RawConverter<'_, Anchor>) -> SyntaxKind { +impl SrcToken, S> for usize { + fn kind(&self, ctx: &RawConverter<'_, Anchor, Ctx>) -> SyntaxKind { ctx.lexed.kind(*self) } - fn to_char(&self, ctx: &RawConverter<'_, Anchor>) -> Option { + fn to_char(&self, ctx: &RawConverter<'_, Anchor, Ctx>) -> Option { ctx.lexed.text(*self).chars().next() } - fn to_text(&self, ctx: &RawConverter<'_, Anchor>) -> SmolStr { + fn to_text(&self, ctx: &RawConverter<'_, Anchor, Ctx>) -> SmolStr { ctx.lexed.text(*self).into() } } @@ -528,7 +509,7 @@ impl SrcToken, S> for usize { } impl TokenConverter> - for RawConverter<'_, Anchor> + for RawConverter<'_, Anchor, Ctx> where SpanData: Span, { @@ -563,7 +544,7 @@ where } fn span_for(&self, range: TextRange) -> SpanData { - SpanData { range, anchor: self.anchor, ctx: Ctx::DUMMY } + SpanData { range, anchor: self.anchor, ctx: self.ctx } } } diff --git a/crates/mbe/src/tt_iter.rs b/crates/mbe/src/tt_iter.rs index 44fbbcfc2068..595691b17736 100644 --- a/crates/mbe/src/tt_iter.rs +++ b/crates/mbe/src/tt_iter.rs @@ -175,7 +175,7 @@ impl<'a, S: Span> TtIter<'a, S> { let res = match res.len() { 0 | 1 => res.pop(), _ => Some(tt::TokenTree::Subtree(tt::Subtree { - delimiter: tt::Delimiter::unspecified(), + delimiter: tt::Delimiter::DUMMY_INVISIBLE, token_trees: res, })), }; diff --git a/crates/proc-macro-api/src/msg.rs b/crates/proc-macro-api/src/msg.rs index ddac514ff709..dd882d82fb64 100644 --- a/crates/proc-macro-api/src/msg.rs +++ b/crates/proc-macro-api/src/msg.rs @@ -62,12 +62,14 @@ pub struct ExpandMacro { pub current_dir: Option, /// marker for serde skip stuff #[serde(skip_serializing_if = "ExpnGlobals::skip_serializing_if")] + #[serde(default)] pub has_global_spans: ExpnGlobals, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Default, Debug, Serialize, Deserialize)] pub struct ExpnGlobals { #[serde(skip_serializing)] + #[serde(default)] pub serialize: bool, pub def_site: usize, pub call_site: usize, diff --git a/crates/proc-macro-api/src/msg/flat.rs b/crates/proc-macro-api/src/msg/flat.rs index 43840fa333d7..5835718628e5 100644 --- a/crates/proc-macro-api/src/msg/flat.rs +++ b/crates/proc-macro-api/src/msg/flat.rs @@ -56,16 +56,6 @@ impl std::fmt::Debug for TokenId { impl tt::Span for TokenId { const DUMMY: Self = TokenId(!0); - - type Anchor = (); - - fn anchor(self) -> Self::Anchor { - () - } - - fn mk(_: Self::Anchor, _: text_size::TextRange) -> Self { - Self::DUMMY - } } #[derive(Serialize, Deserialize, Debug)] diff --git a/crates/proc-macro-srv/src/server.rs b/crates/proc-macro-srv/src/server.rs index fc080eccc0f4..54430e0d1905 100644 --- a/crates/proc-macro-srv/src/server.rs +++ b/crates/proc-macro-srv/src/server.rs @@ -426,8 +426,6 @@ impl LiteralFormatter { #[cfg(test)] mod tests { - use ::tt::Span; - use super::*; #[test] @@ -436,16 +434,16 @@ mod tests { token_trees: vec![ tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: "struct".into(), - span: tt::TokenId::DUMMY, + span: tt::TokenId(0), })), tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: "T".into(), - span: tt::TokenId::DUMMY, + span: tt::TokenId(0), })), tt::TokenTree::Subtree(tt::Subtree { delimiter: tt::Delimiter { - open: tt::TokenId::DUMMY, - close: tt::TokenId::DUMMY, + open: tt::TokenId(0), + close: tt::TokenId(0), kind: tt::DelimiterKind::Brace, }, token_trees: vec![], @@ -460,30 +458,30 @@ mod tests { fn test_ra_server_from_str() { let subtree_paren_a = tt::TokenTree::Subtree(tt::Subtree { delimiter: tt::Delimiter { - open: tt::TokenId::DUMMY, - close: tt::TokenId::DUMMY, + open: tt::TokenId(0), + close: tt::TokenId(0), kind: tt::DelimiterKind::Parenthesis, }, token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: "a".into(), - span: tt::TokenId::DUMMY, + span: tt::TokenId(0), }))], }); - let t1 = TokenStream::from_str("(a)", tt::TokenId::DUMMY).unwrap(); + let t1 = TokenStream::from_str("(a)", tt::TokenId(0)).unwrap(); assert_eq!(t1.token_trees.len(), 1); assert_eq!(t1.token_trees[0], subtree_paren_a); - let t2 = TokenStream::from_str("(a);", tt::TokenId::DUMMY).unwrap(); + let t2 = TokenStream::from_str("(a);", tt::TokenId(0)).unwrap(); assert_eq!(t2.token_trees.len(), 2); assert_eq!(t2.token_trees[0], subtree_paren_a); - let underscore = TokenStream::from_str("_", tt::TokenId::DUMMY).unwrap(); + let underscore = TokenStream::from_str("_", tt::TokenId(0)).unwrap(); assert_eq!( underscore.token_trees[0], tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: "_".into(), - span: tt::TokenId::DUMMY, + span: tt::TokenId(0), })) ); } diff --git a/crates/proc-macro-srv/src/tests/mod.rs b/crates/proc-macro-srv/src/tests/mod.rs index 04a0ae7bc720..b04e3ca19ac1 100644 --- a/crates/proc-macro-srv/src/tests/mod.rs +++ b/crates/proc-macro-srv/src/tests/mod.rs @@ -8,7 +8,7 @@ use expect_test::expect; #[test] fn test_derive_empty() { - assert_expand("DeriveEmpty", r#"struct S;"#, expect!["SUBTREE $$ 4294967295 4294967295"]); + assert_expand("DeriveEmpty", r#"struct S;"#, expect!["SUBTREE $$ 1 1"]); } #[test] @@ -17,12 +17,12 @@ fn test_derive_error() { "DeriveError", r#"struct S;"#, expect![[r##" - SUBTREE $$ 4294967295 4294967295 - IDENT compile_error 4294967295 - PUNCH ! [alone] 4294967295 - SUBTREE () 4294967295 4294967295 - LITERAL "#[derive(DeriveError)] struct S ;" 4294967295 - PUNCH ; [alone] 4294967295"##]], + SUBTREE $$ 1 1 + IDENT compile_error 1 + PUNCH ! [alone] 1 + SUBTREE () 1 1 + LITERAL "#[derive(DeriveError)] struct S ;" 1 + PUNCH ; [alone] 1"##]], ); } @@ -32,14 +32,14 @@ fn test_fn_like_macro_noop() { "fn_like_noop", r#"ident, 0, 1, []"#, expect![[r#" - SUBTREE $$ 4294967295 4294967295 - IDENT ident 4294967295 - PUNCH , [alone] 4294967295 - LITERAL 0 4294967295 - PUNCH , [alone] 4294967295 - LITERAL 1 4294967295 - PUNCH , [alone] 4294967295 - SUBTREE [] 4294967295 4294967295"#]], + SUBTREE $$ 1 1 + IDENT ident 1 + PUNCH , [alone] 1 + LITERAL 0 1 + PUNCH , [alone] 1 + LITERAL 1 1 + PUNCH , [alone] 1 + SUBTREE [] 1 1"#]], ); } @@ -49,10 +49,10 @@ fn test_fn_like_macro_clone_ident_subtree() { "fn_like_clone_tokens", r#"ident, []"#, expect![[r#" - SUBTREE $$ 4294967295 4294967295 - IDENT ident 4294967295 - PUNCH , [alone] 4294967295 - SUBTREE [] 4294967295 4294967295"#]], + SUBTREE $$ 1 1 + IDENT ident 1 + PUNCH , [alone] 1 + SUBTREE [] 1 1"#]], ); } @@ -62,8 +62,8 @@ fn test_fn_like_macro_clone_raw_ident() { "fn_like_clone_tokens", "r#async", expect![[r#" - SUBTREE $$ 4294967295 4294967295 - IDENT r#async 4294967295"#]], + SUBTREE $$ 1 1 + IDENT r#async 1"#]], ); } @@ -73,14 +73,14 @@ fn test_fn_like_mk_literals() { "fn_like_mk_literals", r#""#, expect![[r#" - SUBTREE $$ 4294967295 4294967295 - LITERAL b"byte_string" 4294967295 - LITERAL 'c' 4294967295 - LITERAL "string" 4294967295 - LITERAL 3.14f64 4294967295 - LITERAL 3.14 4294967295 - LITERAL 123i64 4294967295 - LITERAL 123 4294967295"#]], + SUBTREE $$ 1 1 + LITERAL b"byte_string" 1 + LITERAL 'c' 1 + LITERAL "string" 1 + LITERAL 3.14f64 1 + LITERAL 3.14 1 + LITERAL 123i64 1 + LITERAL 123 1"#]], ); } @@ -90,9 +90,9 @@ fn test_fn_like_mk_idents() { "fn_like_mk_idents", r#""#, expect![[r#" - SUBTREE $$ 4294967295 4294967295 - IDENT standard 4294967295 - IDENT r#raw 4294967295"#]], + SUBTREE $$ 1 1 + IDENT standard 1 + IDENT r#raw 1"#]], ); } @@ -102,17 +102,17 @@ fn test_fn_like_macro_clone_literals() { "fn_like_clone_tokens", r#"1u16, 2_u32, -4i64, 3.14f32, "hello bridge""#, expect![[r#" - SUBTREE $$ 4294967295 4294967295 - LITERAL 1u16 4294967295 - PUNCH , [alone] 4294967295 - LITERAL 2_u32 4294967295 - PUNCH , [alone] 4294967295 - PUNCH - [alone] 4294967295 - LITERAL 4i64 4294967295 - PUNCH , [alone] 4294967295 - LITERAL 3.14f32 4294967295 - PUNCH , [alone] 4294967295 - LITERAL "hello bridge" 4294967295"#]], + SUBTREE $$ 1 1 + LITERAL 1u16 1 + PUNCH , [alone] 1 + LITERAL 2_u32 1 + PUNCH , [alone] 1 + PUNCH - [alone] 1 + LITERAL 4i64 1 + PUNCH , [alone] 1 + LITERAL 3.14f32 1 + PUNCH , [alone] 1 + LITERAL "hello bridge" 1"#]], ); } @@ -126,12 +126,12 @@ fn test_attr_macro() { r#"mod m {}"#, r#"some arguments"#, expect![[r##" - SUBTREE $$ 4294967295 4294967295 - IDENT compile_error 4294967295 - PUNCH ! [alone] 4294967295 - SUBTREE () 4294967295 4294967295 - LITERAL "#[attr_error(some arguments)] mod m {}" 4294967295 - PUNCH ; [alone] 4294967295"##]], + SUBTREE $$ 1 1 + IDENT compile_error 1 + PUNCH ! [alone] 1 + SUBTREE () 1 1 + LITERAL "#[attr_error(some arguments)] mod m {}" 1 + PUNCH ; [alone] 1"##]], ); } diff --git a/crates/proc-macro-srv/src/tests/utils.rs b/crates/proc-macro-srv/src/tests/utils.rs index ccfefafb2cfc..c12096d140c9 100644 --- a/crates/proc-macro-srv/src/tests/utils.rs +++ b/crates/proc-macro-srv/src/tests/utils.rs @@ -2,7 +2,6 @@ use expect_test::Expect; use proc_macro_api::msg::TokenId; -use tt::Span; use crate::{dylib, proc_macro_test_dylib_path, ProcMacroSrv}; @@ -25,7 +24,9 @@ pub fn assert_expand_attr(macro_name: &str, ra_fixture: &str, attr_args: &str, e } fn assert_expand_impl(macro_name: &str, input: &str, attr: Option<&str>, expect: Expect) { - let call_site = TokenId::DUMMY; + let def_site = TokenId(0); + let call_site = TokenId(1); + let mixed_site = TokenId(2); let path = proc_macro_test_dylib_path(); let expander = dylib::Expander::new(&path).unwrap(); let fixture = parse_string(input, call_site).unwrap(); @@ -36,9 +37,9 @@ fn assert_expand_impl(macro_name: &str, input: &str, attr: Option<&str>, expect: macro_name, &fixture.into_subtree(call_site), attr.as_ref(), - TokenId::DUMMY, - TokenId::DUMMY, - TokenId::DUMMY, + def_site, + call_site, + mixed_site, ) .unwrap(); expect.assert_eq(&format!("{res:?}")); diff --git a/crates/tt/src/lib.rs b/crates/tt/src/lib.rs index 7977d97797ad..b1f218516210 100644 --- a/crates/tt/src/lib.rs +++ b/crates/tt/src/lib.rs @@ -23,18 +23,11 @@ pub struct SpanData { } impl Span for SpanData { - type Anchor = Anchor; const DUMMY: Self = SpanData { range: TextRange::empty(TextSize::new(0)), anchor: Anchor::DUMMY, ctx: Ctx::DUMMY, }; - fn anchor(self) -> Self::Anchor { - self.anchor - } - fn mk(anchor: Self::Anchor, range: TextRange) -> Self { - SpanData { anchor, range, ctx: Ctx::DUMMY } - } } pub trait SpanAnchor: @@ -46,9 +39,6 @@ pub trait SpanAnchor: // FIXME: Get rid of this trait? pub trait Span: std::fmt::Debug + Copy + Sized + Eq { const DUMMY: Self; - type Anchor: Copy + fmt::Debug + Eq + std::hash::Hash; - fn anchor(self) -> Self::Anchor; - fn mk(anchor: Self::Anchor, range: TextRange) -> Self; } pub trait SyntaxContext: std::fmt::Debug + Copy + Sized + Eq { @@ -62,18 +52,30 @@ pub enum TokenTree { } impl_from!(Leaf, Subtree for TokenTree); impl TokenTree { - pub const fn empty() -> Self { - Self::Subtree(Subtree { delimiter: Delimiter::UNSPECIFIED, token_trees: vec![] }) + pub const fn empty(span: S) -> Self { + Self::Subtree(Subtree { + delimiter: Delimiter::invisible_spanned(span), + token_trees: vec![], + }) } pub fn subtree_or_wrap(self) -> Subtree { match self { TokenTree::Leaf(_) => { - Subtree { delimiter: Delimiter::UNSPECIFIED, token_trees: vec![self] } + Subtree { delimiter: Delimiter::DUMMY_INVISIBLE, token_trees: vec![self] } } TokenTree::Subtree(s) => s, } } + pub fn subtree_or_wrap2(self, span: DelimSpan) -> Subtree { + match self { + TokenTree::Leaf(_) => Subtree { + delimiter: Delimiter::invisible_delim_spanned(span), + token_trees: vec![self], + }, + TokenTree::Subtree(s) => s, + } + } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -101,8 +103,8 @@ pub struct Subtree { } impl Subtree { - pub const fn empty() -> Self { - Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] } + pub const fn empty(span: DelimSpan) -> Self { + Subtree { delimiter: Delimiter::invisible_delim_spanned(span), token_trees: vec![] } } pub fn visit_ids(&mut self, f: &mut impl FnMut(S) -> S) { @@ -119,6 +121,16 @@ impl Subtree { } } +#[derive(Debug, Copy, Clone, PartialEq)] +pub struct DelimSpan { + pub open: S, + pub close: S, +} + +impl DelimSpan { + pub const DUMMY: Self = Self { open: S::DUMMY, close: S::DUMMY }; +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct Delimiter { pub open: S, @@ -127,10 +139,23 @@ pub struct Delimiter { } impl Delimiter { - pub const UNSPECIFIED: Self = + pub const DUMMY_INVISIBLE: Self = Self { open: S::DUMMY, close: S::DUMMY, kind: DelimiterKind::Invisible }; - pub const fn unspecified() -> Self { - Self::UNSPECIFIED + + pub const fn dummy_invisible() -> Self { + Self::DUMMY_INVISIBLE + } + + pub const fn invisible_spanned(span: S) -> Self { + Delimiter { open: span, close: span, kind: DelimiterKind::Invisible } + } + + pub const fn invisible_delim_spanned(span: DelimSpan) -> Self { + Delimiter { open: span.open, close: span.close, kind: DelimiterKind::Invisible } + } + + pub fn delim_span(&self) -> DelimSpan { + DelimSpan { open: self.open, close: self.close } } } diff --git a/crates/vfs/src/lib.rs b/crates/vfs/src/lib.rs index 06004adad34a..c6866bbfb9b4 100644 --- a/crates/vfs/src/lib.rs +++ b/crates/vfs/src/lib.rs @@ -62,6 +62,11 @@ pub use paths::{AbsPath, AbsPathBuf}; #[derive(Copy, Clone, Debug, Ord, PartialOrd, Eq, PartialEq, Hash)] pub struct FileId(pub u32); +impl FileId { + /// Think twice about using this. If this ends up in a wrong place it will cause panics! + pub const BOGUS: FileId = FileId(u32::MAX); +} + /// safe because `FileId` is a newtype of `u32` impl nohash_hasher::IsEnabled for FileId {} From c11737cd63da1893947d0d7d43ab4bfb0fdeb0ad Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Fri, 1 Dec 2023 14:58:57 +0100 Subject: [PATCH 18/24] Simplify include handling --- crates/hir-expand/src/builtin_fn_macro.rs | 55 +++++++++---------- crates/hir-expand/src/db.rs | 6 -- crates/hir-expand/src/lib.rs | 7 +-- crates/hir/src/db.rs | 4 +- crates/ide-db/src/apply_change.rs | 1 - .../src/handlers/unresolved_macro_call.rs | 16 ++++++ 6 files changed, 46 insertions(+), 43 deletions(-) diff --git a/crates/hir-expand/src/builtin_fn_macro.rs b/crates/hir-expand/src/builtin_fn_macro.rs index 726b9835369f..74ca2f7ec44e 100644 --- a/crates/hir-expand/src/builtin_fn_macro.rs +++ b/crates/hir-expand/src/builtin_fn_macro.rs @@ -17,7 +17,7 @@ use crate::{ hygiene::span_with_def_site_ctxt, name, quote, tt::{self, DelimSpan}, - EagerCallInfo, ExpandError, ExpandResult, HirFileIdExt, MacroCallId, MacroCallLoc, + ExpandError, ExpandResult, HirFileIdExt, MacroCallId, MacroCallLoc, }; macro_rules! register_builtin { @@ -575,36 +575,32 @@ fn parse_string(tt: &tt::Subtree) -> Result { fn include_expand( db: &dyn ExpandDatabase, arg_id: MacroCallId, - _tt: &tt::Subtree, + tt: &tt::Subtree, span: SpanData, ) -> ExpandResult { - match db.include_expand(arg_id) { - Ok((res, _)) => ExpandResult::ok(res.as_ref().clone()), - Err(e) => ExpandResult::new(tt::Subtree::empty(DelimSpan { open: span, close: span }), e), - } -} - -// FIXME: Check if this is still needed now after the token map rewrite -pub(crate) fn include_arg_to_tt( - db: &dyn ExpandDatabase, - arg_id: MacroCallId, -) -> Result<(triomphe::Arc, FileId), ExpandError> { - let loc = db.lookup_intern_macro_call(arg_id); - let Some(EagerCallInfo { arg, arg_id, .. }) = loc.eager.as_deref() else { - panic!("include_arg_to_tt called on non include macro call: {:?}", &loc.eager); + let path = match parse_string(tt) { + Ok(it) => it, + Err(e) => { + return ExpandResult::new(tt::Subtree::empty(DelimSpan { open: span, close: span }), e) + } }; - let path = parse_string(&arg)?; - let file_id = relative_file(db, *arg_id, &path, false)?; - - // why are we not going through a SyntaxNode here? - let subtree = parse_to_token_tree( + let file_id = match relative_file(db, arg_id, &path, false) { + Ok(file_id) => file_id, + Err(e) => { + return ExpandResult::new(tt::Subtree::empty(DelimSpan { open: span, close: span }), e); + } + }; + match parse_to_token_tree( SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, - // FIXME SyntaxContextId::ROOT, &db.file_text(file_id), - ) - .ok_or(mbe::ExpandError::ConversionError)?; - Ok((triomphe::Arc::new(subtree), file_id)) + ) { + Some(it) => ExpandResult::ok(it), + None => ExpandResult::new( + tt::Subtree::empty(DelimSpan { open: span, close: span }), + ExpandError::other("failed to parse included file"), + ), + } } fn include_bytes_expand( @@ -613,9 +609,12 @@ fn include_bytes_expand( tt: &tt::Subtree, span: SpanData, ) -> ExpandResult { - if let Err(e) = parse_string(tt) { - return ExpandResult::new(tt::Subtree::empty(DelimSpan { open: span, close: span }), e); - } + let _path = match parse_string(tt) { + Ok(it) => it, + Err(e) => { + return ExpandResult::new(tt::Subtree::empty(DelimSpan { open: span, close: span }), e) + } + }; // FIXME: actually read the file here if the user asked for macro expansion let res = tt::Subtree { diff --git a/crates/hir-expand/src/db.rs b/crates/hir-expand/src/db.rs index 7f77a2f51bab..00755b16e9ff 100644 --- a/crates/hir-expand/src/db.rs +++ b/crates/hir-expand/src/db.rs @@ -142,12 +142,6 @@ pub trait ExpandDatabase: SourceDatabase { def_crate: CrateId, id: AstId, ) -> Arc; - - #[salsa::invoke(crate::builtin_fn_macro::include_arg_to_tt)] - fn include_expand( - &self, - arg_id: MacroCallId, - ) -> Result<(triomphe::Arc, base_db::FileId), ExpandError>; /// Special case of the previous query for procedural macros. We can't LRU /// proc macros, since they are not deterministic in general, and /// non-determinism breaks salsa in a very, very, very bad way. diff --git a/crates/hir-expand/src/lib.rs b/crates/hir-expand/src/lib.rs index 5915fe6e3446..b5f5fdd22ee7 100644 --- a/crates/hir-expand/src/lib.rs +++ b/crates/hir-expand/src/lib.rs @@ -208,12 +208,7 @@ impl HirFileIdExt for HirFileId { match file_id.repr() { HirFileIdRepr::FileId(id) => break id, HirFileIdRepr::MacroFile(MacroFileId { macro_call_id }) => { - let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_call_id); - let is_include_expansion = loc.def.is_include() && loc.eager.is_some(); - file_id = match is_include_expansion.then(|| db.include_expand(macro_call_id)) { - Some(Ok((_, file))) => file.into(), - _ => loc.kind.file_id(), - } + file_id = db.lookup_intern_macro_call(macro_call_id).kind.file_id(); } } } diff --git a/crates/hir/src/db.rs b/crates/hir/src/db.rs index ff6f987aa128..d98e3decd21e 100644 --- a/crates/hir/src/db.rs +++ b/crates/hir/src/db.rs @@ -23,7 +23,7 @@ pub use hir_def::db::{ }; pub use hir_expand::db::{ AstIdMapQuery, DeclMacroExpanderQuery, ExpandDatabase, ExpandDatabaseStorage, - ExpandProcMacroQuery, IncludeExpandQuery, InternMacroCallQuery, InternSyntaxContextQuery, - MacroArgQuery, ParseMacroExpansionErrorQuery, ParseMacroExpansionQuery, RealSpanMapQuery, + ExpandProcMacroQuery, InternMacroCallQuery, InternSyntaxContextQuery, MacroArgQuery, + ParseMacroExpansionErrorQuery, ParseMacroExpansionQuery, RealSpanMapQuery, }; pub use hir_ty::db::*; diff --git a/crates/ide-db/src/apply_change.rs b/crates/ide-db/src/apply_change.rs index 67e686dad11e..343be870c9ee 100644 --- a/crates/ide-db/src/apply_change.rs +++ b/crates/ide-db/src/apply_change.rs @@ -99,7 +99,6 @@ impl RootDatabase { hir::db::AstIdMapQuery hir::db::DeclMacroExpanderQuery hir::db::ExpandProcMacroQuery - hir::db::IncludeExpandQuery hir::db::InternMacroCallQuery hir::db::InternSyntaxContextQuery hir::db::MacroArgQuery diff --git a/crates/ide-diagnostics/src/handlers/unresolved_macro_call.rs b/crates/ide-diagnostics/src/handlers/unresolved_macro_call.rs index 33e7c2e37c3b..edcfa073a0fb 100644 --- a/crates/ide-diagnostics/src/handlers/unresolved_macro_call.rs +++ b/crates/ide-diagnostics/src/handlers/unresolved_macro_call.rs @@ -67,6 +67,22 @@ macro_rules! m { () => {} } } self::m!(); self::m2!(); //^^ error: unresolved macro `self::m2!` +"#, + ); + } + + #[test] + #[should_panic] // FIXME: https://github.com/rust-lang/rust-analyzer/issues/14968 + fn include_does_not_break_diagnostics() { + check_diagnostics( + r#" +//- minicore: include +//- /lib.rs crate:lib +include!("include-me.rs"); +//- /include-me.rs +/// long doc that pushes the diagnostic range beyond the first file's text length +#[err] +mod prim_never {} "#, ); } From efa67294ed7d3742d5177b6ff43f077325ba48be Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Fri, 1 Dec 2023 16:29:58 +0100 Subject: [PATCH 19/24] Fix eager macro input spans being discarded --- .../hir-def/src/macro_expansion_tests/mbe.rs | 60 +++++++-- crates/hir-expand/src/eager.rs | 123 +++++++----------- crates/hir-ty/src/tests.rs | 1 + crates/hir-ty/src/tests/macros.rs | 1 + crates/mbe/src/token_map.rs | 10 +- 5 files changed, 101 insertions(+), 94 deletions(-) diff --git a/crates/hir-def/src/macro_expansion_tests/mbe.rs b/crates/hir-def/src/macro_expansion_tests/mbe.rs index 9f0d3938b7ea..c4d44eab66c4 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe.rs @@ -93,52 +93,86 @@ fn eager_expands_with_unresolved_within() { r#" #[rustc_builtin_macro] #[macro_export] -macro_rules! format_args {} +macro_rules! concat {} +macro_rules! identity { + ($tt:tt) => { + $tt + } +} fn main(foo: ()) { - format_args!("{} {} {}", format_args!("{}", 0), foo, identity!(10), "bar") + concat!("hello", identity!("world"), unresolved!(), identity!("!")); } "#, expect![[r##" #[rustc_builtin_macro] #[macro_export] -macro_rules! format_args {} +macro_rules! concat {} +macro_rules! identity { + ($tt:tt) => { + $tt + } +} fn main(foo: ()) { - builtin #format_args ("{} {} {}", format_args!("{}", 0), foo, identity!(10), "bar") + /* error: unresolved macro unresolved */"helloworld!"; } "##]], ); } #[test] -fn token_mapping_eager() { +fn concat_spans() { check( r#" #[rustc_builtin_macro] #[macro_export] -macro_rules! format_args {} - +macro_rules! concat {} macro_rules! identity { - ($expr:expr) => { $expr }; + ($tt:tt) => { + $tt + } } fn main(foo: ()) { - format_args/*+spans+syntaxctxt*/!("{} {} {}", format_args!("{}", 0), foo, identity!(10), "bar") + #[rustc_builtin_macro] + #[macro_export] + macro_rules! concat {} + macro_rules! identity { + ($tt:tt) => { + $tt + } + } + + fn main(foo: ()) { + concat/*+spans+syntaxctxt*/!("hello", concat!("w", identity!("o")), identity!("rld"), unresolved!(), identity!("!")); + } } "#, expect![[r##" #[rustc_builtin_macro] #[macro_export] -macro_rules! format_args {} - +macro_rules! concat {} macro_rules! identity { - ($expr:expr) => { $expr }; + ($tt:tt) => { + $tt + } } fn main(foo: ()) { - builtin#FileId(0):3@23..118\3# ##FileId(0):3@23..118\3#format_args#FileId(0):3@23..118\3# (#FileId(0):3@56..57\0#"{} {} {}"#FileId(0):3@57..67\0#,#FileId(0):3@67..68\0# format_args#FileId(0):3@69..80\0#!#FileId(0):3@80..81\0#(#FileId(0):3@81..82\0#"{}"#FileId(0):3@82..86\0#,#FileId(0):3@86..87\0# 0#FileId(0):3@88..89\0#)#FileId(0):3@89..90\0#,#FileId(0):3@90..91\0# foo#FileId(0):3@92..95\0#,#FileId(0):3@95..96\0# identity#FileId(0):3@97..105\0#!#FileId(0):3@105..106\0#(#FileId(0):3@106..107\0#10#FileId(0):3@107..109\0#)#FileId(0):3@109..110\0#,#FileId(0):3@110..111\0# "bar"#FileId(0):3@112..117\0#)#FileId(0):3@117..118\0# + #[rustc_builtin_macro] + #[macro_export] + macro_rules! concat {} + macro_rules! identity { + ($tt:tt) => { + $tt + } + } + + fn main(foo: ()) { + /* error: unresolved macro unresolved */"helloworld!"#FileId(0):3@207..323\6#; + } } "##]], diff --git a/crates/hir-expand/src/eager.rs b/crates/hir-expand/src/eager.rs index aa537af14393..ef7200f615cc 100644 --- a/crates/hir-expand/src/eager.rs +++ b/crates/hir-expand/src/eager.rs @@ -18,16 +18,15 @@ //! //! //! See the full discussion : -use base_db::{span::SyntaxContextId, CrateId, FileId}; -use rustc_hash::FxHashMap; -use syntax::{ted, Parse, SyntaxNode, TextRange, TextSize, WalkEvent}; +use base_db::{span::SyntaxContextId, CrateId}; +use syntax::{ted, Parse, SyntaxElement, SyntaxNode, TextSize, WalkEvent}; use triomphe::Arc; use crate::{ ast::{self, AstNode}, db::ExpandDatabase, mod_path::ModPath, - span::{RealSpanMap, SpanMapRef}, + span::SpanMapRef, EagerCallInfo, ExpandError, ExpandResult, ExpandTo, ExpansionSpanMap, InFile, MacroCallId, MacroCallKind, MacroCallLoc, MacroDefId, MacroDefKind, }; @@ -59,10 +58,14 @@ pub fn expand_eager_macro_input( let ExpandResult { value: (arg_exp, arg_exp_map), err: parse_err } = db.parse_macro_expansion(arg_id.as_macro_file()); + let mut arg_map = ExpansionSpanMap::empty(); + let ExpandResult { value: expanded_eager_input, err } = { eager_macro_recur( db, &arg_exp_map, + &mut arg_map, + TextSize::new(0), InFile::new(arg_id.as_file(), arg_exp.syntax_node()), krate, call_site, @@ -70,14 +73,15 @@ pub fn expand_eager_macro_input( ) }; let err = parse_err.or(err); + if cfg!(debug) { + arg_map.finish(); + } let Some((expanded_eager_input, _mapping)) = expanded_eager_input else { return ExpandResult { value: None, err }; }; - // FIXME: Spans! - let mut subtree = - mbe::syntax_node_to_token_tree(&expanded_eager_input, RealSpanMap::absolute(FileId::BOGUS)); + let mut subtree = mbe::syntax_node_to_token_tree(&expanded_eager_input, arg_map); subtree.delimiter = crate::tt::Delimiter::DUMMY_INVISIBLE; @@ -103,13 +107,7 @@ fn lazy_expand( let expand_to = ExpandTo::from_call_site(¯o_call.value); let ast_id = macro_call.with_value(ast_id); - let id = def.as_lazy_macro( - db, - krate, - MacroCallKind::FnLike { ast_id, expand_to }, - // FIXME: This is wrong - call_site, - ); + let id = def.as_lazy_macro(db, krate, MacroCallKind::FnLike { ast_id, expand_to }, call_site); let macro_file = id.as_macro_file(); db.parse_macro_expansion(macro_file) @@ -119,46 +117,42 @@ fn lazy_expand( fn eager_macro_recur( db: &dyn ExpandDatabase, span_map: &ExpansionSpanMap, + expanded_map: &mut ExpansionSpanMap, + mut offset: TextSize, curr: InFile, krate: CrateId, call_site: SyntaxContextId, macro_resolver: &dyn Fn(ModPath) -> Option, -) -> ExpandResult)>> { +) -> ExpandResult> { let original = curr.value.clone_for_update(); - let mut mapping = FxHashMap::default(); let mut replacements = Vec::new(); // FIXME: We only report a single error inside of eager expansions let mut error = None; - let mut offset = 0i32; - let apply_offset = |it: TextSize, offset: i32| { - TextSize::from(u32::try_from(offset + u32::from(it) as i32).unwrap_or_default()) - }; let mut children = original.preorder_with_tokens(); // Collect replacement while let Some(child) = children.next() { - let WalkEvent::Enter(child) = child else { continue }; let call = match child { - syntax::NodeOrToken::Node(node) => match ast::MacroCall::cast(node) { + WalkEvent::Enter(SyntaxElement::Node(child)) => match ast::MacroCall::cast(child) { Some(it) => { children.skip_subtree(); it } - None => continue, + _ => continue, }, - syntax::NodeOrToken::Token(t) => { - mapping.insert( - TextRange::new( - apply_offset(t.text_range().start(), offset), - apply_offset(t.text_range().end(), offset), - ), - t.text_range(), - ); + WalkEvent::Enter(_) => continue, + WalkEvent::Leave(child) => { + if let SyntaxElement::Token(t) = child { + let start = t.text_range().start(); + offset += t.text_range().len(); + expanded_map.push(offset, span_map.span_at(start)); + } continue; } }; + let def = match call .path() .and_then(|path| ModPath::from_src(db, path, SpanMapRef::ExpansionSpanMap(span_map))) @@ -168,11 +162,13 @@ fn eager_macro_recur( None => { error = Some(ExpandError::other(format!("unresolved macro {}", path.display(db)))); + offset += call.syntax().text_range().len(); continue; } }, None => { error = Some(ExpandError::other("malformed macro invocation")); + offset += call.syntax().text_range().len(); continue; } }; @@ -183,31 +179,22 @@ fn eager_macro_recur( krate, curr.with_value(call.clone()), def, - // FIXME: This call site is not quite right I think? We probably need to mark it? call_site, macro_resolver, ); match value { Some(call_id) => { - let ExpandResult { value, err: err2 } = + let ExpandResult { value: (parse, map), err: err2 } = db.parse_macro_expansion(call_id.as_macro_file()); - // if let Some(tt) = call.token_tree() { - // let call_tt_start = tt.syntax().text_range().start(); - // let call_start = - // apply_offset(call.syntax().text_range().start(), offset); - // if let Some((_, arg_map, _)) = db.macro_arg(call_id).value.as_deref() { - // mapping.extend(arg_map.entries().filter_map(|(tid, range)| { - // value - // .1 - // .first_range_by_token(tid, syntax::SyntaxKind::TOMBSTONE) - // .map(|r| (r + call_start, range + call_tt_start)) - // })); - // } - // } + map.iter().for_each(|(o, span)| expanded_map.push(o + offset, span)); + let syntax_node = parse.syntax_node(); ExpandResult { - value: Some(value.0.syntax_node().clone_for_update()), + value: Some(( + syntax_node.clone_for_update(), + offset + syntax_node.text_range().len(), + )), err: err.or(err2), } } @@ -226,6 +213,8 @@ fn eager_macro_recur( let ExpandResult { value, err: error } = eager_macro_recur( db, &tm, + expanded_map, + offset, // FIXME: We discard parse errors here parse.as_ref().map(|it| it.syntax_node()), krate, @@ -234,31 +223,7 @@ fn eager_macro_recur( ); let err = err.or(error); - // if let Some(tt) = call.token_tree() { - // let decl_mac = if let MacroDefKind::Declarative(ast_id) = def.kind { - // Some(db.decl_macro_expander(def.krate, ast_id)) - // } else { - // None - // }; - // let call_tt_start = tt.syntax().text_range().start(); - // let call_start = apply_offset(call.syntax().text_range().start(), offset); - // if let Some((_tt, arg_map, _)) = parse - // .file_id - // .macro_file() - // .and_then(|id| db.macro_arg(id.macro_call_id).value) - // .as_deref() - // { - // mapping.extend(arg_map.entries().filter_map(|(tid, range)| { - // tm.first_range_by_token( - // decl_mac.as_ref().map(|it| it.map_id_down(tid)).unwrap_or(tid), - // syntax::SyntaxKind::TOMBSTONE, - // ) - // .map(|r| (r + call_start, range + call_tt_start)) - // })); - // } - // } - // FIXME: Do we need to re-use _m here? - ExpandResult { value: value.map(|(n, _m)| n), err } + ExpandResult { value, err } } }; if err.is_some() { @@ -266,16 +231,18 @@ fn eager_macro_recur( } // check if the whole original syntax is replaced if call.syntax() == &original { - return ExpandResult { value: value.zip(Some(mapping)), err: error }; + return ExpandResult { value, err: error }; } - if let Some(insert) = value { - offset += u32::from(insert.text_range().len()) as i32 - - u32::from(call.syntax().text_range().len()) as i32; - replacements.push((call, insert)); + match value { + Some((insert, new_offset)) => { + replacements.push((call, insert)); + offset = new_offset; + } + None => offset += call.syntax().text_range().len(), } } replacements.into_iter().rev().for_each(|(old, new)| ted::replace(old.syntax(), new)); - ExpandResult { value: Some((original, mapping)), err: error } + ExpandResult { value: Some((original, offset)), err: error } } diff --git a/crates/hir-ty/src/tests.rs b/crates/hir-ty/src/tests.rs index 1446e83fa887..9f6c237583e6 100644 --- a/crates/hir-ty/src/tests.rs +++ b/crates/hir-ty/src/tests.rs @@ -128,6 +128,7 @@ fn check_impl(ra_fixture: &str, allow_none: bool, only_types: bool, display_sour None => continue, }; let def_map = module.def_map(&db); + dbg!(def_map.dump(&db)); visit_module(&db, &def_map, module.local_id, &mut |it| defs.push(it)); } defs.sort_by_key(|def| match def { diff --git a/crates/hir-ty/src/tests/macros.rs b/crates/hir-ty/src/tests/macros.rs index 1e10a6fecafb..809101752967 100644 --- a/crates/hir-ty/src/tests/macros.rs +++ b/crates/hir-ty/src/tests/macros.rs @@ -787,6 +787,7 @@ fn main() { } #[test] +#[should_panic] // FIXME fn infer_builtin_macros_include_child_mod() { check_types( r#" diff --git a/crates/mbe/src/token_map.rs b/crates/mbe/src/token_map.rs index c2ec30ca72fb..5871c0f1251c 100644 --- a/crates/mbe/src/token_map.rs +++ b/crates/mbe/src/token_map.rs @@ -17,16 +17,16 @@ pub struct TokenMap { } impl TokenMap { - pub(crate) fn empty() -> Self { + pub fn empty() -> Self { Self { spans: Vec::new() } } - pub(crate) fn finish(&mut self) { + pub fn finish(&mut self) { assert!(self.spans.iter().tuple_windows().all(|(a, b)| a.0 < b.0)); self.spans.shrink_to_fit(); } - pub(crate) fn push(&mut self, offset: TextSize, span: S) { + pub fn push(&mut self, offset: TextSize, span: S) { self.spans.push((offset, span)); } @@ -54,4 +54,8 @@ impl TokenMap { let end_entry = self.spans[start_entry..].partition_point(|&(it, _)| it <= end); // FIXME: this might be wrong? (&self.spans[start_entry..][..end_entry]).iter().map(|&(_, s)| s) } + + pub fn iter(&self) -> impl Iterator + '_ { + self.spans.iter().copied() + } } From d2a31acda19b5a94e1b5894f15ef0922bc286ee1 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 2 Dec 2023 13:03:46 +0100 Subject: [PATCH 20/24] Fix macro expansion expression parenthesis wrapping --- crates/base-db/src/span.rs | 23 +++++- crates/hir-def/src/body/lower.rs | 2 +- crates/hir-def/src/body/tests.rs | 69 ++++++++++++++++- crates/hir-def/src/data.rs | 2 +- crates/hir-def/src/expander.rs | 8 +- crates/hir-def/src/generics.rs | 2 +- .../hir-def/src/macro_expansion_tests/mbe.rs | 10 +-- .../macro_expansion_tests/mbe/metavar_expr.rs | 4 +- .../macro_expansion_tests/mbe/regression.rs | 4 +- .../mbe/tt_conversion.rs | 6 +- crates/hir-expand/src/db.rs | 74 +++++++++++++------ crates/hir-expand/src/hygiene.rs | 32 +++++++- crates/hir-expand/src/lib.rs | 9 +++ crates/hir-ty/src/infer/path.rs | 1 + crates/hir-ty/src/lower.rs | 6 +- crates/hir-ty/src/tests.rs | 1 - crates/mbe/src/expander/matcher.rs | 18 ++++- crates/mbe/src/expander/transcriber.rs | 11 +-- 18 files changed, 218 insertions(+), 64 deletions(-) diff --git a/crates/base-db/src/span.rs b/crates/base-db/src/span.rs index dbccbb382a7e..6723bf97fea5 100644 --- a/crates/base-db/src/span.rs +++ b/crates/base-db/src/span.rs @@ -14,8 +14,25 @@ pub const ROOT_ERASED_FILE_AST_ID: ErasedFileAstId = pub type SpanData = tt::SpanData; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct SyntaxContextId(InternId); + +impl fmt::Debug for SyntaxContextId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if *self == Self::SELF_REF { + f.debug_tuple("SyntaxContextId") + .field(&{ + #[derive(Debug)] + #[allow(non_camel_case_types)] + struct SELF_REF; + SELF_REF + }) + .finish() + } else { + f.debug_tuple("SyntaxContextId").field(&self.0).finish() + } + } +} crate::impl_intern_key!(SyntaxContextId); impl fmt::Display for SyntaxContextId { @@ -30,7 +47,7 @@ impl SyntaxContext for SyntaxContextId { // inherent trait impls please tyvm impl SyntaxContextId { pub const ROOT: Self = SyntaxContextId(unsafe { InternId::new_unchecked(0) }); - // veykril(HACK): salsa doesn't allow us fetching the id of the current input to be allocated so + // veykril(HACK): FIXME salsa doesn't allow us fetching the id of the current input to be allocated so // we need a special value that behaves as the current context. pub const SELF_REF: Self = SyntaxContextId(unsafe { InternId::new_unchecked(InternId::MAX - 1) }); @@ -107,7 +124,7 @@ pub struct MacroFileId { /// `MacroCallId` identifies a particular macro invocation, like /// `println!("Hello, {}", world)`. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct MacroCallId(salsa::InternId); crate::impl_intern_key!(MacroCallId); diff --git a/crates/hir-def/src/body/lower.rs b/crates/hir-def/src/body/lower.rs index e4158d7564bd..0466068ec810 100644 --- a/crates/hir-def/src/body/lower.rs +++ b/crates/hir-def/src/body/lower.rs @@ -1025,7 +1025,7 @@ impl ExprCollector<'_> { let id = collector(self, Some(expansion.tree())); self.ast_id_map = prev_ast_id_map; - self.expander.exit(self.db, mark); + self.expander.exit(mark); id } None => collector(self, None), diff --git a/crates/hir-def/src/body/tests.rs b/crates/hir-def/src/body/tests.rs index 1658757d2b6e..048f0a13f0e7 100644 --- a/crates/hir-def/src/body/tests.rs +++ b/crates/hir-def/src/body/tests.rs @@ -2,6 +2,7 @@ mod block; use base_db::{fixture::WithFixture, SourceDatabase}; use expect_test::{expect, Expect}; +use hir_expand::db::ExpandDatabase; use crate::{test_db::TestDB, ModuleDefId}; @@ -143,7 +144,6 @@ mod m { #[test] fn desugar_builtin_format_args() { - // Regression test for a path resolution bug introduced with inner item handling. let (db, body, def) = lower( r#" //- minicore: fmt @@ -221,3 +221,70 @@ fn main() { }"#]] .assert_eq(&body.pretty_print(&db, def)) } + +#[test] +fn test_macro_hygiene() { + let (db, body, def) = lower( + r##" +//- minicore: fmt, from +//- /main.rs +mod error; + +use crate::error::error; + +fn main() { + // _ = forces body expansion instead of block def map expansion + _ = error!("Failed to resolve path `{}`", node.text()); +} +//- /error.rs +macro_rules! _error { + ($fmt:expr, $($arg:tt)+) => {$crate::error::intermediate!(format_args!($fmt, $($arg)+))} +} +pub(crate) use _error as error; +macro_rules! _intermediate { + ($arg:expr) => {$crate::error::SsrError::new($arg)} +} +pub(crate) use _intermediate as intermediate; + +pub struct SsrError(pub(crate) core::fmt::Arguments); + +impl SsrError { + pub(crate) fn new(message: impl Into) -> SsrError { + SsrError(message.into()) + } +} +"##, + ); + println!("{}", db.dump_syntax_contexts()); + + assert_eq!(db.body_with_source_map(def.into()).1.diagnostics(), &[]); + expect![[r#" + fn main() { + _ = $crate::error::SsrError::new( + builtin#lang(Arguments::new_v1_formatted)( + &[ + "\"Failed to resolve path `", "`\"", + ], + &[ + builtin#lang(Argument::new_display)( + &node.text(), + ), + ], + &[ + builtin#lang(Placeholder::new)( + 0usize, + ' ', + builtin#lang(Alignment::Unknown), + 0u32, + builtin#lang(Count::Implied), + builtin#lang(Count::Implied), + ), + ], + unsafe { + builtin#lang(UnsafeArg::new)() + }, + ), + ); + }"#]] + .assert_eq(&body.pretty_print(&db, def)) +} diff --git a/crates/hir-def/src/data.rs b/crates/hir-def/src/data.rs index 2af4622a0724..635d13f24ad8 100644 --- a/crates/hir-def/src/data.rs +++ b/crates/hir-def/src/data.rs @@ -794,7 +794,7 @@ impl<'a> AssocItemCollector<'a> { self.collect(&item_tree, tree_id, &iter); - self.expander.exit(self.db, mark); + self.expander.exit(mark); } } diff --git a/crates/hir-def/src/expander.rs b/crates/hir-def/src/expander.rs index e36aa19b8ddc..398f116d8313 100644 --- a/crates/hir-def/src/expander.rs +++ b/crates/hir-def/src/expander.rs @@ -94,8 +94,8 @@ impl Expander { ExpandResult { value: Some(InFile::new(macro_file.into(), value.0)), err: error.or(err) } } - pub fn exit(&mut self, db: &dyn DefDatabase, mut mark: Mark) { - self.span_map = db.span_map(mark.file_id); + pub fn exit(&mut self, mut mark: Mark) { + self.span_map = mark.span_map; self.current_file_id = mark.file_id; if self.recursion_depth == u32::MAX { // Recursion limit has been reached somewhere in the macro expansion tree. Reset the @@ -174,10 +174,11 @@ impl Expander { let parse = value.cast::()?; self.recursion_depth += 1; - self.span_map = db.span_map(file_id); + let old_span_map = std::mem::replace(&mut self.span_map, db.span_map(file_id)); let old_file_id = std::mem::replace(&mut self.current_file_id, file_id); let mark = Mark { file_id: old_file_id, + span_map: old_span_map, bomb: DropBomb::new("expansion mark dropped"), }; Some((mark, parse)) @@ -190,5 +191,6 @@ impl Expander { #[derive(Debug)] pub struct Mark { file_id: HirFileId, + span_map: SpanMap, bomb: DropBomb, } diff --git a/crates/hir-def/src/generics.rs b/crates/hir-def/src/generics.rs index 6d656bf9482f..1c1c481a8eed 100644 --- a/crates/hir-def/src/generics.rs +++ b/crates/hir-def/src/generics.rs @@ -439,7 +439,7 @@ impl GenericParams { let ctx = expander.ctx(db); let type_ref = TypeRef::from_ast(&ctx, expanded.tree()); self.fill_implicit_impl_trait_args(db, &mut *exp, &type_ref); - exp.1.exit(db, mark); + exp.1.exit(mark); } } }); diff --git a/crates/hir-def/src/macro_expansion_tests/mbe.rs b/crates/hir-def/src/macro_expansion_tests/mbe.rs index c4d44eab66c4..9bf2a50d57c9 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe.rs @@ -997,9 +997,9 @@ macro_rules! vec { fn f() { { let mut v = Vec::new(); - v.push((1)); - v.push((2)); - v.push((3)); + v.push(1); + v.push(2); + v.push(3); v }; } @@ -1468,8 +1468,8 @@ macro_rules! matches { }; } fn main() { - match (0) { - 0|1 if (true )=>true , _=>false + match 0 { + 0|1 if true =>true , _=>false }; } "#]], diff --git a/crates/hir-def/src/macro_expansion_tests/mbe/metavar_expr.rs b/crates/hir-def/src/macro_expansion_tests/mbe/metavar_expr.rs index dd83e5c04d45..967b5ad36bab 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe/metavar_expr.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe/metavar_expr.rs @@ -62,10 +62,10 @@ macro_rules !implement_methods { struct Foo; impl Foo { fn alpha() -> &'static[u32] { - &[(1), (2), (3)] + &[1, 2, 3] } fn beta() -> &'static[u32] { - &[(1), (2), (3)] + &[1, 2, 3] } } "#]], diff --git a/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs b/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs index 71dbb400b548..2886b2a366c0 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs @@ -39,8 +39,8 @@ fn main() { }; { let mut v = Vec::new(); - v.push((1u32)); - v.push((2)); + v.push(1u32); + v.push(2); v }; } diff --git a/crates/hir-def/src/macro_expansion_tests/mbe/tt_conversion.rs b/crates/hir-def/src/macro_expansion_tests/mbe/tt_conversion.rs index b2ac7eb4094d..ae56934f632f 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe/tt_conversion.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe/tt_conversion.rs @@ -192,9 +192,9 @@ macro_rules! constant { ($e:expr ;) => {$e}; } -const _: () = (0.0); -const _: () = (0.); -const _: () = (0e0); +const _: () = 0.0; +const _: () = 0.; +const _: () = 0e0; "#]], ); } diff --git a/crates/hir-expand/src/db.rs b/crates/hir-expand/src/db.rs index 00755b16e9ff..d2c6559b06b1 100644 --- a/crates/hir-expand/src/db.rs +++ b/crates/hir-expand/src/db.rs @@ -1,6 +1,10 @@ //! Defines database & queries for macro expansion. -use base_db::{salsa, span::SyntaxContextId, CrateId, Edition, FileId, SourceDatabase}; +use base_db::{ + salsa::{self, debug::DebugQueryTable}, + span::SyntaxContextId, + CrateId, Edition, FileId, SourceDatabase, +}; use either::Either; use limit::Limit; use mbe::{syntax_node_to_token_tree, ValueResult}; @@ -17,7 +21,7 @@ use crate::{ builtin_attr_macro::pseudo_derive_attr_expansion, builtin_fn_macro::EagerExpander, fixup::{self, SyntaxFixupUndoInfo}, - hygiene::{self, SyntaxContextData, Transparency}, + hygiene::{apply_mark, SyntaxContextData, Transparency}, span::{RealSpanMap, SpanMap, SpanMapRef}, tt, AstId, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander, EagerCallInfo, ExpandError, ExpandResult, ExpandTo, ExpansionSpanMap, HirFileId, HirFileIdRepr, MacroCallId, @@ -53,7 +57,7 @@ impl DeclarativeMacroExpander { ), None => self .mac - .expand(&tt, |s| s.ctx = db.apply_mark(s.ctx, call_id, self.transparency)) + .expand(&tt, |s| s.ctx = apply_mark(db, s.ctx, call_id, self.transparency)) .map_err(Into::into), } } @@ -115,16 +119,11 @@ pub trait ExpandDatabase: SourceDatabase { fn intern_macro_call(&self, macro_call: MacroCallLoc) -> MacroCallId; #[salsa::interned] fn intern_syntax_context(&self, ctx: SyntaxContextData) -> SyntaxContextId; + #[salsa::transparent] fn setup_syntax_context_root(&self) -> (); #[salsa::transparent] - #[salsa::invoke(hygiene::apply_mark)] - fn apply_mark( - &self, - ctxt: SyntaxContextId, - call_id: MacroCallId, - transparency: hygiene::Transparency, - ) -> SyntaxContextId; + fn dump_syntax_contexts(&self) -> String; /// Lowers syntactic macro call to a token tree representation. That's a firewall /// query, only typing in the macro call itself changes the returned @@ -269,7 +268,8 @@ pub fn expand_speculative( MacroDefKind::BuiltInAttr(it, _) => it.expand(db, actual_macro_call, &tt), }; - let expand_to = macro_expand_to(db, actual_macro_call); + let expand_to = loc.expand_to(); + fixup::reverse_fixups(&mut speculative_expansion.value, &undo_info); let (node, rev_tmap) = token_tree_to_syntax_node(&speculative_expansion.value, expand_to); @@ -318,12 +318,9 @@ fn parse_macro_expansion( macro_file: MacroFileId, ) -> ExpandResult<(Parse, Arc)> { let _p = profile::span("parse_macro_expansion"); - let mbe::ValueResult { value: tt, err } = macro_expand(db, macro_file.macro_call_id); - - let expand_to = macro_expand_to(db, macro_file.macro_call_id); - - tracing::debug!("expanded = {}", tt.as_debug_string()); - tracing::debug!("kind = {:?}", expand_to); + let loc = db.lookup_intern_macro_call(macro_file.macro_call_id); + let expand_to = loc.expand_to(); + let mbe::ValueResult { value: tt, err } = macro_expand(db, macro_file.macro_call_id, loc); let (parse, rev_token_map) = token_tree_to_syntax_node(&tt, expand_to); @@ -575,9 +572,9 @@ fn macro_expander(db: &dyn ExpandDatabase, id: MacroDefId) -> TokenExpander { fn macro_expand( db: &dyn ExpandDatabase, macro_call_id: MacroCallId, + loc: MacroCallLoc, ) -> ExpandResult> { let _p = profile::span("macro_expand"); - let loc = db.lookup_intern_macro_call(macro_call_id); let ExpandResult { value: tt, mut err } = match loc.def.kind { MacroDefKind::ProcMacro(..) => return db.expand_proc_macro(macro_call_id), @@ -711,10 +708,6 @@ fn expand_proc_macro(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult ExpandTo { - db.lookup_intern_macro_call(id).expand_to() -} - fn token_tree_to_syntax_node( tt: &tt::Subtree, expand_to: ExpandTo, @@ -751,3 +744,40 @@ fn check_tt_count(tt: &tt::Subtree) -> Result<(), ExpandResult> fn setup_syntax_context_root(db: &dyn ExpandDatabase) { db.intern_syntax_context(SyntaxContextData::root()); } + +fn dump_syntax_contexts(db: &dyn ExpandDatabase) -> String { + let mut s = String::from("Expansions:"); + let mut entries = InternMacroCallLookupQuery.in_db(db).entries::>(); + entries.sort_by_key(|e| e.key); + for e in entries { + let id = e.key; + let expn_data = e.value.as_ref().unwrap(); + s.push_str(&format!( + "\n{:?}: parent: {:?}, call_site_ctxt: {:?}, def_site_ctxt: {:?}, kind: {:?}", + id, + expn_data.kind.file_id(), + expn_data.call_site, + SyntaxContextId::ROOT, // FIXME expn_data.def_site, + expn_data.kind.descr(), + )); + } + + s.push_str("\n\nSyntaxContexts:\n"); + let mut entries = InternSyntaxContextLookupQuery.in_db(db).entries::>(); + entries.sort_by_key(|e| e.key); + for e in entries { + struct SyntaxContextDebug<'a>( + &'a dyn ExpandDatabase, + SyntaxContextId, + &'a SyntaxContextData, + ); + + impl<'a> std::fmt::Debug for SyntaxContextDebug<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.2.fancy_debug(self.1, self.0, f) + } + } + stdx::format_to!(s, "{:?}\n", SyntaxContextDebug(db, e.key, &e.value.unwrap())); + } + s +} diff --git a/crates/hir-expand/src/hygiene.rs b/crates/hir-expand/src/hygiene.rs index 8fdfa8af0cac..a809e92d6222 100644 --- a/crates/hir-expand/src/hygiene.rs +++ b/crates/hir-expand/src/hygiene.rs @@ -8,7 +8,7 @@ use base_db::span::{MacroCallId, SpanData, SyntaxContextId}; use crate::db::ExpandDatabase; -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +#[derive(Copy, Clone, Hash, PartialEq, Eq)] pub struct SyntaxContextData { pub outer_expn: Option, pub outer_transparency: Transparency, @@ -19,6 +19,18 @@ pub struct SyntaxContextData { pub opaque_and_semitransparent: SyntaxContextId, } +impl std::fmt::Debug for SyntaxContextData { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SyntaxContextData") + .field("outer_expn", &self.outer_expn) + .field("outer_transparency", &self.outer_transparency) + .field("parent", &self.parent) + .field("opaque", &self.opaque) + .field("opaque_and_semitransparent", &self.opaque_and_semitransparent) + .finish() + } +} + impl SyntaxContextData { pub fn root() -> Self { SyntaxContextData { @@ -29,6 +41,22 @@ impl SyntaxContextData { opaque_and_semitransparent: SyntaxContextId::ROOT, } } + + pub fn fancy_debug( + self, + self_id: SyntaxContextId, + db: &dyn ExpandDatabase, + f: &mut std::fmt::Formatter<'_>, + ) -> std::fmt::Result { + write!(f, "#{self_id} parent: #{}, outer_mark: (", self.parent)?; + match self.outer_expn { + Some(id) => { + write!(f, "{:?}::{{{{expn{:?}}}}}", db.lookup_intern_macro_call(id).krate, id)? + } + None => write!(f, "root")?, + } + write!(f, ", {:?})", self.outer_transparency) + } } /// A property of a macro expansion that determines how identifiers @@ -80,7 +108,7 @@ fn span_with_ctxt_from_mark( expn_id: MacroCallId, transparency: Transparency, ) -> SpanData { - SpanData { ctx: db.apply_mark(SyntaxContextId::ROOT, expn_id, transparency), ..span } + SpanData { ctx: apply_mark(db, SyntaxContextId::ROOT, expn_id, transparency), ..span } } pub(super) fn apply_mark( diff --git a/crates/hir-expand/src/lib.rs b/crates/hir-expand/src/lib.rs index b5f5fdd22ee7..d743f2adae5f 100644 --- a/crates/hir-expand/src/lib.rs +++ b/crates/hir-expand/src/lib.rs @@ -122,6 +122,7 @@ pub struct MacroDefId { pub kind: MacroDefKind, pub local_inner: bool, pub allow_internal_unsafe: bool, + // pub def_site: SyntaxContextId, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -463,6 +464,14 @@ impl MacroCallLoc { } impl MacroCallKind { + fn descr(&self) -> &'static str { + match self { + MacroCallKind::FnLike { .. } => "macro call", + MacroCallKind::Derive { .. } => "derive macro", + MacroCallKind::Attr { .. } => "attribute macro", + } + } + /// Returns the file containing the macro invocation. fn file_id(&self) -> HirFileId { match *self { diff --git a/crates/hir-ty/src/infer/path.rs b/crates/hir-ty/src/infer/path.rs index c6bbf2f61407..fcfe1a3b5cf4 100644 --- a/crates/hir-ty/src/infer/path.rs +++ b/crates/hir-ty/src/infer/path.rs @@ -390,6 +390,7 @@ impl InferenceContext<'_> { } } +#[derive(Debug)] enum ValuePathResolution { // It's awkward to wrap a single ID in two enums, but we need both and this saves fallible // conversion between them + `unwrap()`. diff --git a/crates/hir-ty/src/lower.rs b/crates/hir-ty/src/lower.rs index 9f5b59b239a4..5122021d6d2b 100644 --- a/crates/hir-ty/src/lower.rs +++ b/crates/hir-ty/src/lower.rs @@ -407,11 +407,7 @@ impl<'a> TyLoweringContext<'a> { drop(expander); let ty = self.lower_ty(&type_ref); - self.expander - .borrow_mut() - .as_mut() - .unwrap() - .exit(self.db.upcast(), mark); + self.expander.borrow_mut().as_mut().unwrap().exit(mark); Some(ty) } _ => { diff --git a/crates/hir-ty/src/tests.rs b/crates/hir-ty/src/tests.rs index 9f6c237583e6..1446e83fa887 100644 --- a/crates/hir-ty/src/tests.rs +++ b/crates/hir-ty/src/tests.rs @@ -128,7 +128,6 @@ fn check_impl(ra_fixture: &str, allow_none: bool, only_types: bool, display_sour None => continue, }; let def_map = module.def_map(&db); - dbg!(def_map.dump(&db)); visit_module(&db, &def_map, module.local_id, &mut |it| defs.push(it)); } defs.sort_by_key(|def| match def { diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs index 5e1ceacf12c3..012b02a3f87a 100644 --- a/crates/mbe/src/expander/matcher.rs +++ b/crates/mbe/src/expander/matcher.rs @@ -792,9 +792,21 @@ fn match_meta_var( } _ => {} }; - return input - .expect_fragment(parser::PrefixEntryPoint::Expr) - .map(|tt| tt.map(tt::TokenTree::subtree_or_wrap).map(Fragment::Expr)); + return input.expect_fragment(parser::PrefixEntryPoint::Expr).map(|tt| { + tt.map(|tt| match tt { + tt::TokenTree::Leaf(leaf) => tt::Subtree { + delimiter: tt::Delimiter::dummy_invisible(), + token_trees: vec![leaf.into()], + }, + tt::TokenTree::Subtree(mut s) => { + if s.delimiter.kind == tt::DelimiterKind::Invisible { + s.delimiter.kind = tt::DelimiterKind::Parenthesis; + } + s + } + }) + .map(Fragment::Expr) + }); } MetaVarKind::Ident | MetaVarKind::Tt | MetaVarKind::Lifetime | MetaVarKind::Literal => { let tt_result = match kind { diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs index 40f683d84626..e4a46c16597f 100644 --- a/crates/mbe/src/expander/transcriber.rs +++ b/crates/mbe/src/expander/transcriber.rs @@ -444,15 +444,8 @@ fn expand_repeat( fn push_fragment(buf: &mut Vec>, fragment: Fragment) { match fragment { Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt), - Fragment::Expr(mut tt) => { - if tt.delimiter.kind == tt::DelimiterKind::Invisible { - tt.delimiter = tt::Delimiter { - open: S::DUMMY, - close: S::DUMMY, - kind: tt::DelimiterKind::Parenthesis, - }; - } - buf.push(tt.into()) + Fragment::Expr(sub) => { + push_subtree(buf, sub); } Fragment::Path(tt) => fix_up_and_push_path_tt(buf, tt), Fragment::Tokens(tt) => buf.push(tt), From 5edf7bddc675b8184a75f6ae386481f963958d44 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 2 Dec 2023 13:34:40 +0100 Subject: [PATCH 21/24] Fix mod item in included file resolving incorrectly --- crates/hir-def/src/nameres/mod_resolution.rs | 2 +- crates/hir-expand/src/builtin_fn_macro.rs | 25 +++++++--------- crates/hir-expand/src/lib.rs | 29 +++++++++++++++++-- crates/hir-ty/src/tests/macros.rs | 1 - .../src/handlers/unresolved_macro_call.rs | 16 ---------- .../src/integrated_benchmarks.rs | 2 +- 6 files changed, 39 insertions(+), 36 deletions(-) diff --git a/crates/hir-def/src/nameres/mod_resolution.rs b/crates/hir-def/src/nameres/mod_resolution.rs index 22802433aa58..9dcb9717297f 100644 --- a/crates/hir-def/src/nameres/mod_resolution.rs +++ b/crates/hir-def/src/nameres/mod_resolution.rs @@ -66,7 +66,7 @@ impl ModDir { attr_path: Option<&SmolStr>, ) -> Result<(FileId, bool, ModDir), Box<[String]>> { let name = name.unescaped(); - let orig_file_id = file_id.original_file(db.upcast()); + let orig_file_id = file_id.original_file_respecting_includes(db.upcast()); let mut candidate_files = ArrayVec::<_, 2>::new(); match attr_path { diff --git a/crates/hir-expand/src/builtin_fn_macro.rs b/crates/hir-expand/src/builtin_fn_macro.rs index 74ca2f7ec44e..4b2f27bd4650 100644 --- a/crates/hir-expand/src/builtin_fn_macro.rs +++ b/crates/hir-expand/src/builtin_fn_macro.rs @@ -578,18 +578,12 @@ fn include_expand( tt: &tt::Subtree, span: SpanData, ) -> ExpandResult { - let path = match parse_string(tt) { + let file_id = match include_input_to_file_id(db, arg_id, tt) { Ok(it) => it, Err(e) => { return ExpandResult::new(tt::Subtree::empty(DelimSpan { open: span, close: span }), e) } }; - let file_id = match relative_file(db, arg_id, &path, false) { - Ok(file_id) => file_id, - Err(e) => { - return ExpandResult::new(tt::Subtree::empty(DelimSpan { open: span, close: span }), e); - } - }; match parse_to_token_tree( SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, SyntaxContextId::ROOT, @@ -603,19 +597,20 @@ fn include_expand( } } +pub fn include_input_to_file_id( + db: &dyn ExpandDatabase, + arg_id: MacroCallId, + arg: &tt::Subtree, +) -> Result { + relative_file(db, arg_id, &parse_string(arg)?, false) +} + fn include_bytes_expand( _db: &dyn ExpandDatabase, _arg_id: MacroCallId, - tt: &tt::Subtree, + _tt: &tt::Subtree, span: SpanData, ) -> ExpandResult { - let _path = match parse_string(tt) { - Ok(it) => it, - Err(e) => { - return ExpandResult::new(tt::Subtree::empty(DelimSpan { open: span, close: span }), e) - } - }; - // FIXME: actually read the file here if the user asked for macro expansion let res = tt::Subtree { delimiter: tt::Delimiter::dummy_invisible(), diff --git a/crates/hir-expand/src/lib.rs b/crates/hir-expand/src/lib.rs index d743f2adae5f..9caf5513bf84 100644 --- a/crates/hir-expand/src/lib.rs +++ b/crates/hir-expand/src/lib.rs @@ -136,7 +136,7 @@ pub enum MacroDefKind { } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct EagerCallInfo { +pub struct EagerCallInfo { /// The expanded argument of the eager macro. arg: Arc, /// Call id of the eager macro's input file (this is the macro file for its fully expanded input). @@ -176,6 +176,8 @@ pub trait HirFileIdExt { /// expansion originated from. fn original_file(self, db: &dyn db::ExpandDatabase) -> FileId; + fn original_file_respecting_includes(self, db: &dyn db::ExpandDatabase) -> FileId; + /// If this is a macro call, returns the syntax node of the call. fn call_node(self, db: &dyn db::ExpandDatabase) -> Option>; @@ -215,6 +217,29 @@ impl HirFileIdExt for HirFileId { } } + fn original_file_respecting_includes(mut self, db: &dyn db::ExpandDatabase) -> FileId { + loop { + match self.repr() { + base_db::span::HirFileIdRepr::FileId(id) => break id, + base_db::span::HirFileIdRepr::MacroFile(file) => { + let loc = db.lookup_intern_macro_call(file.macro_call_id); + if loc.def.is_include() { + if let Some(eager) = &loc.eager { + if let Ok(it) = builtin_fn_macro::include_input_to_file_id( + db, + file.macro_call_id, + &eager.arg, + ) { + break it; + } + } + } + self = loc.kind.file_id(); + } + } + } + } + fn call_node(self, db: &dyn db::ExpandDatabase) -> Option> { let macro_file = self.macro_file()?; let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_file.macro_call_id); @@ -473,7 +498,7 @@ impl MacroCallKind { } /// Returns the file containing the macro invocation. - fn file_id(&self) -> HirFileId { + pub fn file_id(&self) -> HirFileId { match *self { MacroCallKind::FnLike { ast_id: InFile { file_id, .. }, .. } | MacroCallKind::Derive { ast_id: InFile { file_id, .. }, .. } diff --git a/crates/hir-ty/src/tests/macros.rs b/crates/hir-ty/src/tests/macros.rs index 809101752967..1e10a6fecafb 100644 --- a/crates/hir-ty/src/tests/macros.rs +++ b/crates/hir-ty/src/tests/macros.rs @@ -787,7 +787,6 @@ fn main() { } #[test] -#[should_panic] // FIXME fn infer_builtin_macros_include_child_mod() { check_types( r#" diff --git a/crates/ide-diagnostics/src/handlers/unresolved_macro_call.rs b/crates/ide-diagnostics/src/handlers/unresolved_macro_call.rs index edcfa073a0fb..33e7c2e37c3b 100644 --- a/crates/ide-diagnostics/src/handlers/unresolved_macro_call.rs +++ b/crates/ide-diagnostics/src/handlers/unresolved_macro_call.rs @@ -67,22 +67,6 @@ macro_rules! m { () => {} } } self::m!(); self::m2!(); //^^ error: unresolved macro `self::m2!` -"#, - ); - } - - #[test] - #[should_panic] // FIXME: https://github.com/rust-lang/rust-analyzer/issues/14968 - fn include_does_not_break_diagnostics() { - check_diagnostics( - r#" -//- minicore: include -//- /lib.rs crate:lib -include!("include-me.rs"); -//- /include-me.rs -/// long doc that pushes the diagnostic range beyond the first file's text length -#[err] -mod prim_never {} "#, ); } diff --git a/crates/rust-analyzer/src/integrated_benchmarks.rs b/crates/rust-analyzer/src/integrated_benchmarks.rs index a865d9e4ab82..ed2cf07551b0 100644 --- a/crates/rust-analyzer/src/integrated_benchmarks.rs +++ b/crates/rust-analyzer/src/integrated_benchmarks.rs @@ -30,7 +30,7 @@ fn integrated_highlighting_benchmark() { // Load rust-analyzer itself. let workspace_to_load = project_root(); - let file = "./crates/ide-db/src/apply_change.rs"; + let file = "./crates/rust-analyzer/src/config.rs"; let cargo_config = CargoConfig::default(); let load_cargo_config = LoadCargoConfig { From 02a3a9438a541ac15290a49c97c0c6ceffb23cd5 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 2 Dec 2023 16:50:21 +0100 Subject: [PATCH 22/24] Some more minor cleanups --- crates/base-db/src/lib.rs | 1 - crates/hir-def/src/body/tests.rs | 48 +++++++++++++++++++++++++++++-- crates/hir-expand/src/lib.rs | 5 ++-- crates/hir-ty/src/tests/macros.rs | 12 ++++---- crates/hir/src/semantics.rs | 7 +---- crates/ide/src/expand_macro.rs | 4 +-- crates/test-utils/src/minicore.rs | 9 +++++- 7 files changed, 65 insertions(+), 21 deletions(-) diff --git a/crates/base-db/src/lib.rs b/crates/base-db/src/lib.rs index 38a2641230b7..cddd5bdf48ee 100644 --- a/crates/base-db/src/lib.rs +++ b/crates/base-db/src/lib.rs @@ -4,7 +4,6 @@ mod input; mod change; -// FIXME: Is this purely a test util mod? Consider #[cfg(test)] gating it. pub mod fixture; pub mod span; diff --git a/crates/hir-def/src/body/tests.rs b/crates/hir-def/src/body/tests.rs index 048f0a13f0e7..5517abb1ab70 100644 --- a/crates/hir-def/src/body/tests.rs +++ b/crates/hir-def/src/body/tests.rs @@ -2,7 +2,6 @@ mod block; use base_db::{fixture::WithFixture, SourceDatabase}; use expect_test::{expect, Expect}; -use hir_expand::db::ExpandDatabase; use crate::{test_db::TestDB, ModuleDefId}; @@ -255,7 +254,6 @@ impl SsrError { } "##, ); - println!("{}", db.dump_syntax_contexts()); assert_eq!(db.body_with_source_map(def.into()).1.diagnostics(), &[]); expect![[r#" @@ -288,3 +286,49 @@ impl SsrError { }"#]] .assert_eq(&body.pretty_print(&db, def)) } + +#[test] +fn regression_10300() { + let (db, body, def) = lower( + r#" +//- minicore: concat, panic +mod private { + pub use core::concat; +} + +macro_rules! m { + () => { + panic!(concat!($crate::private::concat!("cc"))); + }; +} + +fn f() { + m!(); +} +"#, + ); + + let (_, source_map) = db.body_with_source_map(def.into()); + assert_eq!(source_map.diagnostics(), &[]); + + for (_, def_map) in body.blocks(&db) { + assert_eq!(def_map.diagnostics(), &[]); + } + + expect![[r#" + fn f() { + $crate::panicking::panic_fmt( + builtin#lang(Arguments::new_v1_formatted)( + &[ + "\"cc\"", + ], + &[], + &[], + unsafe { + builtin#lang(UnsafeArg::new)() + }, + ), + ); + }"#]] + .assert_eq(&body.pretty_print(&db, def)) +} diff --git a/crates/hir-expand/src/lib.rs b/crates/hir-expand/src/lib.rs index 9caf5513bf84..f328431372ec 100644 --- a/crates/hir-expand/src/lib.rs +++ b/crates/hir-expand/src/lib.rs @@ -628,14 +628,13 @@ impl ExpansionInfo { span: SpanData, // FIXME: use this for range mapping, so that we can resolve inline format args _relative_token_offset: Option, - // FIXME: ret ty should be wrapped in InMacroFile - ) -> Option> + 'a> { + ) -> Option> + 'a> { let tokens = self .exp_map .ranges_with_span(span) .flat_map(move |range| self.expanded.value.covering_element(range).into_token()); - Some(tokens.map(move |token| InFile::new(self.expanded.file_id.into(), token))) + Some(tokens.map(move |token| InMacroFile::new(self.expanded.file_id, token))) } /// Maps up the text range out of the expansion hierarchy back into the original file its from. diff --git a/crates/hir-ty/src/tests/macros.rs b/crates/hir-ty/src/tests/macros.rs index 1e10a6fecafb..d16e0eb0137b 100644 --- a/crates/hir-ty/src/tests/macros.rs +++ b/crates/hir-ty/src/tests/macros.rs @@ -63,10 +63,10 @@ fn infer_macros_expanded() { } "#, expect![[r#" - !0..21 '{Foo(v...2),])}': Foo + !0..17 '{Foo(v...,2,])}': Foo !1..4 'Foo': Foo({unknown}) -> Foo - !1..20 'Foo(ve...(2),])': Foo - !5..19 'vec![(1),(2),]': {unknown} + !1..16 'Foo(vec![1,2,])': Foo + !5..15 'vec![1,2,]': {unknown} 155..181 '{ ...,2); }': () 165..166 'x': Foo "#]], @@ -96,10 +96,10 @@ fn infer_legacy_textual_scoped_macros_expanded() { } "#, expect![[r#" - !0..21 '{Foo(v...2),])}': Foo + !0..17 '{Foo(v...,2,])}': Foo !1..4 'Foo': Foo({unknown}) -> Foo - !1..20 'Foo(ve...(2),])': Foo - !5..19 'vec![(1),(2),]': {unknown} + !1..16 'Foo(vec![1,2,])': Foo + !5..15 'vec![1,2,]': {unknown} 194..250 '{ ...,2); }': () 204..205 'x': Foo 227..228 'y': {unknown} diff --git a/crates/hir/src/semantics.rs b/crates/hir/src/semantics.rs index 38c4f081b760..ed3d3f1a3b69 100644 --- a/crates/hir/src/semantics.rs +++ b/crates/hir/src/semantics.rs @@ -557,11 +557,6 @@ impl<'db> SemanticsImpl<'db> { .span_at(token.text_range().start()), }; - // fetch span information of token in real file, then use that look through expansions of - // calls the token is in and afterwards recursively with the same span. - // what about things where spans change? Due to being joined etc, that is we don't find the - // exact span anymore? - let def_map = sa.resolver.def_map(); let mut stack: SmallVec<[_; 4]> = smallvec![InFile::new(sa.file_id, token)]; @@ -580,7 +575,7 @@ impl<'db> SemanticsImpl<'db> { let len = stack.len(); // requeue the tokens we got from mapping our current token down - stack.extend(mapped_tokens); + stack.extend(mapped_tokens.map(Into::into)); // if the length changed we have found a mapping for the token (stack.len() != len).then_some(()) }; diff --git a/crates/ide/src/expand_macro.rs b/crates/ide/src/expand_macro.rs index cca3e68d5024..a70f335ada3c 100644 --- a/crates/ide/src/expand_macro.rs +++ b/crates/ide/src/expand_macro.rs @@ -340,8 +340,8 @@ fn main() { expect![[r#" match_ast! { - if let Some(it) = ast::TraitDef::cast((container).clone()){} - else if let Some(it) = ast::ImplDef::cast((container).clone()){} + if let Some(it) = ast::TraitDef::cast(container.clone()){} + else if let Some(it) = ast::ImplDef::cast(container.clone()){} else { { continue diff --git a/crates/test-utils/src/minicore.rs b/crates/test-utils/src/minicore.rs index f2ca9d82ed0d..8ebc9909b6b2 100644 --- a/crates/test-utils/src/minicore.rs +++ b/crates/test-utils/src/minicore.rs @@ -15,6 +15,7 @@ //! cell: copy, drop //! clone: sized //! coerce_unsized: unsize +//! concat: //! copy: clone //! default: sized //! deref_mut: deref @@ -1353,7 +1354,7 @@ mod panicking { mod macros { // region:panic #[macro_export] - #[rustc_builtin_macro(std_panic)] + #[rustc_builtin_macro(core_panic)] macro_rules! panic { ($($arg:tt)*) => { /* compiler built-in */ @@ -1406,6 +1407,12 @@ mod macros { ($file:expr $(,)?) => {{ /* compiler built-in */ }}; } // endregion:include + + // region:concat + #[rustc_builtin_macro] + #[macro_export] + macro_rules! concat {} + // endregion:concat } // region:non_zero From 81410ab50065db9c657cfbb3ef6333119a209d5d Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 2 Dec 2023 19:32:53 +0100 Subject: [PATCH 23/24] Cleanup FileId stuff --- crates/hir-expand/src/files.rs | 79 ++++++++++--------- crates/hir-expand/src/lib.rs | 13 +-- .../src/handlers/remove_unused_imports.rs | 4 +- crates/ide-db/src/search.rs | 7 +- crates/ide/src/navigation_target.rs | 9 ++- 5 files changed, 64 insertions(+), 48 deletions(-) diff --git a/crates/hir-expand/src/files.rs b/crates/hir-expand/src/files.rs index 36eb2dbb27c4..ed8639d7a1d9 100644 --- a/crates/hir-expand/src/files.rs +++ b/crates/hir-expand/src/files.rs @@ -83,26 +83,41 @@ impl InFileWrapper> { // endregion:transpose impls -// region:specific impls +trait FileIdToSyntax: Copy { + fn file_syntax(self, db: &dyn db::ExpandDatabase) -> SyntaxNode; +} -impl InFile { - pub fn file_syntax(&self, db: &dyn db::ExpandDatabase) -> SyntaxNode { - db.parse_or_expand(self.file_id) +impl FileIdToSyntax for FileId { + fn file_syntax(self, db: &dyn db::ExpandDatabase) -> SyntaxNode { + db.parse(self).syntax_node() + } +} +impl FileIdToSyntax for MacroFileId { + fn file_syntax(self, db: &dyn db::ExpandDatabase) -> SyntaxNode { + db.parse_macro_expansion(self).value.0.syntax_node() + } +} +impl FileIdToSyntax for HirFileId { + fn file_syntax(self, db: &dyn db::ExpandDatabase) -> SyntaxNode { + db.parse_or_expand(self) } } -impl InRealFile { +#[allow(private_bounds)] +impl InFileWrapper { pub fn file_syntax(&self, db: &dyn db::ExpandDatabase) -> SyntaxNode { - db.parse(self.file_id).syntax_node() + FileIdToSyntax::file_syntax(self.file_id, db) } } -impl InMacroFile { - pub fn file_syntax(&self, db: &dyn db::ExpandDatabase) -> SyntaxNode { - db.parse_macro_expansion(self.file_id).value.0.syntax_node() +impl InFileWrapper { + pub fn syntax(&self) -> InFileWrapper { + self.with_value(self.value.syntax()) } } +// region:specific impls + impl InFile<&SyntaxNode> { pub fn ancestors_with_macros( self, @@ -241,9 +256,15 @@ impl InFile { match self.file_id.repr() { HirFileIdRepr::FileId(file_id) => FileRange { file_id, range: self.value.text_range() }, HirFileIdRepr::MacroFile(mac_file) => { - if let Some(res) = self.original_file_range_opt(db) { - return res; + let (range, ctxt) = ExpansionInfo::new(db, mac_file) + .map_token_range_up(db, self.value.text_range()); + + // FIXME: Figure out an API that makes proper use of ctx, this only exists to + // keep pre-token map rewrite behaviour. + if ctxt.is_root() { + return range; } + // Fall back to whole macro call. let loc = db.lookup_intern_macro_call(mac_file.macro_call_id); loc.kind.original_call_range(db) @@ -257,8 +278,9 @@ impl InFile { HirFileIdRepr::FileId(file_id) => { Some(FileRange { file_id, range: self.value.text_range() }) } - HirFileIdRepr::MacroFile(_) => { - let (range, ctxt) = ascend_range_up_macros(db, self.map(|it| it.text_range())); + HirFileIdRepr::MacroFile(mac_file) => { + let (range, ctxt) = ExpansionInfo::new(db, mac_file) + .map_token_range_up(db, self.value.text_range()); // FIXME: Figure out an API that makes proper use of ctx, this only exists to // keep pre-token map rewrite behaviour. @@ -275,16 +297,19 @@ impl InFile { impl InFile { /// Attempts to map the syntax node back up its macro calls. pub fn original_file_range(self, db: &dyn db::ExpandDatabase) -> FileRange { - let (range, _ctxt) = ascend_range_up_macros(db, self); + let (range, _ctxt) = match self.file_id.repr() { + HirFileIdRepr::FileId(file_id) => { + (FileRange { file_id, range: self.value }, SyntaxContextId::ROOT) + } + HirFileIdRepr::MacroFile(m) => { + ExpansionInfo::new(db, m).map_token_range_up(db, self.value) + } + }; range } } impl InFile { - pub fn descendants(self) -> impl Iterator> { - self.value.syntax().descendants().filter_map(T::cast).map(move |n| self.with_value(n)) - } - pub fn original_ast_node(self, db: &dyn db::ExpandDatabase) -> Option> { // This kind of upmapping can only be achieved in attribute expanded files, // as we don't have node inputs otherwise and therefore can't find an `N` node in the input @@ -312,22 +337,4 @@ impl InFile { let value = anc.ancestors().find_map(N::cast)?; Some(InRealFile::new(file_id, value)) } - - pub fn syntax(&self) -> InFile<&SyntaxNode> { - self.with_value(self.value.syntax()) - } -} - -fn ascend_range_up_macros( - db: &dyn db::ExpandDatabase, - range: InFile, -) -> (FileRange, SyntaxContextId) { - match range.file_id.repr() { - HirFileIdRepr::FileId(file_id) => { - (FileRange { file_id, range: range.value }, SyntaxContextId::ROOT) - } - HirFileIdRepr::MacroFile(m) => { - ExpansionInfo::new(db, m).map_token_range_up(db, range.value) - } - } } diff --git a/crates/hir-expand/src/lib.rs b/crates/hir-expand/src/lib.rs index f328431372ec..1d5b7dfa5943 100644 --- a/crates/hir-expand/src/lib.rs +++ b/crates/hir-expand/src/lib.rs @@ -172,17 +172,18 @@ pub enum MacroCallKind { } pub trait HirFileIdExt { - /// For macro-expansion files, returns the file original source file the - /// expansion originated from. + /// Returns the original file of this macro call hierarchy. fn original_file(self, db: &dyn db::ExpandDatabase) -> FileId; + /// Returns the original file of this macro call hierarchy while going into the included file if + /// one of the calls comes from an `include!``. fn original_file_respecting_includes(self, db: &dyn db::ExpandDatabase) -> FileId; /// If this is a macro call, returns the syntax node of the call. fn call_node(self, db: &dyn db::ExpandDatabase) -> Option>; /// If this is a macro call, returns the syntax node of the very first macro call this file resides in. - fn original_call_node(self, db: &dyn db::ExpandDatabase) -> Option<(FileId, SyntaxNode)>; + fn original_call_node(self, db: &dyn db::ExpandDatabase) -> Option>; /// Return expansion information if it is a macro-expansion file fn expansion_info(self, db: &dyn db::ExpandDatabase) -> Option; @@ -246,11 +247,13 @@ impl HirFileIdExt for HirFileId { Some(loc.to_node(db)) } - fn original_call_node(self, db: &dyn db::ExpandDatabase) -> Option<(FileId, SyntaxNode)> { + fn original_call_node(self, db: &dyn db::ExpandDatabase) -> Option> { let mut call = db.lookup_intern_macro_call(self.macro_file()?.macro_call_id).to_node(db); loop { match call.file_id.repr() { - HirFileIdRepr::FileId(file_id) => break Some((file_id, call.value)), + HirFileIdRepr::FileId(file_id) => { + break Some(InRealFile { file_id, value: call.value }) + } HirFileIdRepr::MacroFile(MacroFileId { macro_call_id }) => { call = db.lookup_intern_macro_call(macro_call_id).to_node(db); } diff --git a/crates/ide-assists/src/handlers/remove_unused_imports.rs b/crates/ide-assists/src/handlers/remove_unused_imports.rs index 10076e60c3e8..ee44064e7c5e 100644 --- a/crates/ide-assists/src/handlers/remove_unused_imports.rs +++ b/crates/ide-assists/src/handlers/remove_unused_imports.rs @@ -1,6 +1,6 @@ use std::collections::{hash_map::Entry, HashMap}; -use hir::{HirFileIdExt, InFile, Module, ModuleSource}; +use hir::{HirFileIdExt, InFile, InRealFile, Module, ModuleSource}; use ide_db::{ base_db::FileRange, defs::Definition, @@ -167,7 +167,7 @@ fn used_once_in_scope(ctx: &AssistContext<'_>, def: Definition, scopes: &Vec Vec { let (file_id, range) = { let InFile { file_id, value } = module.definition_source(db); - if let Some((file_id, call_source)) = file_id.original_call_node(db) { + if let Some(InRealFile { file_id, value: call_source }) = file_id.original_call_node(db) { (file_id, Some(call_source.text_range())) } else { ( diff --git a/crates/ide-db/src/search.rs b/crates/ide-db/src/search.rs index 68f2ad494570..2ce036c044ad 100644 --- a/crates/ide-db/src/search.rs +++ b/crates/ide-db/src/search.rs @@ -8,8 +8,8 @@ use std::mem; use base_db::{salsa::Database, FileId, FileRange, SourceDatabase, SourceDatabaseExt}; use hir::{ - AsAssocItem, DefWithBody, HasAttrs, HasSource, HirFileIdExt, InFile, ModuleSource, Semantics, - Visibility, + AsAssocItem, DefWithBody, HasAttrs, HasSource, HirFileIdExt, InFile, InRealFile, ModuleSource, + Semantics, Visibility, }; use memchr::memmem::Finder; use nohash_hasher::IntMap; @@ -133,7 +133,8 @@ impl SearchScope { let (file_id, range) = { let InFile { file_id, value } = module.definition_source(db); - if let Some((file_id, call_source)) = file_id.original_call_node(db) { + if let Some(InRealFile { file_id, value: call_source }) = file_id.original_call_node(db) + { (file_id, Some(call_source.text_range())) } else { ( diff --git a/crates/ide/src/navigation_target.rs b/crates/ide/src/navigation_target.rs index 1e4d0e8cdc6e..df0c4a6adee2 100644 --- a/crates/ide/src/navigation_target.rs +++ b/crates/ide/src/navigation_target.rs @@ -169,8 +169,13 @@ impl TryToNav for FileSymbol { fn try_to_nav(&self, db: &RootDatabase) -> Option { let full_range = self.loc.original_range(db); let focus_range = self.loc.original_name_range(db); - let focus_range = - if focus_range.file_id == full_range.file_id { Some(focus_range.range) } else { None }; + let focus_range = if focus_range.file_id == full_range.file_id + && full_range.range.contains_range(focus_range.range) + { + Some(focus_range.range) + } else { + None + }; Some(NavigationTarget { file_id: full_range.file_id, From 18f1a3c3c6ffc6179119e2503854807abbf9cd32 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sun, 3 Dec 2023 18:50:29 +0100 Subject: [PATCH 24/24] Some final touches --- crates/hir-expand/src/lib.rs | 2 +- crates/hir-expand/src/quote.rs | 25 +++++++++++++--------- crates/hir-expand/src/span.rs | 3 +-- crates/mbe/src/benchmark.rs | 29 +++++++------------------- crates/mbe/src/expander/transcriber.rs | 17 +++++++++++---- crates/mbe/src/lib.rs | 2 +- crates/mbe/src/syntax_bridge.rs | 29 ++++++++++++++++---------- crates/mbe/src/token_map.rs | 24 ++++++++++++--------- crates/tt/src/lib.rs | 20 +++++++++++++----- crates/vfs/src/lib.rs | 2 +- 10 files changed, 86 insertions(+), 67 deletions(-) diff --git a/crates/hir-expand/src/lib.rs b/crates/hir-expand/src/lib.rs index 1d5b7dfa5943..bc7b5fb211d1 100644 --- a/crates/hir-expand/src/lib.rs +++ b/crates/hir-expand/src/lib.rs @@ -663,7 +663,7 @@ impl ExpansionInfo { range: TextRange, ) -> Option<(FileRange, SyntaxContextId)> { debug_assert!(self.expanded.value.text_range().contains_range(range)); - let mut spans = self.exp_map.spans_for_node_range(range); + let mut spans = self.exp_map.spans_for_range(range); let SpanData { range, anchor, ctx } = spans.next()?; let mut start = range.start(); let mut end = range.end(); diff --git a/crates/hir-expand/src/quote.rs b/crates/hir-expand/src/quote.rs index 069bcc3bd8e2..0950f5d28755 100644 --- a/crates/hir-expand/src/quote.rs +++ b/crates/hir-expand/src/quote.rs @@ -215,10 +215,18 @@ impl_to_to_tokentrees! { #[cfg(test)] mod tests { use crate::tt; - use ::tt::Span; + use base_db::{ + span::{SpanAnchor, SyntaxContextId, ROOT_ERASED_FILE_AST_ID}, + FileId, + }; use expect_test::expect; + use syntax::{TextRange, TextSize}; - const DUMMY: tt::SpanData = tt::SpanData::DUMMY; + const DUMMY: tt::SpanData = tt::SpanData { + range: TextRange::empty(TextSize::new(0)), + anchor: SpanAnchor { file_id: FileId::BOGUS, ast_id: ROOT_ERASED_FILE_AST_ID }, + ctx: SyntaxContextId::ROOT, + }; #[test] fn test_quote_delimiters() { @@ -242,10 +250,7 @@ mod tests { } fn mk_ident(name: &str) -> crate::tt::Ident { - crate::tt::Ident { - text: name.into(), - span: ::DUMMY, - } + crate::tt::Ident { text: name.into(), span: DUMMY } } #[test] @@ -256,8 +261,8 @@ mod tests { assert_eq!(quoted.to_string(), "hello"); let t = format!("{quoted:?}"); expect![[r#" - SUBTREE $$ SpanData { range: 0..0, anchor: SpanAnchor(FileId(0), 0), ctx: SyntaxContextId(0) } SpanData { range: 0..0, anchor: SpanAnchor(FileId(0), 0), ctx: SyntaxContextId(0) } - IDENT hello SpanData { range: 0..0, anchor: SpanAnchor(FileId(0), 0), ctx: SyntaxContextId(0) }"#]].assert_eq(&t); + SUBTREE $$ SpanData { range: 0..0, anchor: SpanAnchor(FileId(4294967295), 0), ctx: SyntaxContextId(0) } SpanData { range: 0..0, anchor: SpanAnchor(FileId(4294967295), 0), ctx: SyntaxContextId(0) } + IDENT hello SpanData { range: 0..0, anchor: SpanAnchor(FileId(4294967295), 0), ctx: SyntaxContextId(0) }"#]].assert_eq(&t); } #[test] @@ -290,8 +295,8 @@ mod tests { let list = crate::tt::Subtree { delimiter: crate::tt::Delimiter { kind: crate::tt::DelimiterKind::Brace, - open: ::DUMMY, - close: ::DUMMY, + open: DUMMY, + close: DUMMY, }, token_trees: fields.collect(), }; diff --git a/crates/hir-expand/src/span.rs b/crates/hir-expand/src/span.rs index 1703923d11a1..0a6c22fe42dc 100644 --- a/crates/hir-expand/src/span.rs +++ b/crates/hir-expand/src/span.rs @@ -4,13 +4,12 @@ use base_db::{ span::{ErasedFileAstId, SpanAnchor, SpanData, SyntaxContextId, ROOT_ERASED_FILE_AST_ID}, FileId, }; -use mbe::TokenMap; use syntax::{ast::HasModuleItem, AstNode, TextRange, TextSize}; use triomphe::Arc; use crate::db::ExpandDatabase; -pub type ExpansionSpanMap = TokenMap; +pub type ExpansionSpanMap = mbe::SpanMap; /// Spanmap for a macro file or a real file #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/crates/mbe/src/benchmark.rs b/crates/mbe/src/benchmark.rs index 271efe1a92e9..f503aecce2c2 100644 --- a/crates/mbe/src/benchmark.rs +++ b/crates/mbe/src/benchmark.rs @@ -6,11 +6,10 @@ use syntax::{ AstNode, SmolStr, }; use test_utils::{bench, bench_fixture, skip_slow_tests}; -use tt::Span; use crate::{ parser::{MetaVarKind, Op, RepeatKind, Separator}, - syntax_node_to_token_tree, DeclarativeMacro, DummyTestSpanData, DummyTestSpanMap, + syntax_node_to_token_tree, DeclarativeMacro, DummyTestSpanData, DummyTestSpanMap, DUMMY, }; #[test] @@ -97,8 +96,8 @@ fn invocation_fixtures( loop { let mut subtree = tt::Subtree { delimiter: tt::Delimiter { - open: DummyTestSpanData::DUMMY, - close: DummyTestSpanData::DUMMY, + open: DUMMY, + close: DUMMY, kind: tt::DelimiterKind::Invisible, }, token_trees: vec![], @@ -211,34 +210,20 @@ fn invocation_fixtures( *seed } fn make_ident(ident: &str) -> tt::TokenTree { - tt::Leaf::Ident(tt::Ident { span: DummyTestSpanData::DUMMY, text: SmolStr::new(ident) }) - .into() + tt::Leaf::Ident(tt::Ident { span: DUMMY, text: SmolStr::new(ident) }).into() } fn make_punct(char: char) -> tt::TokenTree { - tt::Leaf::Punct(tt::Punct { - span: DummyTestSpanData::DUMMY, - char, - spacing: tt::Spacing::Alone, - }) - .into() + tt::Leaf::Punct(tt::Punct { span: DUMMY, char, spacing: tt::Spacing::Alone }).into() } fn make_literal(lit: &str) -> tt::TokenTree { - tt::Leaf::Literal(tt::Literal { - span: DummyTestSpanData::DUMMY, - text: SmolStr::new(lit), - }) - .into() + tt::Leaf::Literal(tt::Literal { span: DUMMY, text: SmolStr::new(lit) }).into() } fn make_subtree( kind: tt::DelimiterKind, token_trees: Option>>, ) -> tt::TokenTree { tt::Subtree { - delimiter: tt::Delimiter { - open: DummyTestSpanData::DUMMY, - close: DummyTestSpanData::DUMMY, - kind, - }, + delimiter: tt::Delimiter { open: DUMMY, close: DUMMY, kind }, token_trees: token_trees.unwrap_or_default(), } .into() diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs index e4a46c16597f..7a3e8653c28f 100644 --- a/crates/mbe/src/expander/transcriber.rs +++ b/crates/mbe/src/expander/transcriber.rs @@ -79,8 +79,8 @@ impl Bindings { } MetaVarKind::Block => Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { delimiter: tt::Delimiter { - open: S::DUMMY, - close: S::DUMMY, + open: span, + close: span, kind: tt::DelimiterKind::Brace, }, token_trees: vec![], @@ -225,6 +225,7 @@ fn expand_subtree( arena.push( tt::Leaf::Literal(tt::Literal { text: index.to_string().into(), + // FIXME span: S::DUMMY, }) .into(), @@ -282,8 +283,12 @@ fn expand_subtree( } }; arena.push( - tt::Leaf::Literal(tt::Literal { text: c.to_string().into(), span: S::DUMMY }) - .into(), + tt::Leaf::Literal(tt::Literal { + text: c.to_string().into(), + // FIXME + span: S::DUMMY, + }) + .into(), ); } } @@ -337,7 +342,9 @@ fn expand_var( } Err(e) => ExpandResult { value: Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree::empty(tt::DelimSpan { + // FIXME open: S::DUMMY, + // FIXME close: S::DUMMY, }))), err: Some(e), @@ -479,6 +486,7 @@ fn fix_up_and_push_path_tt(buf: &mut Vec>, subtree: tt tt::Leaf::Punct(tt::Punct { char: ':', spacing: tt::Spacing::Joint, + // FIXME span: S::DUMMY, }) .into(), @@ -487,6 +495,7 @@ fn fix_up_and_push_path_tt(buf: &mut Vec>, subtree: tt tt::Leaf::Punct(tt::Punct { char: ':', spacing: tt::Spacing::Alone, + // FIXME span: S::DUMMY, }) .into(), diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index 10eb59bb8352..2b52e04b251e 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -38,7 +38,7 @@ pub use crate::{ syntax_node_to_token_tree, syntax_node_to_token_tree_modified, token_tree_to_syntax_node, SpanMapper, }, - token_map::TokenMap, + token_map::SpanMap, }; pub use crate::syntax_bridge::dummy_test_span_utils::*; diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 5722a5bd8eb0..1c46471a3832 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -13,7 +13,7 @@ use tt::{ Span, SpanData, SyntaxContext, }; -use crate::{to_parser_input::to_parser_input, tt_iter::TtIter, TokenMap}; +use crate::{to_parser_input::to_parser_input, tt_iter::TtIter, SpanMap}; #[cfg(test)] mod tests; @@ -22,7 +22,7 @@ pub trait SpanMapper { fn span_for(&self, range: TextRange) -> S; } -impl SpanMapper for TokenMap { +impl SpanMapper for SpanMap { fn span_for(&self, range: TextRange) -> S { self.span_at(range.start()) } @@ -34,10 +34,12 @@ impl> SpanMapper for &SM { } } +/// Dummy things for testing where spans don't matter. pub(crate) mod dummy_test_span_utils { use super::*; pub type DummyTestSpanData = tt::SpanData; + pub const DUMMY: DummyTestSpanData = DummyTestSpanData::DUMMY; #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] pub struct DummyTestSpanAnchor; @@ -62,9 +64,8 @@ pub(crate) mod dummy_test_span_utils { } } -/// Convert the syntax node to a `TokenTree` (what macro -/// will consume). -/// FIXME: Flesh out the doc comment more thoroughly +/// Converts a syntax tree to a [`tt::Subtree`] using the provided span map to populate the +/// subtree's spans. pub fn syntax_node_to_token_tree( node: &SyntaxNode, map: SpanMap, @@ -79,6 +80,9 @@ where convert_tokens(&mut c) } +/// Converts a syntax tree to a [`tt::Subtree`] using the provided span map to populate the +/// subtree's spans. Additionally using the append and remove parameters, the additional tokens can +/// be injected or hidden from the output. pub fn syntax_node_to_token_tree_modified( node: &SyntaxNode, map: SpanMap, @@ -107,10 +111,12 @@ where // * AssocItems(SmallVec<[ast::AssocItem; 1]>) // * ForeignItems(SmallVec<[ast::ForeignItem; 1]> +/// Converts a [`tt::Subtree`] back to a [`SyntaxNode`]. +/// The produced `SpanMap` contains a mapping from the syntax nodes offsets to the subtree's spans. pub fn token_tree_to_syntax_node( tt: &tt::Subtree>, entry_point: parser::TopEntryPoint, -) -> (Parse, TokenMap>) +) -> (Parse, SpanMap>) where SpanData: Span, Anchor: Copy, @@ -142,7 +148,8 @@ where tree_sink.finish() } -/// Convert a string to a `TokenTree` +/// Convert a string to a `TokenTree`. The spans of the subtree will be anchored to the provided +/// anchor with the given context. pub fn parse_to_token_tree( anchor: Anchor, ctx: Ctx, @@ -161,7 +168,7 @@ where Some(convert_tokens(&mut conv)) } -/// Convert a string to a `TokenTree` +/// Convert a string to a `TokenTree`. The passed span will be used for all spans of the produced subtree. pub fn parse_to_token_tree_static_span(span: S, text: &str) -> Option> where S: Span, @@ -798,7 +805,7 @@ where cursor: Cursor<'a, SpanData>, text_pos: TextSize, inner: SyntaxTreeBuilder, - token_map: TokenMap>, + token_map: SpanMap>, } impl<'a, Anchor, Ctx> TtTreeSink<'a, Anchor, Ctx> @@ -811,11 +818,11 @@ where cursor, text_pos: 0.into(), inner: SyntaxTreeBuilder::default(), - token_map: TokenMap::empty(), + token_map: SpanMap::empty(), } } - fn finish(mut self) -> (Parse, TokenMap>) { + fn finish(mut self) -> (Parse, SpanMap>) { self.token_map.finish(); (self.inner.finish(), self.token_map) } diff --git a/crates/mbe/src/token_map.rs b/crates/mbe/src/token_map.rs index 5871c0f1251c..b51d7575a113 100644 --- a/crates/mbe/src/token_map.rs +++ b/crates/mbe/src/token_map.rs @@ -8,30 +8,33 @@ use tt::Span; /// Maps absolute text ranges for the corresponding file to the relevant span data. #[derive(Debug, PartialEq, Eq, Clone, Hash)] -// FIXME: Rename to SpanMap -pub struct TokenMap { - // FIXME: This needs to be sorted by (FileId, AstId) - // Then we can do a binary search on the file id, - // then a bin search on the ast id? +pub struct SpanMap { spans: Vec<(TextSize, S)>, } -impl TokenMap { +impl SpanMap { + /// Creates a new empty [`SpanMap`]. pub fn empty() -> Self { Self { spans: Vec::new() } } + /// Finalizes the [`SpanMap`], shrinking its backing storage and validating that the offsets are + /// in order. pub fn finish(&mut self) { assert!(self.spans.iter().tuple_windows().all(|(a, b)| a.0 < b.0)); self.spans.shrink_to_fit(); } + /// Pushes a new span onto the [`SpanMap`]. pub fn push(&mut self, offset: TextSize, span: S) { + debug_assert!(self.spans.last().map_or(true, |&(last_offset, _)| last_offset < offset)); self.spans.push((offset, span)); } + /// Returns all [`TextRange`]s that correspond to the given span. + /// + /// Note this does a linear search through the entire backing vector. pub fn ranges_with_span(&self, span: S) -> impl Iterator + '_ { - // FIXME: linear search self.spans.iter().enumerate().filter_map(move |(idx, &(end, s))| { if s != span { return None; @@ -41,14 +44,15 @@ impl TokenMap { }) } - // FIXME: We need APIs for fetching the span of a token as well as for a whole node. The node - // one *is* fallible though. + /// Returns the span at the given position. pub fn span_at(&self, offset: TextSize) -> S { let entry = self.spans.partition_point(|&(it, _)| it <= offset); self.spans[entry].1 } - pub fn spans_for_node_range(&self, range: TextRange) -> impl Iterator + '_ { + /// Returns the spans associated with the given range. + /// In other words, this will return all spans that correspond to all offsets within the given range. + pub fn spans_for_range(&self, range: TextRange) -> impl Iterator + '_ { let (start, end) = (range.start(), range.end()); let start_entry = self.spans.partition_point(|&(it, _)| it <= start); let end_entry = self.spans[start_entry..].partition_point(|&(it, _)| it <= end); // FIXME: this might be wrong? diff --git a/crates/tt/src/lib.rs b/crates/tt/src/lib.rs index b1f218516210..10e05862189f 100644 --- a/crates/tt/src/lib.rs +++ b/crates/tt/src/lib.rs @@ -23,6 +23,7 @@ pub struct SpanData { } impl Span for SpanData { + #[allow(deprecated)] const DUMMY: Self = SpanData { range: TextRange::empty(TextSize::new(0)), anchor: Anchor::DUMMY, @@ -30,18 +31,24 @@ impl Span for SpanData { }; } -pub trait SpanAnchor: - std::fmt::Debug + Copy + Sized + Eq + Copy + fmt::Debug + std::hash::Hash -{ +pub trait Span: std::fmt::Debug + Copy + Sized + Eq { + // FIXME: Should not exist. Dummy spans will always be wrong if they leak somewhere. Instead, + // the call site or def site spans should be used in relevant places, its just that we don't + // expose those everywhere in the yet. const DUMMY: Self; } -// FIXME: Get rid of this trait? -pub trait Span: std::fmt::Debug + Copy + Sized + Eq { +// FIXME: Should not exist +pub trait SpanAnchor: + std::fmt::Debug + Copy + Sized + Eq + Copy + fmt::Debug + std::hash::Hash +{ + #[deprecated(note = "this should not exist")] const DUMMY: Self; } +// FIXME: Should not exist pub trait SyntaxContext: std::fmt::Debug + Copy + Sized + Eq { + #[deprecated(note = "this should not exist")] const DUMMY: Self; } @@ -128,6 +135,7 @@ pub struct DelimSpan { } impl DelimSpan { + // FIXME should not exist pub const DUMMY: Self = Self { open: S::DUMMY, close: S::DUMMY }; } @@ -139,9 +147,11 @@ pub struct Delimiter { } impl Delimiter { + // FIXME should not exist pub const DUMMY_INVISIBLE: Self = Self { open: S::DUMMY, close: S::DUMMY, kind: DelimiterKind::Invisible }; + // FIXME should not exist pub const fn dummy_invisible() -> Self { Self::DUMMY_INVISIBLE } diff --git a/crates/vfs/src/lib.rs b/crates/vfs/src/lib.rs index c6866bbfb9b4..128559727bdc 100644 --- a/crates/vfs/src/lib.rs +++ b/crates/vfs/src/lib.rs @@ -63,7 +63,7 @@ pub use paths::{AbsPath, AbsPathBuf}; pub struct FileId(pub u32); impl FileId { - /// Think twice about using this. If this ends up in a wrong place it will cause panics! + /// Think twice about using this outside of tests. If this ends up in a wrong place it will cause panics! pub const BOGUS: FileId = FileId(u32::MAX); }