employer
/
tame
1
0
Fork 0
Code Packages Releases Activity

tamer: nir::desugar::interp: Token {SPair=>Attr} 

This changes the input token from a more generic `SPair` to `Attr`, which
reflects the new target integration point in the `attr_parse!`
parser-generator.

This is a compromise---I'd like for it to remain generic and have stitching
deal with all integration concerns, but I have spent far too much time on
this and need to keep moving.

With that said, we do benefit from knowing where this must fit in---it's
easier to reason about in a more concrete way, and we can take advantage of
the extra information rather than being burdened by its presence and
ignoring it.  We need to be able to convert back into `XirfToken` (see a
recent commit that discusses that) for `StitchExpansion`, which is why
`Attr` is here.  And since it is, we can use it to explain to the user not
just the interpolation specification used to derive params, but also the
attribute it is associated with.  This is what TAME (in XSLT) does today,
IIRC (I wrote it, I just forget exactly).  It also means that I can name the
parameters after the attribute.

So, that'll be in a following commit; I was disappointed when my prior
approach with `SPair` didn't give me enough information to be able to do
that, since I think it's important that the system be as descriptive as
possible in how it derives information.  Of course, traces would reveal how
the parser came about the derivation, but that requires recompilation in a
special tracing mode.

DEV-13156
main
Mike Gerwitz 2022-11-18 13:09:47 -05:00
parent 99dcba690f
commit d0a728c27f
3 changed files with 78 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
tamer/src/nir/desugar/interp.rs
View File

@ -103,16 +103,13 @@ use super::super::{PlainNir, PlainNirSymbol};
use crate::{ use crate::{
diagnose::{AnnotatedSpan, Diagnostic}, diagnose::{AnnotatedSpan, Diagnostic},
fmt::{DisplayWrapper, TtQuote}, fmt::{DisplayWrapper, TtQuote},
parse::{ parse::{prelude::*, util::expand::Expansion, NoContext},
prelude::*,
util::{expand::Expansion, SPair},
NoContext,
},
span::Span, span::Span,
sym::{ sym::{
st::quick_contains_byte, GlobalSymbolIntern, GlobalSymbolResolve, st::quick_contains_byte, GlobalSymbolIntern, GlobalSymbolResolve,
SymbolId, SymbolId,
}, },
xir::attr::{Attr, AttrSpan},
}; };
use std::{error::Error, fmt::Display}; use std::{error::Error, fmt::Display};
@ -208,8 +205,8 @@ impl Display for InterpState {
} }
impl ParseState for InterpState { impl ParseState for InterpState {
type Token = SPair; type Token = Attr;
type Object = Expansion<SPair, PlainNir>; type Object = Expansion<Self::Token, PlainNir>;
type Error = InterpError; type Error = InterpError;
fn parse_token( fn parse_token(
@ -217,7 +214,11 @@ impl ParseState for InterpState {
tok: Self::Token, tok: Self::Token,
_: NoContext, _: NoContext,
) -> TransitionResult<Self> { ) -> TransitionResult<Self> {
match (self, tok.into()) { let spair = match tok {
Attr(_, val, AttrSpan(_, vspan)) => (val, vspan),
};
match (self, spair) {
// When receiving a new symbol, // When receiving a new symbol,
// we must make a quick determination as to whether it // we must make a quick determination as to whether it
// requires desugaring. // requires desugaring.
@ -229,22 +230,23 @@ impl ParseState for InterpState {
// Symbols that require no interpoolation are simply echoed back. // Symbols that require no interpoolation are simply echoed back.
(Ready, (sym, span)) => { (Ready, (sym, span)) => {
if needs_interpolation(sym) { if needs_interpolation(sym) {
Self::begin_expansion(sym, span) Self::begin_expansion(sym, span).with_lookahead(tok)
} else { } else {
// No desugaring is needed. // No desugaring is needed.
Self::yield_symbol(sym, span) Self::yield_tok(tok)
} }
} }
// The outermost parsing context is that of the literal, // The outermost parsing context is that of the literal,
// where a sequence of characters up to `{` stand for // where a sequence of characters up to `{` stand for
// themselves. // themselves.
(ParseLiteralAt(s, gen_param, offset), (sym, span)) => { (ParseLiteralAt(s, gen_param, offset), (_, span)) => {
if offset == s.len() { if offset == s.len() {
// We've reached the end of the specification string. // We've reached the end of the specification string.
// Since we're in the outermost (literal) context, // Since we're in the outermost (literal) context,
// we're safe to complete. // we're safe to complete.
return Self::end_expansion(s, gen_param, sym, span); return Self::end_expansion(s, gen_param, span)
.with_lookahead(tok);
} }
// Note that this is the position _relative to the offset_, // Note that this is the position _relative to the offset_,
@ -256,7 +258,7 @@ impl ParseState for InterpState {
Some(0) => { Some(0) => {
Transition(ParseInterpAt(s, gen_param, offset + 1)) Transition(ParseInterpAt(s, gen_param, offset + 1))
.incomplete() .incomplete()
.with_lookahead((sym, span).into()) .with_lookahead(tok)
} }
// Everything from the offset until the curly brace is a // Everything from the offset until the curly brace is a
@ -273,7 +275,7 @@ impl ParseState for InterpState {
Transition(ParseInterpAt(s, gen_param, end + 1)) Transition(ParseInterpAt(s, gen_param, end + 1))
.ok(Expanded(text)) .ok(Expanded(text))
.with_lookahead((sym, span).into()) .with_lookahead(tok)
} }
// The remainder of the specification is a literal. // The remainder of the specification is a literal.
@ -289,7 +291,7 @@ impl ParseState for InterpState {
// we'll complete parsing next pass. // we'll complete parsing next pass.
Transition(ParseLiteralAt(s, gen_param, s.len())) Transition(ParseLiteralAt(s, gen_param, s.len()))
.ok(Expanded(text)) .ok(Expanded(text))
.with_lookahead((sym, span).into()) .with_lookahead(tok)
} }
} }
} }
@ -299,7 +301,7 @@ impl ParseState for InterpState {
// This is an inner context that cannot complete without being // This is an inner context that cannot complete without being
// explicitly closed, // explicitly closed,
// and cannot not be nested. // and cannot not be nested.
(ParseInterpAt(s, gen_param, offset), (sym, span)) => { (ParseInterpAt(s, gen_param, offset), (_, span)) => {
// TODO: Make sure offset exists, avoid panic // TODO: Make sure offset exists, avoid panic
// TODO: Prevent nested `{`. // TODO: Prevent nested `{`.
@ -326,7 +328,7 @@ impl ParseState for InterpState {
// back in a literal context. // back in a literal context.
Transition(ParseLiteralAt(s, gen_param, end + 1)) Transition(ParseLiteralAt(s, gen_param, end + 1))
.ok(Expanded(param_value)) .ok(Expanded(param_value))
.with_lookahead((sym, span).into()) .with_lookahead(tok)
} }
None => todo!("missing closing '}}'"), None => todo!("missing closing '}}'"),
@ -338,8 +340,8 @@ impl ParseState for InterpState {
// (the interpolation specification) // (the interpolation specification)
// with a metavariable referencing the parameter that we just // with a metavariable referencing the parameter that we just
// generated. // generated.
(FinishSym(_, GenIdentSymbolId(gen_param)), (_, span)) => { (FinishSym(_, GenIdentSymbolId(gen_param)), _) => {
Self::yield_symbol(gen_param, span) Self::yield_tok(tok.replace_value_derived(gen_param))
} }
} }
} }
@ -362,8 +364,8 @@ impl InterpState {
/// ///
/// This transitions back to [`Ready`] and finally releases the /// This transitions back to [`Ready`] and finally releases the
/// lookahead symbol. /// lookahead symbol.
fn yield_symbol(sym: SymbolId, span: Span) -> TransitionResult<Self> { fn yield_tok(tok: Attr) -> TransitionResult<Self> {
Transition(Ready).ok(DoneExpanding((sym, span).into())) Transition(Ready).ok(DoneExpanding(tok))
} }
/// Begin expansion of an interpolation specification by generating a /// Begin expansion of an interpolation specification by generating a
@ -397,7 +399,6 @@ impl InterpState {
// prefixes. // prefixes.
Transition(ParseLiteralAt(sym.lookup_str(), gen_param, 0)) Transition(ParseLiteralAt(sym.lookup_str(), gen_param, 0))
.ok(Expanded(open)) .ok(Expanded(open))
.with_lookahead((sym, span).into())
} }
/// Complete expansion of an interpolation specification string. /// Complete expansion of an interpolation specification string.
@ -407,7 +408,6 @@ impl InterpState {
fn end_expansion( fn end_expansion(
s: SpecSlice, s: SpecSlice,
gen_param: GenIdentSymbolId, gen_param: GenIdentSymbolId,
sym: SymbolId,
span: Span, span: Span,
) -> TransitionResult<Self> { ) -> TransitionResult<Self> {
let close = PlainNir::TplParamClose(span); let close = PlainNir::TplParamClose(span);
@ -416,9 +416,7 @@ impl InterpState {
// which is to perform the final replacement of the original // which is to perform the final replacement of the original
// symbol that we've been fed // symbol that we've been fed
// (the specification string). // (the specification string).
Transition(FinishSym(s, gen_param)) Transition(FinishSym(s, gen_param)).ok(Expanded(close))
.ok(Expanded(close))
.with_lookahead((sym, span).into())
} }
} }

106
tamer/src/nir/desugar/interp/test.rs
View File

@ -19,6 +19,7 @@
use super::*; use super::*;
use crate::{ use crate::{
convert::ExpectInto,
nir::PlainNirSymbol, nir::PlainNirSymbol,
parse::Parsed, parse::Parsed,
span::dummy::{DUMMY_CONTEXT as DC, *}, span::dummy::{DUMMY_CONTEXT as DC, *},
@ -39,11 +40,13 @@ fn does_not_desugar_literal_only() {
// `@bar@` is a metavariable, // `@bar@` is a metavariable,
// but it's also a literal because it's not enclosed in braces. // but it's also a literal because it's not enclosed in braces.
for literal in ["foo", "@bar@"] { for literal in ["foo", "@bar@"] {
let name = "foo".unwrap_into();
let sym = literal.into(); let sym = literal.into();
let toks = vec![SPair(sym, S1)]; let toks = vec![Attr::new(name, sym, (S1, S2))];
// Attr should be unchanged.
assert_eq!( assert_eq!(
Ok(vec![Object(DoneExpanding(SPair(sym, S1)))]), Ok(vec![Object(DoneExpanding(Attr::new(name, sym, (S1, S2))))]),
Sut::parse(toks.into_iter()).collect(), Sut::parse(toks.into_iter()).collect(),
"literal `{literal}` must not desugar", "literal `{literal}` must not desugar",
); );
@ -69,11 +72,12 @@ fn desugars_literal_with_ending_var() {
let b = DC.span(10, 3); let b = DC.span(10, 3);
let c = DC.span(14, 5); let c = DC.span(14, 5);
let given_sym = SPair(given_val.into(), a); let name = "foo".unwrap_into();
let given_sym = Attr::new(name, given_val.into(), (S1, a));
let toks = vec![given_sym]; let toks = vec![given_sym];
let GenIdentSymbolId(expect_name) = gen_tpl_param_ident_at_offset(a); let GenIdentSymbolId(expect_pname) = gen_tpl_param_ident_at_offset(a);
let expect_dfn = PlainNirSymbol::Todo(expect_name.into(), a); let expect_dfn = PlainNirSymbol::Todo(expect_pname.into(), a);
let expect_text = PlainNirSymbol::Todo("foo".into(), b); let expect_text = PlainNirSymbol::Todo("foo".into(), b);
let expect_param = PlainNirSymbol::Todo("@bar@".into(), c); let expect_param = PlainNirSymbol::Todo("@bar@".into(), c);
@ -121,10 +125,13 @@ fn desugars_literal_with_ending_var() {
// we replace the original provided attribute // we replace the original provided attribute
// (the interpolation specification) // (the interpolation specification)
// with a metavariable reference to the generated parameter. // with a metavariable reference to the generated parameter.
assert_matches!( assert_eq!(
sut.next(), sut.next(),
Some(Ok(Object(DoneExpanding(SPair(given_replace, given_span))))) Some(Ok(Object(DoneExpanding(Attr::new(
if given_replace == expect_name && given_span == a name,
expect_pname,
(S1, a)
)))))
); );
assert_eq!(sut.next(), None); assert_eq!(sut.next(), None);
@ -148,11 +155,12 @@ fn desugars_var_with_ending_literal() {
let b = DC.span(21, 5); let b = DC.span(21, 5);
let c = DC.span(27, 3); let c = DC.span(27, 3);
let given_sym = SPair(given_val.into(), a); let name = "foo".unwrap_into();
let given_sym = Attr::new(name, given_val.into(), (S1, a));
let toks = vec![given_sym]; let toks = vec![given_sym];
let GenIdentSymbolId(expect_name) = gen_tpl_param_ident_at_offset(a); let GenIdentSymbolId(expect_pname) = gen_tpl_param_ident_at_offset(a);
let expect_dfn = PlainNirSymbol::Todo(expect_name.into(), a); let expect_dfn = PlainNirSymbol::Todo(expect_pname.into(), a);
let expect_param = PlainNirSymbol::Todo("@foo@".into(), b); let expect_param = PlainNirSymbol::Todo("@foo@".into(), b);
let expect_text = PlainNirSymbol::Todo("bar".into(), c); let expect_text = PlainNirSymbol::Todo("bar".into(), c);
@ -173,27 +181,14 @@ fn desugars_var_with_ending_literal() {
); );
assert_eq!( assert_eq!(
sut.next(), Ok(vec![
Some(Ok(Object(Expanded(PlainNir::TplParamValue(expect_param))))), Object(Expanded(PlainNir::TplParamValue(expect_param))),
Object(Expanded(PlainNir::TplParamText(expect_text))),
Object(Expanded(PlainNir::TplParamClose(a))),
Object(DoneExpanding(Attr::new(name, expect_pname, (S1, a)))),
]),
sut.collect(),
); );
assert_eq!(
sut.next(),
Some(Ok(Object(Expanded(PlainNir::TplParamText(expect_text)))))
);
assert_eq!(
sut.next(),
Some(Ok(Object(Expanded(PlainNir::TplParamClose(a)))))
);
assert_matches!(
sut.next(),
Some(Ok(Object(DoneExpanding(SPair(given_replace, given_span)))))
if given_replace == expect_name && given_span == a
);
assert_eq!(sut.next(), None);
} }
// Combination of the above two tests. // Combination of the above two tests.
@ -215,11 +210,12 @@ fn desugars_many_vars_and_literals() {
let d = DC.span(40, 3); let d = DC.span(40, 3);
let e = DC.span(44, 6); let e = DC.span(44, 6);
let given_sym = SPair(given_val.into(), a); let name = "foo".unwrap_into();
let given_sym = Attr::new(name, given_val.into(), (S1, a));
let toks = vec![given_sym]; let toks = vec![given_sym];
let GenIdentSymbolId(expect_name) = gen_tpl_param_ident_at_offset(a); let GenIdentSymbolId(expect_pname) = gen_tpl_param_ident_at_offset(a);
let expect_dfn = PlainNirSymbol::Todo(expect_name.into(), a); let expect_dfn = PlainNirSymbol::Todo(expect_pname.into(), a);
let expect_text1 = PlainNirSymbol::Todo("foo".into(), b); let expect_text1 = PlainNirSymbol::Todo("foo".into(), b);
let expect_param1 = PlainNirSymbol::Todo("@bar@".into(), c); let expect_param1 = PlainNirSymbol::Todo("@bar@".into(), c);
let expect_text2 = PlainNirSymbol::Todo("baz".into(), d); let expect_text2 = PlainNirSymbol::Todo("baz".into(), d);
@ -252,22 +248,11 @@ fn desugars_many_vars_and_literals() {
// offsets. // offsets.
Object(Expanded(PlainNir::TplParamText(expect_text2))), Object(Expanded(PlainNir::TplParamText(expect_text2))),
Object(Expanded(PlainNir::TplParamValue(expect_param2))), Object(Expanded(PlainNir::TplParamValue(expect_param2))),
Object(Expanded(PlainNir::TplParamClose(a))),
Object(DoneExpanding(Attr::new(name, expect_pname, (S1, a)))),
]), ]),
sut.by_ref().take(4).collect(), sut.collect(),
); );
assert_eq!(
sut.next(),
Some(Ok(Object(Expanded(PlainNir::TplParamClose(a)))))
);
assert_matches!(
sut.next(),
Some(Ok(Object(DoneExpanding(SPair(given_replace, given_span)))))
if given_replace == expect_name && given_span == a
);
assert_eq!(sut.next(), None);
} }
// Adjacent vars with empty literal between them. // Adjacent vars with empty literal between them.
@ -286,11 +271,12 @@ fn desugars_adjacent_interpolated_vars() {
let c = DC.span(48, 5); let c = DC.span(48, 5);
let d = DC.span(55, 5); let d = DC.span(55, 5);
let given_sym = SPair(given_val.into(), a); let name = "foo".unwrap_into();
let given_sym = Attr::new(name, given_val.into(), (S1, a));
let toks = vec![given_sym]; let toks = vec![given_sym];
let GenIdentSymbolId(expect_name) = gen_tpl_param_ident_at_offset(a); let GenIdentSymbolId(expect_pname) = gen_tpl_param_ident_at_offset(a);
let expect_dfn = PlainNirSymbol::Todo(expect_name.into(), a); let expect_dfn = PlainNirSymbol::Todo(expect_pname.into(), a);
let expect_param1 = PlainNirSymbol::Todo("@foo@".into(), b); let expect_param1 = PlainNirSymbol::Todo("@foo@".into(), b);
let expect_param2 = PlainNirSymbol::Todo("@bar@".into(), c); let expect_param2 = PlainNirSymbol::Todo("@bar@".into(), c);
let expect_param3 = PlainNirSymbol::Todo("@baz@".into(), d); let expect_param3 = PlainNirSymbol::Todo("@baz@".into(), d);
@ -311,26 +297,14 @@ fn desugars_adjacent_interpolated_vars() {
&& desc_span == a && desc_span == a
); );
// These are the three adjacent vars.
assert_eq!( assert_eq!(
Ok(vec![ Ok(vec![
Object(Expanded(PlainNir::TplParamValue(expect_param1))), Object(Expanded(PlainNir::TplParamValue(expect_param1))),
Object(Expanded(PlainNir::TplParamValue(expect_param2))), Object(Expanded(PlainNir::TplParamValue(expect_param2))),
Object(Expanded(PlainNir::TplParamValue(expect_param3))), Object(Expanded(PlainNir::TplParamValue(expect_param3))),
Object(Expanded(PlainNir::TplParamClose(a))),
Object(DoneExpanding(Attr::new(name, expect_pname, (S1, a)))),
]), ]),
sut.by_ref().take(3).collect(), sut.collect(),
); );
assert_eq!(
sut.next(),
Some(Ok(Object(Expanded(PlainNir::TplParamClose(a)))))
);
assert_matches!(
sut.next(),
Some(Ok(Object(DoneExpanding(SPair(given_replace, given_span)))))
if given_replace == expect_name && given_span == a
);
assert_eq!(sut.next(), None);
} }

14
tamer/src/xir/attr.rs
View File

@ -157,6 +157,20 @@ impl Attr {
Attr(.., span) => span, Attr(.., span) => span,
} }
} }
/// Replace the attribute's value with a new one that has been derived
/// from the original.
///
/// _This does not update the value span!_
/// The intent is for this operation to be used during term rewriting,
/// where the replaced value is _derived from_ the original and so
/// should retain the original span to provide the proper context to
/// the user.
pub fn replace_value_derived(self, value_new: SymbolId) -> Self {
match self {
Self(name, _, span) => Self(name, value_new, span),
}
}
} }
impl Token for Attr { impl Token for Attr {