employer
/
tame
1
0
Fork 0
Code Packages Releases Activity

tamer: asg::air: Extract template parsing into own parser 

Just as was done with the expression parser, which this will utilize.  This
initializes it, but doesn't yet make use of it (`AirExprAggregate`).

Refactoring was definitely needed; decomposing this is quite a bit of work,
in no small part because of the complexity.  This helps significantly.

DEV-13708
main
Mike Gerwitz 2023-03-07 12:41:47 -05:00
parent 25c0aa180e
commit f307f2d70b
5 changed files with 284 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

231
tamer/src/asg/air.rs
View File

@ -271,11 +271,7 @@ pub enum AirAggregate {
/// All objects encountered until the closing [`Air::TplClose`] will be
/// parented to this template rather than the parent [`Pkg`].
/// See [`Air::TplOpen`] for more information.
BuildingTpl(
(ObjectIndex<Pkg>, AirExprAggregate),
ObjectIndex<Tpl>,
Option<SPair>,
),
PkgTpl(ObjectIndex<Pkg>, AirExprAggregate, AirTplAggregate),
}
impl Display for AirAggregate {
@ -290,15 +286,8 @@ impl Display for AirAggregate {
PkgExpr(_, expr) => {
write!(f, "defining a package expression: {expr}")
}
BuildingTpl((_, expr), _, None) => {
write!(f, "building anonymous template with {expr}")
}
BuildingTpl((_, expr), _, Some(name)) => {
write!(
f,
"building named template {} with {expr}",
TtQuote::wrap(name)
)
PkgTpl(_, _, tpl) => {
write!(f, "building a template: {tpl}",)
}
}
}
@ -320,8 +309,7 @@ impl ParseState for AirAggregate {
asg: &mut Self::Context,
) -> crate::parse::TransitionResult<Self> {
use ir::{
AirBind::*, AirIdent::*, AirPkg::*, AirSubsets::*, AirTodo::*,
AirTpl::*,
AirIdent::*, AirPkg::*, AirSubsets::*, AirTodo::*, AirTpl::*,
};
use AirAggregate::*;
@ -345,16 +333,6 @@ impl ParseState for AirAggregate {
.err(AsgError::NestedPkgOpen(span, oi_pkg.span()))
}
(
PkgExpr(oi_pkg, expr) | PkgHead(oi_pkg, expr),
AirTpl(TplOpen(span)),
) => {
let oi_tpl = asg.create(Tpl::new(span));
Transition(BuildingTpl((oi_pkg, expr), oi_tpl, None))
.incomplete()
}
// No expression was started.
(PkgHead(oi_pkg, _expr), AirPkg(PkgClose(span))) => {
oi_pkg.close(asg, span);
@ -371,6 +349,19 @@ impl ParseState for AirAggregate {
.with_lookahead(tok)
}
// Note that templates may preempt expressions at any point,
// unlike in NIR at the time of writing.
(
PkgHead(oi_pkg, expr) | PkgExpr(oi_pkg, expr),
tok @ AirTpl(..),
) => Transition(PkgTpl(
oi_pkg,
expr,
AirTplAggregate::define_in_pkg(oi_pkg),
))
.incomplete()
.with_lookahead(tok),
// Note: We unfortunately can't match on `AirExpr | AirBind`
// and delegate in the same block
// (without having to duplicate type checks and then handle
@ -383,8 +374,14 @@ impl ParseState for AirAggregate {
Self::delegate_expr(asg, oi_pkg, expr, etok)
}
(PkgExpr(..), AirTpl(TplClose(..))) => {
todo!("PkgExpr AirTpl::TplClose")
// Templates can contain just about anything,
// so completely hand over parsing when we're in template mode.
(PkgTpl(oi_pkg, stored_expr, tplst), tok) => {
Self::delegate_tpl(asg, oi_pkg, stored_expr, tplst, tok.into())
}
(Empty, AirTpl(TplClose(..))) => {
todo!("Empty AirTpl::TplClose")
}
(Empty, AirPkg(PkgClose(span))) => {
@ -432,30 +429,6 @@ impl ParseState for AirAggregate {
Transition(Empty).incomplete()
}
(
BuildingTpl((oi_pkg, expr), oi_tpl, None),
AirBind(BindIdent(name)),
) => asg
.lookup_or_missing(name)
.bind_definition(asg, name, oi_tpl)
.map(|oi_ident| oi_pkg.defines(asg, oi_ident))
.map(|_| ())
.transition(BuildingTpl((oi_pkg, expr), oi_tpl, Some(name))),
(
BuildingTpl((oi_pkg, expr), oi_tpl, _),
AirTpl(TplClose(span)),
) => {
oi_tpl.close(asg, span);
Transition(PkgHead(oi_pkg, expr)).incomplete()
}
(BuildingTpl(..), tok) => todo!("BuildingTpl body: {tok:?}"),
(st @ (Empty | PkgHead(..)), AirTpl(TplClose(span))) => {
Transition(st).err(AsgError::UnbalancedTpl(span))
}
(st, tok @ AirIdent(..)) => todo!("{st:?}, {tok:?}"),
}
}
@ -500,6 +473,35 @@ impl AirAggregate {
(),
)
}
/// Delegate to the expression parser [`AirTplAggregate`].
///
/// After template parsing is complete
/// (when reaching a dead state),
/// the stored expression [`AirExprAggregate`] is reinstated,
/// allowing parsing to continue where it left off before being
/// preempted by template parsing.
fn delegate_tpl(
asg: &mut <Self as ParseState>::Context,
oi_pkg: ObjectIndex<Pkg>,
stored_expr: AirExprAggregate,
tplst: AirTplAggregate,
tok: Air,
) -> TransitionResult<Self> {
tplst
.parse_token(tok, asg)
.branch_dead::<Self, AirExprAggregate>(
|_, stored_expr| {
Transition(Self::PkgExpr(oi_pkg, stored_expr)).incomplete()
},
|tplst, result, stored_expr| {
result
.map(ParseStatus::reflexivity)
.transition(Self::PkgTpl(oi_pkg, stored_expr, tplst))
},
stored_expr,
)
}
}
/// Parse an AIR expression with binding support.
@ -631,5 +633,130 @@ impl AirExprAggregate {
}
}
/// Template parser and token aggregator.
///
/// A template consists of
///
/// - Metadata about the template,
/// including its parameters; and
/// - A collection of [`Air`] tokens representing the body of the
/// template that will be expanded into the application site when the
/// template is applied.
///
/// This contains an embedded [`AirExprAggregate`] parser for handling
/// expressions just the same as [`AirAggregate`] does with packages.
#[derive(Debug, PartialEq, Eq)]
pub enum AirTplAggregate {
/// Ready for a template,
/// defined as part of the given package.
///
/// This state also include the template header;
/// unlike NIR,
/// AIR has no restrictions on when template header tokens are
/// provided,
/// which simplifies AIR generation.
///
/// The expression parser [`AirExprAggregate`] is initialized at this
/// state to avoid re-allocating its stack for adjacent templates.
/// This will not save any allocations if,
/// after reaching a dead state,
/// the caller
/// (or parent parser)
/// chooses to deallocate us.
Ready(ObjectIndex<Pkg>, AirExprAggregate),
/// Aggregating tokens into a template.
BuildingTpl(
ObjectIndex<Pkg>,
ObjectIndex<Tpl>,
AirExprAggregate,
Option<SPair>,
),
}
impl Display for AirTplAggregate {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
Self::Ready(_, _) => write!(f, "ready for template definition"),
Self::BuildingTpl(_, expr, _, None) => {
write!(f, "building anonymous template with {expr}")
}
Self::BuildingTpl(_, expr, _, Some(name)) => {
write!(
f,
"building named template {} with {expr}",
TtQuote::wrap(name)
)
}
}
}
}
impl ParseState for AirTplAggregate {
type Token = Air;
type Object = ();
type Error = AsgError;
type Context = Asg;
fn parse_token(
self,
tok: Self::Token,
asg: &mut Self::Context,
) -> TransitionResult<Self::Super> {
use ir::{AirBind::*, AirSubsets::*, AirTodo::*, AirTpl::*};
use AirTplAggregate::*;
match (self, tok.into()) {
(st, AirTodo(Todo(_))) => Transition(st).incomplete(),
(Ready(oi_pkg, expr), AirTpl(TplOpen(span))) => {
let oi_tpl = asg.create(Tpl::new(span));
Transition(BuildingTpl(oi_pkg, oi_tpl, expr, None)).incomplete()
}
(
BuildingTpl(oi_pkg, oi_tpl, expr, _),
AirBind(BindIdent(name)),
) => asg
.lookup_or_missing(name)
.bind_definition(asg, name, oi_tpl)
.map(|oi_ident| oi_pkg.defines(asg, oi_ident))
.map(|_| ())
.transition(BuildingTpl(oi_pkg, oi_tpl, expr, Some(name))),
(BuildingTpl(oi_pkg, oi_tpl, expr, _), AirTpl(TplClose(span))) => {
oi_tpl.close(asg, span);
Transition(Ready(oi_pkg, expr)).incomplete()
}
(BuildingTpl(..), AirPkg(_)) => {
todo!("template cannot define packages")
}
(BuildingTpl(..), tok) => todo!("BuildingTpl body: {tok:?}"),
(st @ Ready(..), AirTpl(TplClose(span))) => {
Transition(st).err(AsgError::UnbalancedTpl(span))
}
(
st @ Ready(..),
tok @ (AirPkg(..) | AirExpr(..) | AirBind(..) | AirIdent(..)),
) => Transition(st).dead(tok.into()),
}
}
fn is_accepting(&self, _: &Self::Context) -> bool {
matches!(self, Self::Ready(..))
}
}
impl AirTplAggregate {
fn define_in_pkg(oi_pkg: ObjectIndex<Pkg>) -> Self {
Self::Ready(oi_pkg, AirExprAggregate::parse_pkg(oi_pkg))
}
}
#[cfg(test)]
mod test;

98
tamer/src/asg/air/test.rs
View File

@ -1155,6 +1155,7 @@ fn tpl_defining_pkg() {
let toks = vec![
Air::PkgOpen(S1),
// This also tests tpl as a transition away from the package header.
Air::TplOpen(S2),
Air::BindIdent(id_tpl),
Air::TplClose(S4),
@ -1175,6 +1176,103 @@ fn tpl_defining_pkg() {
);
}
#[test]
fn tpl_after_expr() {
let id_expr = SPair("expr".into(), S3);
let id_tpl = SPair("_tpl_".into(), S6);
#[rustfmt::skip]
let toks = vec![
Air::PkgOpen(S1),
// This expression is incidental to this test;
// it need only parse.
Air::ExprOpen(ExprOp::Sum, S2),
Air::BindIdent(id_expr),
Air::ExprClose(S4),
// Open after an expression.
Air::TplOpen(S5),
Air::BindIdent(id_tpl),
Air::TplClose(S7),
Air::PkgClose(S8),
];
let mut sut = Sut::parse(toks.into_iter());
assert!(sut.all(|x| x.is_ok()));
let asg = sut.finalize().unwrap().into_context();
let tpl = asg.expect_ident_obj::<Tpl>(id_tpl);
assert_eq!(S5.merge(S7).unwrap(), tpl.span());
}
// Templates within expressions are permitted by NIR at the time of writing
// (and so cannot be observed in system tests using TAME's source
// language),
// but it _is_ permitted by AIR,
// to simplify lowering, desugaring, and template expansion.
//
// This test is a rather important one,
// since it ensures that expression context is properly restored
// regardless of whether a template is encountered.
// This context includes the entire active expression stack.
#[test]
fn tpl_within_expr() {
let id_expr = SPair("expr".into(), S3);
let id_tpl = SPair("_tpl_".into(), S7);
#[rustfmt::skip]
let toks = vec![
Air::PkgOpen(S1),
Air::ExprOpen(ExprOp::Sum, S2),
Air::BindIdent(id_expr),
// Child expression before the template to ensure that the
// context is properly restored after template parsing.
Air::ExprOpen(ExprOp::Sum, S4),
Air::ExprClose(S5),
// Template _within_ an expression.
// This will not be present in the final expression,
// as if it were hoisted out.
Air::TplOpen(S6),
Air::BindIdent(id_tpl),
Air::TplClose(S8),
// Child expression _after_ the template for the same reason.
Air::ExprOpen(ExprOp::Sum, S9),
Air::ExprClose(S10),
Air::ExprClose(S11),
Air::PkgClose(S12),
];
let mut sut = Sut::parse(toks.into_iter());
assert!(sut.all(|x| x.is_ok()));
let asg = sut.finalize().unwrap().into_context();
// The inner template.
let tpl = asg.expect_ident_obj::<Tpl>(id_tpl);
assert_eq!(S6.merge(S8).unwrap(), tpl.span());
// The expression that was produced on the graph ought to be equivalent
// to the expression without the template being present at all
// (noting that the spans are of course not adjusted).
let oi_expr = asg.expect_ident_oi::<Expr>(id_expr);
let expr = oi_expr.resolve(&asg);
assert_eq!(S2.merge(S11).unwrap(), expr.span());
assert_eq!(
#[rustfmt::skip]
vec![
S4.merge(S5).unwrap(),
S9.merge(S10).unwrap(),
],
collect_subexprs(&asg, oi_expr)
.iter()
.map(|(_, expr)| expr.span())
.rev()
.collect::<Vec<_>>(),
);
}
fn asg_from_toks<I: IntoIterator<Item = Air>>(toks: I) -> Asg
where
I::IntoIter: Debug,

3
tamer/src/lib.rs
View File

@ -81,6 +81,9 @@
// should it become too complex then we should re-evaluate what we ought
// to be doing relative to the status of this feature.
#![feature(return_position_impl_trait_in_trait)]
// Added for use with `rustfmt::skip`,
// so that we can ignore formatting more precisely.
#![feature(stmt_expr_attributes)]
// We build docs for private items.
#![allow(rustdoc::private_intra_doc_links)]
// For sym::prefill recursive macro `static_symbols!`.

2
tamer/tests/xmli/expected.xml
View File

@ -33,4 +33,6 @@
</classify>
<template name="_empty_" />
<rate yields="postTpl" />
</package>

2
tamer/tests/xmli/src.xml
View File

@ -37,4 +37,6 @@
</classify>
<template name="_empty_" />
<rate yields="postTpl" />
</package>