employer
/
tame
1
0
Fork 0
Code Packages Releases Activity

tamer: xir::parse::ele: Correct handling of sum dead state post-recovery 

Along with this change we also had to change how we handle dead states in
the superstate.  So there were two problems here:

  1. Sum states were not yielding a dead state after recovery, which meant
     that parsing was unable to continue (we still have a `todo!`); and
  2. The superstate considered it an error when there was nothing left on
     the stack, because I assumed that ought not happen.

Regarding #2---it _shouldn't_ happen, _unless_ we have extra input after we
have completed parsing.  Which happens to be the case for this test case,
but more importantly, we shouldn't be panicing with errors about TAMER bugs
if somebody puts extra input after a closing root tag in a source file.

DEV-7145
main
Mike Gerwitz 2022-08-11 10:39:00 -04:00
parent b95ec5a9d8
commit f8a9e952e5
3 changed files with 51 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
tamer/src/parse/state.rs
View File

@ -316,7 +316,11 @@ where
self,
tok: Self::Token,
mut context: C,
dead: impl FnOnce(Self::Token, C) -> TransitionResult<Self::Super>,
dead: impl FnOnce(
Self::Super,
Self::Token,
C,
) -> TransitionResult<Self::Super>,
) -> TransitionResult<Self::Super>
where
C: AsMut<<Self as ParseState>::Context>,
@ -326,7 +330,7 @@ where
match data {
TransitionData::Dead(Lookahead(lookahead)) => {
dead(lookahead, context)
dead(newst, lookahead, context)
}
// Since this is child state,

46
tamer/src/xir/parse/ele.rs
View File

@ -23,11 +23,10 @@ use arrayvec::ArrayVec;
use std::fmt::Display;
use crate::{
diagnose::{panic::DiagnosticPanic, Annotate},
diagnostic_panic,
fmt::{DisplayWrapper, TtQuote},
parse::{
ClosedParseState, Context, ParseState, Token, Transition,
ClosedParseState, Context, ParseState, Transition,
TransitionResult,
},
xir::{Prefix, QName},
@ -112,7 +111,7 @@ impl<S: ClosedParseState> StateStack<S> {
/// `ret` will be pop'd from the stack and we'll transition back to
/// it.
/// Note that this method is not responsible for returning;
/// see [`Self::ret`] to perform a return.
/// see [`Self::ret_or_dead`] to perform a return.
///
/// However,
/// the calling [`ParseState`] is not responsible for its return,
@ -163,8 +162,9 @@ impl<S: ClosedParseState> StateStack<S> {
target
}
/// Return to a previous [`ParseState`] that transferred control away
/// from itself.
/// Attempt to return to a previous [`ParseState`] that transferred
/// control away from itself,
/// otherwise yield a dead state transition to `deadst`.
///
/// Conceptually,
/// this is like returning from a function call,
@ -173,26 +173,25 @@ impl<S: ClosedParseState> StateStack<S> {
/// this system is more akin to CPS
/// (continuation passing style);
/// see [`Self::transfer_with_ret`] for important information.
pub fn ret(&mut self, lookahead: S::Token) -> TransitionResult<S> {
///
/// If there is no state to return to on the stack,
/// then it is assumed that we have received more input than expected
/// after having completed a full parse.
pub fn ret_or_dead(
&mut self,
lookahead: S::Token,
deadst: S,
) -> TransitionResult<S> {
let Self(stack) = self;
// This should certainly never happen unless there is a bug in the
// `ele_parse!` parser-generator,
// since it means that we're trying to return to a caller that
// does not exist.
let st = stack.pop().diagnostic_expect(
lookahead
.span()
.internal_error("while processing this token")
.with_help(
"this implies a bug in TAMER's `ele_parse` \
parser-generator",
)
.into(),
"missing expected return ParseState",
);
Transition(st).incomplete().with_lookahead(lookahead)
match stack.pop() {
Some(st) => Transition(st).incomplete().with_lookahead(lookahead),
None => Transition(deadst).dead(lookahead),
}
}
}
@ -1121,7 +1120,10 @@ macro_rules! ele_parse {
Transition(st).incomplete()
},
(st @ Self::Done_, tok) => Transition(st).dead(tok),
(
st @ (Done_ | RecoverEleIgnoreClosed_(..)),
tok
) => Transition(st).dead(tok),
todo => todo!("sum {todo:?}"),
}
@ -1276,7 +1278,9 @@ macro_rules! ele_parse {
Self::$nt(st) => st.delegate_child(
tok,
stack,
|tok, stack| stack.ret(tok),
|deadst, tok, stack| {
stack.ret_or_dead(tok, deadst)
},
),
)*
}

20
tamer/src/xir/parse/ele/test.rs
View File

@ -1166,6 +1166,10 @@ fn sum_nonterminal_error_recovery() {
let depth = Depth(5);
let depth_child = Depth(6);
// An extra token to yield after we're done parsing to ensure that we
// properly yield a dead state transition.
let dead_tok = XirfToken::Open(QN_A, OpenSpan(S5, N), depth);
let toks = vec![
// Neither A nor B,
// which will produce an error and enter recovery.
@ -1181,6 +1185,9 @@ fn sum_nonterminal_error_recovery() {
// Closing token for the bad element at the corresponding depth,
// which will end recovery.
XirfToken::Close(Some(unexpected), CloseSpan(S4, N), depth),
// Should result in a dead state post-recovery,
// just as we would expect if we _didn't_ recover.
dead_tok.clone(),
];
let mut sut = Sut::parse(toks.into_iter());
@ -1221,6 +1228,19 @@ fn sum_nonterminal_error_recovery() {
// But since we are not emitting tokens,
// we'll still be marked as incomplete.
assert_eq!(Some(Ok(Parsed::Incomplete)), sut.next()); // Close root
// Encountering any tokens post-recovery should result in a dead state
// just the same as if we had closed normally.
let err = sut.next().unwrap().unwrap_err();
assert_matches!(
err,
ParseError::UnexpectedToken(given_tok, _) if given_tok == dead_tok,
);
// Having otherwise completed successfully,
// and now yielding dead states,
// we must indicate that parsing has completed successfully so that
// the caller knows that it can safely move on.
sut.finalize()
.expect("recovery must complete in an accepting state");
}