diff --git a/tamer/src/parse/lower.rs b/tamer/src/parse/lower.rs index f57cc95f..167622b6 100644 --- a/tamer/src/parse/lower.rs +++ b/tamer/src/parse/lower.rs @@ -194,12 +194,16 @@ where /// Errors from `LS` are widened into `E`. #[inline] fn next(&mut self) -> Option { + // TODO: This is a maintenance burden with Parser's Iterator impl; + // they can easily get out of sync, + // as evidenced by the commit introducing this comment. let tok = self .lower .take_lookahead_tok() .map(Parsed::Object) .map(Ok) - .or_else(|| self.toks.next()); + .or_else(|| self.toks.next()) + .or_else(|| self.lower.eof_tok().map(Parsed::Object).map(Ok)); match tok { // We are done when no tokens remain. @@ -311,7 +315,11 @@ mod test { use super::*; #[derive(Debug, PartialEq, Eq, Default)] - struct StubEchoParseState {} + enum StubEchoParseState { + #[default] + PreEof, + PostEof, + } impl Display for StubEchoParseState { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { @@ -329,17 +337,24 @@ mod test { tok: Self::Token, _: &mut Self::Context, ) -> TransitionResult { - Transition(self).ok(tok) + match tok { + StubToken::Foo => Transition(Self::PostEof).ok(tok), + _ => Transition(self).ok(tok), + } } fn is_accepting(&self, _: &Self::Context) -> bool { true } + + fn eof_tok(&self, _ctx: &Self::Context) -> Option { + matches!(self, Self::PreEof).then_some(StubToken::Foo) + } } // Similar to tests in parse::parser::test. #[test] - fn can_emit_object_with_lookahead_for_lower_iter() { + fn can_emit_object_with_lookahead_and_eof_for_lower_iter() { let given = 27; // some value let toks = vec![StubToken::YieldWithLookahead(given)]; @@ -362,6 +377,14 @@ mod test { "lookahead token did not take effect" ); + // Prior to end, + // we give parsers the opportunity to emit an EOF token. + assert_eq!( + sut.next(), + Some(Ok(Parsed::Object(StubObject::FromFoo))), + "EOF token was note emitted", + ); + // And now this should be the end, // provided that the lookahead token was actually consumed and not // copied and retained. diff --git a/tamer/src/parse/parser.rs b/tamer/src/parse/parser.rs index 99b87df0..45e8821c 100644 --- a/tamer/src/parse/parser.rs +++ b/tamer/src/parse/parser.rs @@ -368,6 +368,14 @@ impl> Parser { pub(super) fn take_lookahead_tok(&mut self) -> Option { self.lookahead.take().map(|Lookahead(tok)| tok) } + + /// An optional token to feed to `Self::feed_tok` after the + /// [`TokenStream`] has ended. + /// + /// See [`ParseState::eof_tok`] for more information. + pub(super) fn eof_tok(&self) -> Option { + self.state.as_ref().unwrap().eof_tok(&self.ctx) + } } impl> Iterator for Parser { @@ -390,7 +398,7 @@ impl> Iterator for Parser { let otok = self .take_lookahead_tok() .or_else(|| self.toks.next()) - .or_else(|| self.state.as_ref().unwrap().eof_tok(&self.ctx)); + .or_else(|| self.eof_tok()); match otok { None => match self.assert_accepting() { @@ -569,6 +577,7 @@ pub mod test { pub enum StubObject { FromYield(usize), FromLookahead(usize), + FromFoo, } impl Object for StubObject {} @@ -600,7 +609,7 @@ pub mod test { StubToken::Lookahead(val) => { Transition(self).ok(StubObject::FromLookahead(val)) } - _ => Transition(self).incomplete(), + StubToken::Foo => Transition(self).ok(StubObject::FromFoo), } }