From a68930589e48abc6d00a50da7bdf743db1e6ace5 Mon Sep 17 00:00:00 2001
From: Mike Gerwitz <mike.gerwitz@ryansg.com>
Date: Tue, 21 Feb 2023 12:05:46 -0500
Subject: [PATCH] tamer: parse::lower: Handle EOF token

This was missed (because it was not used) when EOF tokens were originally
introduced via `ParseState::eof_tok`---`LowerIter` also needs to consider
the token.

This separation betwen the two iterators is a maintenance burden that needs
to be taken care of; I knew that at the time, and then I forgot about it,
and here we are.

This was caught while beginning to wire together a POC graph lowering
pipeline to emit derived sources.

DEV-13708
---
 tamer/src/parse/lower.rs  | 31 +++++++++++++++++++++++++++----
 tamer/src/parse/parser.rs | 13 +++++++++++--
 2 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/tamer/src/parse/lower.rs b/tamer/src/parse/lower.rs
index f57cc95f..167622b6 100644
--- a/tamer/src/parse/lower.rs
+++ b/tamer/src/parse/lower.rs
@@ -194,12 +194,16 @@ where
     /// Errors from `LS` are widened into `E`.
     #[inline]
     fn next(&mut self) -> Option<Self::Item> {
+        // TODO: This is a maintenance burden with Parser's Iterator impl;
+        //   they can easily get out of sync,
+        //     as evidenced by the commit introducing this comment.
         let tok = self
             .lower
             .take_lookahead_tok()
             .map(Parsed::Object)
             .map(Ok)
-            .or_else(|| self.toks.next());
+            .or_else(|| self.toks.next())
+            .or_else(|| self.lower.eof_tok().map(Parsed::Object).map(Ok));
 
         match tok {
             // We are done when no tokens remain.
@@ -311,7 +315,11 @@ mod test {
     use super::*;
 
     #[derive(Debug, PartialEq, Eq, Default)]
-    struct StubEchoParseState {}
+    enum StubEchoParseState {
+        #[default]
+        PreEof,
+        PostEof,
+    }
 
     impl Display for StubEchoParseState {
         fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
@@ -329,17 +337,24 @@ mod test {
             tok: Self::Token,
             _: &mut Self::Context,
         ) -> TransitionResult<Self> {
-            Transition(self).ok(tok)
+            match tok {
+                StubToken::Foo => Transition(Self::PostEof).ok(tok),
+                _ => Transition(self).ok(tok),
+            }
         }
 
         fn is_accepting(&self, _: &Self::Context) -> bool {
             true
         }
+
+        fn eof_tok(&self, _ctx: &Self::Context) -> Option<Self::Token> {
+            matches!(self, Self::PreEof).then_some(StubToken::Foo)
+        }
     }
 
     // Similar to tests in parse::parser::test.
     #[test]
-    fn can_emit_object_with_lookahead_for_lower_iter() {
+    fn can_emit_object_with_lookahead_and_eof_for_lower_iter() {
         let given = 27; // some value
         let toks = vec![StubToken::YieldWithLookahead(given)];
 
@@ -362,6 +377,14 @@ mod test {
                     "lookahead token did not take effect"
                 );
 
+                // Prior to end,
+                //   we give parsers the opportunity to emit an EOF token.
+                assert_eq!(
+                    sut.next(),
+                    Some(Ok(Parsed::Object(StubObject::FromFoo))),
+                    "EOF token was note emitted",
+                );
+
                 // And now this should be the end,
                 //   provided that the lookahead token was actually consumed and not
                 //   copied and retained.
diff --git a/tamer/src/parse/parser.rs b/tamer/src/parse/parser.rs
index 99b87df0..45e8821c 100644
--- a/tamer/src/parse/parser.rs
+++ b/tamer/src/parse/parser.rs
@@ -368,6 +368,14 @@ impl<S: ClosedParseState, I: TokenStream<S::Token>> Parser<S, I> {
     pub(super) fn take_lookahead_tok(&mut self) -> Option<S::Token> {
         self.lookahead.take().map(|Lookahead(tok)| tok)
     }
+
+    /// An optional token to feed to `Self::feed_tok` after the
+    ///   [`TokenStream`] has ended.
+    ///
+    /// See [`ParseState::eof_tok`] for more information.
+    pub(super) fn eof_tok(&self) -> Option<S::Token> {
+        self.state.as_ref().unwrap().eof_tok(&self.ctx)
+    }
 }
 
 impl<S: ClosedParseState, I: TokenStream<S::Token>> Iterator for Parser<S, I> {
@@ -390,7 +398,7 @@ impl<S: ClosedParseState, I: TokenStream<S::Token>> Iterator for Parser<S, I> {
         let otok = self
             .take_lookahead_tok()
             .or_else(|| self.toks.next())
-            .or_else(|| self.state.as_ref().unwrap().eof_tok(&self.ctx));
+            .or_else(|| self.eof_tok());
 
         match otok {
             None => match self.assert_accepting() {
@@ -569,6 +577,7 @@ pub mod test {
     pub enum StubObject {
         FromYield(usize),
         FromLookahead(usize),
+        FromFoo,
     }
 
     impl Object for StubObject {}
@@ -600,7 +609,7 @@ pub mod test {
                 StubToken::Lookahead(val) => {
                     Transition(self).ok(StubObject::FromLookahead(val))
                 }
-                _ => Transition(self).incomplete(),
+                StubToken::Foo => Transition(self).ok(StubObject::FromFoo),
             }
         }