tamer: parse: Abstract lookahead token replacement panic

There's no use in duplicating this in util::expand.

Lookahead tokens are one of the few invariants that I haven't taken the time
of enforcing using the type system, because it'd be quite a bit of work that
I do not have time for, and may not be worth it with changes that may make
the system less ergonomic.  Nonetheless, I do hope to address it at some
point in the (possibly-far) future.

If ever you encounter this diagnostic message, ask yourself how stable TAMER
otherwise is and how many other issues like this have been entirely
prevented through compile-time proofs using the type system.

DEV-13156

tamer/src/parse/state/transition.rs

@@ -79,6 +79,17 @@ impl<S: ParseState> TransitionResult<S> {
     /// Indicate that this transition include a single token of lookahead,
     ///   which should be provided back to the parser in place of the
     ///   next token from the input stream.
+    ///
+    /// Panics
+    /// ======
+    /// A critical invariant of this system is that lookahead tokens must
+    ///   never be discarded without explicit handling.
+    /// If this [`TransitionResult`] contains an existing token of lookahead,
+    ///   the system will panic when attempting to overwrite it.
+    /// This represents a bug in the system,
+    ///   since parsers should never permit this to occur.
+    ///
+    /// Ideally this will be enforced using the type system in the future.
     pub fn with_lookahead(self, lookahead: S::Token) -> Self {
         match self {
             Self(transition, TransitionData::Result(result, None)) => Self(
@@ -92,22 +103,12 @@ impl<S: ParseState> TransitionResult<S> {
             //   ever such a thing is deemed to be worth doing.
             Self(
                 ..,
-                TransitionData::Result(_, Some(Lookahead(prev)))
-                | TransitionData::Dead(Lookahead(prev)),
-            ) => {
-                let desc = vec![
-                    prev.span().note("this token of lookahead would be lost"),
-                    lookahead.span().internal_error(
-                        "attempting to replace previous \
-                            lookahead token with this one",
-                    ),
-                ];
-
-                diagnostic_panic!(
-                    desc,
-                    "cannot overwrite unused lookahead token"
-                )
-            }
+                TransitionData::Result(_, Some(prev))
+                | TransitionData::Dead(prev),
+            ) => prev.overwrite_panic(
+                lookahead,
+                "cannot overwrite unused lookahead token",
+            ),
         }
     }
 
@@ -231,6 +232,29 @@ impl<S: ParseState> TransitionResult<S> {
 #[derive(Debug, PartialEq)]
 pub struct Lookahead<T: Token>(pub(in super::super) T);
 
+impl<T: Token> Lookahead<T> {
+    /// Panic with diagnostic information about a lookup token and its
+    ///   attempted replacement.
+    ///
+    /// A critical system invariant is that lookahead tokens must never be
+    ///   lost without explicit handling.
+    /// Since this is not yet enforced using the type system,
+    ///   these checks must be performed at runtime.
+    pub(in super::super) fn overwrite_panic(self, other: T, msg: &str) -> ! {
+        let Self(prev) = self;
+
+        let desc = vec![
+            prev.span().note("this token of lookahead would be lost"),
+            other.span().internal_error(
+                "attempting to replace previous lookahead token \
+                   with this one",
+            ),
+        ];
+
+        diagnostic_panic!(desc, "{msg}",)
+    }
+}
+
 /// Information about the state transition.
 ///
 /// Note: Ideally a state wouldn't even be required for

tamer/src/parse/util/expand.rs

@@ -22,8 +22,6 @@
 //! _Expansion_ refers to the production of many [`Object`]s that are
 //!   derived from a single [`Token`].
 
-use crate::{diagnose::Annotate, diagnostic_panic};
-
 use super::super::{
     prelude::*,
     state::{Lookahead, StitchableParseState, TransitionData},
@@ -131,31 +129,15 @@ where
                         TransitionData::Result(Ok(ParseStatus::Object(obj)), la)
                     }
 
-                    // A parser must never throw away lookahead tokens.
                     // Since we are converting the `DoneExpanding` variant
                     //   into a lookahead token,
                     //     we would have nothing to do with a token of
                     //     lookahead if one were provided to us.
-                    // Ideally this would be prevented using types,
-                    //   but such a change is too much effort at the time of
-                    //   writing.
-                    (DoneExpanding(tok), Some(Lookahead(la_tok))) => {
-                        let desc = vec![
-                            tok.span().note(
-                                "while processing this \
-                            Expansion::DoneExpanding token",
-                            ),
-                            la_tok.span().internal_error(
-                                "encountered this unexpected lookahead token",
-                            ),
-                        ];
-
-                        diagnostic_panic!(
-                            desc,
-                            "cannot provide lookahead token with \
-                        Expansion::DoneExpanding",
-                        )
-                    }
+                    (DoneExpanding(tok), Some(la)) => la.overwrite_panic(
+                        tok,
+                        "cannot provide lookahead token with \
+                            Expansion::DoneExpanding",
+                    ),
 
                     (DoneExpanding(tok), None) => {
                         TransitionData::Dead(Lookahead(tok))