tamer: parser::Parser: cfg(test) tracing

This produces useful parse traces that are output as part of a failing test
case.  The parser generator macros can be a bit confusing to deal with when
things go wrong, so this helps to clarify matters.

This is _not_ intended to be machine-readable, but it does show that it
would be possible to generate machine-readable output to visualize the
entire lowering pipeline.  Perhaps something for the future.

I left these inline in Parser::feed_tok because they help to elucidate what
is going on, just by reading what the trace would output---that is, it helps
to make the method more self-documenting, albeit a tad bit more
verbose.  But with that said, it should probably be extracted at some point;
I don't want this to set a precedent where composition is feasible.

Here's an example from test cases:

  [Parser::feed_tok] (input IR: XIRF)
  |  ==> Parser before tok is parsing attributes for `package`.
  |   |  Attrs_(SutAttrsState_ { ___ctx: (QName(None, LocalPart(NCName(SymbolId(46 "package")))), OpenSpan(Span { len: 0, offset: 0, ctx: Context(SymbolId(1 "#!DUMMY")) }, 10)), ___done: false })
  |
  |  ==> XIRF tok: `<unexpected>`
  |   |  Open(QName(None, LocalPart(NCName(SymbolId(82 "unexpected")))), OpenSpan(Span { len: 0, offset: 1, ctx: Context(SymbolId(1 "#!DUMMY")) }, 10), Depth(1))
  |
  |  ==> Parser after tok is expecting opening tag `<classify>`.
  |   |  ChildA(Expecting_)
  |   |  Lookahead: Some(Lookahead(Open(QName(None, LocalPart(NCName(SymbolId(82 "unexpected")))), OpenSpan(Span { len: 0, offset: 1, ctx: Context(SymbolId(1 "#!DUMMY")) }, 10), Depth(1))))
  = note: this trace was output as a debugging aid because `cfg(test)`.

  [Parser::feed_tok] (input IR: XIRF)
  |  ==> Parser before tok is expecting opening tag `<classify>`.
  |   |  ChildA(Expecting_)
  |
  |  ==> XIRF tok: `<unexpected>`
  |   |  Open(QName(None, LocalPart(NCName(SymbolId(82 "unexpected")))), OpenSpan(Span { len: 0, offset: 1, ctx: Context(SymbolId(1 "#!DUMMY")) }, 10), Depth(1))
  |
  |  ==> Parser after tok is attempting to recover by ignoring element with unexpected name `unexpected` (expected `classify`).
  |   |  ChildA(RecoverEleIgnore_(QName(None, LocalPart(NCName(SymbolId(82 "unexpected")))), OpenSpan(Span { len: 0, offset: 1, ctx: Context(SymbolId(1 "#!DUMMY")) }, 10), Depth(1)))
  |   |  Lookahead: None
  = note: this trace was output as a debugging aid because `cfg(test)`.

DEV-7145

tamer/src/asg/air.rs

@@ -73,6 +73,10 @@ pub enum AirToken {
 }
 
 impl Token for AirToken {
+    fn ir_name() -> &'static str {
+        "AIR"
+    }
+
     fn span(&self) -> crate::span::Span {
         // TODO: This can be provided once the xmlo files store source
         //   locations for symbols.

tamer/src/obj/xmlo/reader.rs

@@ -92,6 +92,10 @@ pub enum XmloToken {
 impl parse::Object for XmloToken {}
 
 impl Token for XmloToken {
+    fn ir_name() -> &'static str {
+        "xmlo"
+    }
+
     fn span(&self) -> Span {
         use XmloToken::*;
 

tamer/src/parse.rs

@@ -48,6 +48,12 @@ use std::{
 pub trait Token: Display + Debug + PartialEq {
     /// Retrieve the [`Span`] representing the source location of the token.
     fn span(&self) -> Span;
+
+    /// Name of the intermediate representation (IR) this token represents.
+    ///
+    /// This is used for diagnostic information,
+    ///   primarily for debugging TAMER itself.
+    fn ir_name() -> &'static str;
 }
 
 impl<T: Token> From<T> for Span {
@@ -68,6 +74,10 @@ impl Token for UnknownToken {
     fn span(&self) -> Span {
         DUMMY_SPAN
     }
+
+    fn ir_name() -> &'static str {
+        "<UNKNOWN IR>"
+    }
 }
 
 impl Display for UnknownToken {
@@ -124,12 +134,16 @@ pub mod test {
     }
 
     impl Display for TestToken {
-        fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-            unimplemented!("fmt::Display")
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            write!(f, "(test token)")
         }
     }
 
     impl Token for TestToken {
+        fn ir_name() -> &'static str {
+            "<PARSE TEST IR>"
+        }
+
         fn span(&self) -> Span {
             use TestToken::*;
             match self {

tamer/src/parse/parser.rs

@@ -250,6 +250,38 @@ impl<S: ParseState, I: TokenStream<S::Token>> Parser<S, I> {
             "lookahead token is available but was not consumed",
         );
 
+        // Human-readable trace that will become part of a failed test
+        //   cases's output.
+        // This describes the state prior to the transition,
+        //   and is left here inline since it also helps to document what
+        //   this method is doing.
+        // This is _not_ intended to be machine-readable or stable,
+        //   so please do not parse it;
+        //     if we want a machine-readable format for e.g. creating a
+        //     visualization of a parse,
+        //       such a system can be created separately.
+        //
+        // Note: if one of these trace blocks does not fully output,
+        //   then you may have a `Display::fmt` or `Debug::fmt` panic,
+        //     like a `todo!` or `unimplemented!`,
+        //     in your `Token` or `ParseState`.
+        #[cfg(test)]
+        {
+            let st = self.state.as_ref().unwrap();
+
+            eprint!(
+                "\
+[Parser::feed_tok] (input IR: {ir})
+|  ==> Parser before tok is {st}.
+|   |  {st:?}
+|
+|  ==> {ir} tok: {tok}
+|   |  {tok:?}
+|\n",
+                ir = S::Token::ir_name()
+            );
+        }
+
         // Parse a single token and perform the requested state transition.
         //
         // This is where the functional `ParseState` is married with a
@@ -267,6 +299,21 @@ impl<S: ParseState, I: TokenStream<S::Token>> Parser<S, I> {
             self.state.take().unwrap().parse_token(tok, &mut self.ctx);
         self.state.replace(state);
 
+        // Remainder of the trace after the transition.
+        #[cfg(test)]
+        {
+            let newst = self.state.as_ref().unwrap();
+
+            eprint!(
+                "\
+|  ==> Parser after tok is {newst}.
+|   |  {newst:?}
+|   |  Lookahead: {:?}
+= note: this trace was output as a debugging aid because `cfg(test)`.\n\n",
+                data.lookahead_ref()
+            );
+        }
+
         use ParseStatus::{Incomplete, Object};
         match data {
             // Nothing handled this dead state,
@@ -480,6 +527,10 @@ pub mod test {
     }
 
     impl Token for StubToken {
+        fn ir_name() -> &'static str {
+            "<PARSER TEST IR>"
+        }
+
         fn span(&self) -> Span {
             DUMMY_SPAN
         }
@@ -488,8 +539,8 @@ pub mod test {
     impl Object for StubToken {}
 
     impl Display for StubToken {
-        fn fmt(&self, _f: &mut std::fmt::Formatter) -> std::fmt::Result {
-            unimplemented!()
+        fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+            write!(f, "(test token)")
         }
     }
 

tamer/src/parse/state/transition.rs

@@ -146,6 +146,21 @@ pub(in super::super) enum TransitionData<S: ParseState> {
     Dead(Lookahead<S::Token>),
 }
 
+impl<S: ParseState> TransitionData<S> {
+    /// Reference to the token of lookahead,
+    ///   if any.
+    #[cfg(test)]
+    pub(in super::super) fn lookahead_ref(
+        &self,
+    ) -> Option<&Lookahead<S::Token>> {
+        match self {
+            TransitionData::Dead(ref la)
+            | TransitionData::Result(_, Some(ref la)) => Some(la),
+            _ => None,
+        }
+    }
+}
+
 /// A verb denoting a state transition.
 ///
 /// This is typically instantiated directly by a [`ParseState`] to perform a

tamer/src/xir.rs

@@ -671,6 +671,10 @@ impl Display for Token {
 }
 
 impl crate::parse::Token for Token {
+    fn ir_name() -> &'static str {
+        "XIR"
+    }
+
     /// Retrieve the [`Span`] associated with a given [`Token`].
     ///
     /// Every token has an associated span.

tamer/src/xir/attr.rs

@@ -160,6 +160,12 @@ impl Attr {
 }
 
 impl Token for Attr {
+    fn ir_name() -> &'static str {
+        // This may be used by multiple things,
+        //   but it's primarily used by XIRF.
+        "XIRF"
+    }
+
     fn span(&self) -> Span {
         match self {
             Attr(.., attr_span) => attr_span.span(),

tamer/src/xir/flat.rs

@@ -117,6 +117,10 @@ pub enum XirfToken {
 }
 
 impl Token for XirfToken {
+    fn ir_name() -> &'static str {
+        "XIRF"
+    }
+
     fn span(&self) -> Span {
         use XirfToken::*;