tamer: parse: SP::Token: From<Self::Token>

Of course I would run into integration issues.  My foresight is lacking.

The purpose of this is to allow for type narrowing before passing data to a
more specialized ParseState, so that the other ParseState doesn't need to
concern itself with the entire domain of inputs that it doesn't need, and
repeat unnecessary narrowing.

For example, consider XIRF: it has an `Attr` variant, which holds an `Attr`
object.  We'll want to desugar that object.  It does not make sense to
require that the desugaring process accept `XirfToken` when we've already
narrowed it to an `Attr`---we should accept an Attr.

However, we run into a problem immediately: what happens with tokens that
bubble back up due to lookahead or errors?  Those tokens need to be
converted _back_ (widened).  Fortunately, widening is a much easier process
than narrowing---we can simply use `From`, as we do today so many other
places.

So, this still keeps the onus of narrowing on the caller, but for now that
seems most appropriate.  I suspect Rust would optimize away duplicate
checks, but that still leaves the maintenance concern---the two narrowings
could get out of sync, and that's not acceptable.

Unfortunately, this is just one of the problems with integration...

DEV-13156

tamer/src/parse/state.rs

@@ -416,7 +416,7 @@ where
 
         match data {
             TransitionData::Dead(Lookahead(lookahead)) => {
-                dead(env).incomplete().with_lookahead(lookahead)
+                dead(env).incomplete().with_lookahead(lookahead.into())
             }
 
             // Consume object and allow processing as part of state
@@ -424,7 +424,10 @@ where
             TransitionData::Result(Ok(Obj(obj)), lookahead) => {
                 TransitionResult(
                     into(newst, Some(obj), env),
-                    TransitionData::Result(Ok(Incomplete), lookahead),
+                    TransitionData::Result(
+                        Ok(Incomplete),
+                        lookahead.map(Lookahead::inner_into),
+                    ),
                 )
             }
 
@@ -435,7 +438,7 @@ where
                         Ok(_) => Ok(Incomplete),
                         Err(e) => Err(e.into()),
                     },
-                    lookahead,
+                    lookahead.map(Lookahead::inner_into),
                 ),
             ),
         }
@@ -474,7 +477,7 @@ pub trait StitchableParseState<SP: ParseState> =
 
 pub trait PartiallyStitchableParseState<SP: ParseState> = ClosedParseState
 where
-    SP: ParseState<Token = <Self as ParseState>::Token>,
+    <SP as ParseState>::Token: From<<Self as ParseState>::Token>,
     <Self as ParseState>::Error: Into<<SP as ParseState>::Error>;
 
 pub mod context {

tamer/src/parse/state/transition.rs

@@ -187,8 +187,11 @@ impl<S: ParseState> TransitionResult<S> {
         S: PartiallyStitchableParseState<SB>,
     {
         self.branch_dead_la(
-            |st, Lookahead(la)| fdead(st).with_lookahead(la),
-            |st, result, la| falive(st, result).maybe_with_lookahead(la),
+            |st, Lookahead(la)| fdead(st).with_lookahead(la.into()),
+            |st, result, la| {
+                falive(st, result)
+                    .maybe_with_lookahead(la.map(Lookahead::inner_into))
+            },
         )
     }
 
@@ -252,11 +255,13 @@ impl<S: ParseState> TransitionResult<S> {
             //   `PartiallyStitchableParseState`,
             //     and `into_inner` requires being able to convert the inner
             //     object that we handled above.
-            Result(Ok(Incomplete), la) => {
-                fother(st).incomplete().maybe_with_lookahead(la)
-            }
-            Result(Err(e), la) => fother(st).err(e).maybe_with_lookahead(la),
-            Dead(Lookahead(la)) => fother(st).dead(la),
+            Result(Ok(Incomplete), la) => fother(st)
+                .incomplete()
+                .maybe_with_lookahead(la.map(Lookahead::inner_into)),
+            Result(Err(e), la) => fother(st)
+                .err(e)
+                .maybe_with_lookahead(la.map(Lookahead::inner_into)),
+            Dead(Lookahead(la)) => fother(st).dead(la.into()),
         }
     }
 }
@@ -287,6 +292,15 @@ impl<T: Token> Lookahead<T> {
 
         diagnostic_panic!(desc, "{msg}",)
     }
+
+    pub fn inner_into<U: Token>(self) -> Lookahead<U>
+    where
+        T: Into<U>,
+    {
+        match self {
+            Self(tok) => Lookahead(tok.into()),
+        }
+    }
 }
 
 /// Information about the state transition.
@@ -431,7 +445,7 @@ impl<S: ParseState> TransitionData<S> {
         use TransitionData::*;
 
         match self {
-            Dead(la) => Dead(la),
+            Dead(la) => Dead(la.inner_into()),
             Result(result, la) => Result(
                 match result {
                     Ok(status) => Ok(status.inner_into()),
@@ -440,7 +454,7 @@ impl<S: ParseState> TransitionData<S> {
                     //     (which will be the same type if SB is closed).
                     Err(e) => Err(e.into().into()),
                 },
-                la,
+                la.map(Lookahead::inner_into),
             ),
         }
     }

tamer/src/parse/util/expand.rs

@@ -111,9 +111,9 @@ pub trait StitchExpansion: ClosedParseState {
 
         self.parse_token(tok, ctx.as_mut()).branch_obj_la(
             |st, obj, la| match (obj, la) {
-                (Expanded(obj), la) => {
-                    into(st).ok(obj).maybe_with_lookahead(la)
-                }
+                (Expanded(obj), la) => into(st)
+                    .ok(obj)
+                    .maybe_with_lookahead(la.map(Lookahead::inner_into)),
 
                 (DoneExpanding(tok), la) => {
                     // Uphold parser lookahead invariant.
@@ -133,7 +133,7 @@ pub trait StitchExpansion: ClosedParseState {
                             Expansion::DoneExpanding",
                     );
 
-                    done(st, tok).into_super()
+                    done(st, tok.into()).into_super()
                 }
             },
             &into,