tamer: xir::tree::parse: Use new Parsed::Done variant over None

This removes Option from ParseState, as mentioned in previous commits.

This is ideal because it not only removes a layer of abstraction, but also
makes the intent very clear; the use of None was too tied to the concept of
an Iterator, which is the concern of Parser, _not_ ParseState.

This is now similar to tree::Parsed, which will help with that refactoring
shortly.

The Done variant is not accessible outside of Parser, since it always
coverts it to None (to halt iteration); given that, we should have another
public-facing type, as was also mentioned in a previous commit.

DEV-11268

tamer/src/xir/tree.rs

@@ -1060,7 +1060,7 @@ pub fn attr_parser_from<'a>(
 
     AttrParserState::parser(toks).filter_map(|parsed| match parsed {
         Ok(Parsed::Object(attr)) => Some(Ok(attr)),
-        Ok(Parsed::Incomplete) => None,
+        Ok(Parsed::Incomplete | Parsed::Done) => None,
         Err(x) => Some(Err(x.into())),
     })
 }

tamer/src/xir/tree/attr/parse.rs

@@ -51,8 +51,8 @@ impl ParseState for AttrParserState {
     fn parse_token(&mut self, tok: Token) -> ParseStateResult<Self> {
         use AttrParserState::*;
 
-        Some(match (take(self), tok) {
-            (Empty, Token::AttrEnd(_)) => return None,
+        match (take(self), tok) {
+            (Empty, Token::AttrEnd(_)) => return Ok(Parsed::Done),
 
             (Empty, Token::AttrName(name, span)) => {
                 *self = Name(name, span);
@@ -70,7 +70,7 @@ impl ParseState for AttrParserState {
                 *self = Name(name, nspan);
                 Err(AttrParseError::AttrValueExpected(name, nspan, invalid))
             }
-        })
+        }
     }
 
     #[inline]
@@ -142,7 +142,7 @@ mod test {
 
         // Fail immediately.
         assert_eq!(
-            Some(Err(AttrParseError::AttrNameExpected(tok.clone()))),
+            Err(AttrParseError::AttrNameExpected(tok.clone())),
             sut.parse_token(tok)
         );
 
@@ -163,14 +163,14 @@ mod test {
         //   and so we are awaiting a value.
         assert_eq!(
             sut.parse_token(Token::AttrName(attr, *S)),
-            Some(Ok(Parsed::Incomplete))
+            Ok(Parsed::Incomplete)
         );
 
         // Once we have a value,
         //   an Attr can be emitted.
         assert_eq!(
             sut.parse_token(Token::AttrValue(val, *S2)),
-            Some(Ok(Parsed::Object(expected)))
+            Ok(Parsed::Object(expected))
         );
     }
 
@@ -184,17 +184,17 @@ mod test {
         //   the token stream.
         assert_eq!(
             sut.parse_token(Token::AttrName(attr, *S)),
-            Some(Ok(Parsed::Incomplete))
+            Ok(Parsed::Incomplete)
         );
 
         // But we provide something else unexpected.
         assert_eq!(
             sut.parse_token(Token::AttrEnd(*S2)),
-            Some(Err(AttrParseError::AttrValueExpected(
+            Err(AttrParseError::AttrValueExpected(
                 attr,
                 *S,
                 Token::AttrEnd(*S2)
-            )))
+            ))
         );
 
         // We should not be in an accepting state,
@@ -210,7 +210,7 @@ mod test {
         let expected = Attr::new(attr, recover, (*S, *S2));
         assert_eq!(
             sut.parse_token(Token::AttrValue(recover, *S2)),
-            Some(Ok(Parsed::Object(expected)))
+            Ok(Parsed::Object(expected))
         );
 
         // Finally, we should now be in an accepting state.
@@ -221,7 +221,7 @@ mod test {
     fn yields_none_on_attr_end() {
         let mut sut = AttrParserState::default();
 
-        assert_eq!(sut.parse_token(Token::AttrEnd(*S)), None);
+        assert_eq!(sut.parse_token(Token::AttrEnd(*S)), Ok(Parsed::Done));
         assert!(sut.is_accepting());
     }
 }

tamer/src/xir/tree/parse.rs

@@ -95,7 +95,7 @@ pub trait ParseState: Default {
 
 /// Result of applying a [`Token`] to a [`ParseState`].
 pub type ParseStateResult<S> =
-    Option<Result<Parsed<<S as ParseState>::Object>, <S as ParseState>::Error>>;
+    Result<Parsed<<S as ParseState>::Object>, <S as ParseState>::Error>;
 
 /// A streaming parser defined by a [`ParseState`] with exclusive
 ///   mutable access to an underlying [`TokenStream`].
@@ -166,9 +166,10 @@ impl<'a, S: ParseState, I: TokenStream> Iterator for Parser<'a, S, I> {
                 //   reporting in case we encounter an EOF.
                 self.last_span = Some(tok.span());
 
-                self.state
-                    .parse_token(tok)
-                    .map(|parsed| parsed.map_err(ParseError::from))
+                match self.state.parse_token(tok) {
+                    Ok(Parsed::Done) => None,
+                    parsed => Some(parsed.map_err(ParseError::from)),
+                }
             }
         }
     }
@@ -249,10 +250,25 @@ impl<'a, S: ParseState, I: TokenStream> From<&'a mut I> for Parser<'a, S, I> {
     }
 }
 
+/// Result of a parsing operation.
 #[derive(Debug, PartialEq, Eq)]
 pub enum Parsed<T> {
+    /// Additional tokens are needed to complete parsing of the next object.
     Incomplete,
+
+    /// Parsing of an object is complete.
+    ///
+    /// This does not indicate that the parser is complete,
+    ///   as more objects may be able to be emitted;
+    ///     see [`Parsed::Done`].
     Object(T),
+
+    /// Parsing is complete.
+    ///
+    /// This should cause an iterator to yield [`None`].
+    /// If a parser is a combination of multiple [`ParserState`]s,
+    ///   this should transition to the next appropriate state.
+    Done,
 }
 
 #[cfg(test)]
@@ -284,12 +300,12 @@ pub mod test {
                     *self = Self::Done;
                 }
                 Token::Close(..) => {
-                    return Some(Err(EchoStateError::InnerError(tok)))
+                    return Err(EchoStateError::InnerError(tok))
                 }
                 _ => {}
             }
 
-            Some(Ok(Parsed::Object(tok)))
+            Ok(Parsed::Object(tok))
         }
 
         fn is_accepting(&self) -> bool {