tamer: Begin XIR-based xmlo reader impl

There isn't a whole lot here, but there is additional work needed in various
places to support upcoming changes and so I want to get this commited to
ease the cognitive burden of what I have thusfar.  And to stop stashing.  We
have a feature flag for a reason.

DEV-10863

tamer/src/ir/xir.rs

@@ -83,6 +83,23 @@ pub mod reader;
 pub mod tree;
 pub mod writer;
 
+/// An infallible [`Token`] stream.
+///
+/// If the token stream originates from an operation that could potentially
+///   fail and ought to be propagated,
+///     use [`TokenResultStream`].
+///
+/// The name "stream" in place of "iterator" is intended to convey that this
+///   type is expected to be processed in real-time as a stream,
+///     not read into memory.
+pub trait TokenStream = Iterator<Item = Token>;
+
+/// A [`Token`] stream that may encounter errors during parsing.
+///
+/// If the stream cannot fail,
+///   consider using [`TokenStream`].
+pub trait TokenResultStream = Iterator<Item = Result<Token, Error>>;
+
 /// A static symbol that can be safely converted into a [`QName`] without
 ///   any checks.
 ///

tamer/src/ir/xir/reader.rs

@@ -224,5 +224,11 @@ impl<B: BufRead> Iterator for XmlXirReader<B> {
     }
 }
 
+impl<B: BufRead> From<B> for XmlXirReader<B> {
+    fn from(reader: B) -> Self {
+        Self::new(reader)
+    }
+}
+
 #[cfg(test)]
 mod test;

tamer/src/ir/xir/tree.rs

@@ -191,7 +191,7 @@
 //! For more information,
 //!   see [`AttrParts`].
 
-use super::{AttrValue, QName, Text, Token};
+use super::{AttrValue, QName, Text, Token, TokenResultStream};
 use crate::span::Span;
 use std::{fmt::Display, mem::take};
 

tamer/src/iter.rs

@@ -26,7 +26,7 @@
 //! ====================
 //! Iterators that can fail,
 //!   such as XIR's
-//!     [`TokenResultIterator`](crate::ir::xir::TokenResultIterator),
+//!     [`TokenResultStream`](crate::ir::xir::TokenResultStream),
 //!   can be confounding and difficult to work with because
 //!     [`Iterator::next`] wraps the [`Result`] within an [`Option`].
 //! Further,
@@ -58,8 +58,8 @@
 //!
 //! Put simply: we take an `Iterator<Item = Result<T, E>` and produce an
 //!   `Iterator<Item = T>`,
-//!     so that consumers of `T` needn't know or care that `T` we could fail
-//!     to produce a `T`.
+//!     so that consumers of `T` needn't know or care that we could fail to
+//!     produce a `T`.
 //!
 //! This iterator is constructed using either [`with_iter_while_ok`] or
 //!   [`into_iter_while_ok`],

tamer/src/ld/poc.rs

@@ -174,9 +174,24 @@ fn load_xmlo<'a, P: AsRef<Path>>(
 
     let (path, file) = cfile.into();
 
-    let xmlo: XmloReader<_> = file.into();
+    let mut state = {
+        #[cfg(not(feature = "wip-xmlo-xir-reader"))]
+        {
+            let xmlo: XmloReader<_> = file.into();
+            depgraph.import_xmlo(xmlo, state)?
+        }
 
-    let mut state = depgraph.import_xmlo(xmlo, state)?;
+        #[cfg(feature = "wip-xmlo-xir-reader")]
+        {
+            use crate::ir::xir::reader::XmlXirReader;
+            use crate::iter::into_iter_while_ok;
+
+            into_iter_while_ok(XmlXirReader::from(file), |toks| {
+                let xmlo: XmloReader<_> = toks.into();
+                depgraph.import_xmlo(xmlo, state)
+            })??
+        }
+    };
 
     let mut dir: PathBuf = path.clone();
     dir.pop();

tamer/src/lib.rs

@@ -33,6 +33,9 @@
 //   simply replace each alias reference with its definition,
 //   or possibly write a trait with a `Self` bound.
 #![feature(trait_alias)]
+// Can be replaced with `assert!(matches!(...))`,
+//   but at a loss of a better error message.
+#![feature(assert_matches)]
 // We build docs for private items.
 #![allow(rustdoc::private_intra_doc_links)]
 

tamer/src/obj/xmlo/reader.rs

@@ -37,41 +37,60 @@ pub use new::XmloReader;
 ///   potentially fail in error.
 pub type XmloResult<T> = Result<T, XmloError>;
 
-/// Re-implementation of `XmloReader` using [`XmlXirReader`].
-///
-/// This module will be merged into [`super`] once complete;
-///   it exists to make feature-flagging less confusing and error-prone.
 #[cfg(feature = "wip-xmlo-xir-reader")]
 mod new {
-    use super::{XmloEvent, XmloResult};
-    use crate::ir::xir::reader::Result as ReaderResult;
-    use crate::ir::xir::reader::XmlXirReader;
-    use crate::ir::xir::Token;
-    use std::io::BufRead;
+    //! Re-implementation of `XmloReader` using a [`TokenStream`].
+    //!
+    //! This module will be merged into [`super`] once complete;
+    //!   it exists to make feature-flagging less confusing and error-prone.
 
-    #[cfg(feature = "wip-xmlo-xir-reader")]
-    pub struct XmloReader<I: Iterator<Item = ReaderResult<Token>>> {
-        _reader: I,
+    use super::{XmloError, XmloEvent, XmloResult};
+    use crate::ir::xir::{Token, TokenStream};
+    use crate::sym::st::*;
+
+    qname_const! {
+        QN_LV_PACKAGE: L_LV:L_PACKAGE,
+        QN_PACKAGE: :L_PACKAGE,
+    }
+
+    pub struct XmloReader<I: TokenStream> {
+        reader: I,
+        seen_root: bool,
     }
 
-    #[cfg(feature = "wip-xmlo-xir-reader")]
     impl<I> XmloReader<I>
     where
-        I: Iterator<Item = ReaderResult<Token>>,
+        I: TokenStream,
     {
         pub fn from_reader(reader: I) -> Self {
-            Self { _reader: reader }
+            Self {
+                reader,
+                seen_root: false,
+            }
         }
 
         pub fn read_event(&mut self) -> XmloResult<XmloEvent> {
-            todo!("XmloReader::read_event")
+            let token = self.reader.next().ok_or(XmloError::UnexpectedEof)?;
+
+            if !self.seen_root {
+                match token {
+                    Token::Open(QN_LV_PACKAGE | QN_PACKAGE, _) => {
+                        //self.seen_root = true;
+                    }
+
+                    _ => return Err(XmloError::UnexpectedRoot),
+                }
+            }
+
+            match token {
+                todo => todo!("read_event: {:?}", todo),
+            }
         }
     }
 
-    #[cfg(feature = "wip-xmlo-xir-reader")]
     impl<I> Iterator for XmloReader<I>
     where
-        I: Iterator<Item = ReaderResult<Token>>,
+        I: TokenStream,
     {
         type Item = XmloResult<XmloEvent>;
 
@@ -80,13 +99,12 @@ mod new {
         }
     }
 
-    #[cfg(feature = "wip-xmlo-xir-reader")]
-    impl<B> From<B> for XmloReader<XmlXirReader<B>>
+    impl<I> From<I> for XmloReader<I>
     where
-        B: BufRead,
+        I: TokenStream,
     {
-        fn from(buf: B) -> Self {
-            Self::from_reader(XmlXirReader::new(buf))
+        fn from(toks: I) -> Self {
+            Self::from_reader(toks)
         }
     }
 }
@@ -148,7 +166,7 @@ pub enum XmloEvent {
 /// TODO: These errors provide no context (byte offset).
 #[derive(Debug, PartialEq)]
 pub enum XmloError {
-    /// XML parsing error.
+    /// XML parsing error (legacy, quick-xml).
     XmlError(XmlError),
     /// The root node was not an `lv:package`.
     UnexpectedRoot,
@@ -171,6 +189,8 @@ pub enum XmloError {
     UnassociatedFragment,
     /// A `preproc:fragment` element was found, but is missing `text()`.
     MissingFragmentText(SymbolId),
+    /// Token stream ended unexpectedly.
+    UnexpectedEof,
 }
 
 impl From<InnerXmlError> for XmloError {
@@ -182,42 +202,43 @@ impl From<InnerXmlError> for XmloError {
 impl Display for XmloError {
     fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
         match self {
-            XmloError::XmlError(e) => e.fmt(fmt),
-            XmloError::UnexpectedRoot => {
+            Self::XmlError(e) => e.fmt(fmt),
+            Self::UnexpectedRoot => {
                 write!(fmt, "unexpected package root (is this a package?)")
             }
-            XmloError::UnassociatedSym => write!(
+            Self::UnassociatedSym => write!(
                 fmt,
                 "unassociated symbol table entry: preproc:sym/@name missing"
             ),
-            XmloError::InvalidType(ty) => {
+            Self::InvalidType(ty) => {
                 write!(fmt, "invalid preproc:sym/@type `{}`", ty)
             }
-            XmloError::InvalidDtype(dtype) => {
+            Self::InvalidDtype(dtype) => {
                 write!(fmt, "invalid preproc:sym/@dtype `{}`", dtype)
             }
-            XmloError::InvalidDim(dim) => {
+            Self::InvalidDim(dim) => {
                 write!(fmt, "invalid preproc:sym/@dim `{}`", dim)
             }
-            XmloError::InvalidMapFrom(msg) => {
+            Self::InvalidMapFrom(msg) => {
                 write!(fmt, "invalid preproc:sym[@type=\"map\"]: {}", msg)
             }
-            XmloError::UnassociatedSymDep => write!(
+            Self::UnassociatedSymDep => write!(
                 fmt,
                 "unassociated dependency list: preproc:sym-dep/@name missing"
             ),
-            XmloError::MalformedSymRef(msg) => {
+            Self::MalformedSymRef(msg) => {
                 write!(fmt, "malformed dependency ref: {}", msg)
             }
-            XmloError::UnassociatedFragment => write!(
+            Self::UnassociatedFragment => write!(
                 fmt,
                 "unassociated fragment: preproc:fragment/@id missing"
             ),
-            XmloError::MissingFragmentText(symname) => write!(
+            Self::MissingFragmentText(symname) => write!(
                 fmt,
                 "fragment found, but missing text for symbol `{}`",
                 symname,
             ),
+            Self::UnexpectedEof => write!(fmt, "unexpected EOF"),
         }
     }
 }
@@ -230,3 +251,7 @@ impl std::error::Error for XmloError {
         }
     }
 }
+
+#[cfg(feature = "wip-xmlo-xir-reader")]
+#[cfg(test)]
+mod test;

tamer/src/obj/xmlo/reader/quickxml/test.rs

@@ -25,6 +25,8 @@ use crate::test::quick_xml::*;
 
 type Sut<B> = XmloReader<B>;
 
+// Tests marked with "DONE" have been migrated to `super::test`.
+
 macro_rules! xmlo_tests {
     ($(fn $fn:ident($sut:ident) $body:block)*) => {
         $(
@@ -53,6 +55,7 @@ xmlo_tests! {
         assert_eq!(Some(false), sut.reader.check_end);
     }
 
+    // DONE
     fn proxies_xml_failures(sut) {
         sut.reader.next_event =
             Some(Box::new(|_, _| Err(InnerXmlError::UnexpectedEof("test".into()))));
@@ -77,6 +80,7 @@ xmlo_tests! {
         }
     }
 
+    // DONE
     fn fails_on_invalid_root(sut) {
         // xmlo_tests macro sets this for us, so we need to clear it to
         // be able to perform the check

tamer/src/obj/xmlo/reader/test.rs

@@ -0,0 +1,44 @@
+// Tests xmlo object file reader
+//
+//  Copyright (C) 2014-2021 Ryan Specialty Group, LLC.
+//
+//  This file is part of TAME.
+//
+//  This program is free software: you can redistribute it and/or modify
+//  it under the terms of the GNU General Public License as published by
+//  the Free Software Foundation, either version 3 of the License, or
+//  (at your option) any later version.
+//
+//  This program is distributed in the hope that it will be useful,
+//  but WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+//  GNU General Public License for more details.
+//
+//  You should have received a copy of the GNU General Public License
+//  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::assert_matches::assert_matches;
+
+use super::*;
+use crate::{convert::ExpectInto, ir::xir::Token, span::DUMMY_SPAN};
+
+type Sut<B> = XmloReader<B>;
+
+#[test]
+fn fail_unexpected_eof() {
+    let mut sut = Sut::from_reader([].into_iter());
+    assert_matches!(sut.next(), Some(Err(XmloError::UnexpectedEof)));
+}
+
+#[test]
+fn fails_on_invalid_root() {
+    let mut sut = Sut::from_reader(
+        [Token::Open(
+            "not-a-valid-package-node".unwrap_into(),
+            DUMMY_SPAN,
+        )]
+        .into_iter(),
+    );
+
+    assert_matches!(sut.next(), Some(Err(XmloError::UnexpectedRoot)));
+}

tamer/src/sym/prefill.rs

@@ -449,6 +449,7 @@ pub mod st {
         L_INTEGER: cid "integer",
         L_L: cid "l",
         L_LPARAM: cid "lparam",
+        L_LV: cid "lv",
         L_MAP: cid "map",
         L_MAP_EXEC: tid "map-exec",
         L_MAP_FROM: tid "map-from",