tamer: Refactor asg_builder into obj::xmlo::lower and asg::air

This finally uses `parse` all the way up to aggregation into the ASG, as can
be seen by the mess in `poc`.  This will be further simplified---I just need
to get this committed so that I can mentally get it off my plate.  I've been
separating this commit into smaller commits, but there's a point where it's
just not worth the effort anymore.  I don't like making large changes such
as this one.

There is still work to do here.  First, it's worth re-mentioning that
`poc` means "proof-of-concept", and represents things that still need a
proper home/abstraction.

Secondly, `poc` is retrieving the context of two parsers---`LowerContext`
and `Asg`.  The latter is desirable, since it's the final aggregation point,
but the former needs to be eliminated; in particular, packages need to be
worked into the ASG so that `found` can be removed.

Recursively loading `xmlo` files still happens in `poc`, but the compiler
will need this as well.  Once packages are on the ASG, along with their
state, that responsibility can be generalized as well.

That will then simplify lowering even further, to the point where hopefully
everything has the same shape (once final aggregation has an abstraction),
after which we can then create a final abstraction to concisely stitch
everything together.  Right now, Rust isn't able to infer `S` for
`Lower<S, LS>`, which is unfortunate, but we'll be able to help it along
with a more explicit abstraction.

DEV-11864

tamer/src/asg/air.rs

@@ -0,0 +1,419 @@
+// ASG IR
+//
+//  Copyright (C) 2014-2022 Ryan Specialty Group, LLC.
+//
+//  This file is part of TAME.
+//
+//  This program is free software: you can redistribute it and/or modify
+//  it under the terms of the GNU General Public License as published by
+//  the Free Software Foundation, either version 3 of the License, or
+//  (at your option) any later version.
+//
+//  This program is distributed in the hope that it will be useful,
+//  but WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+//  GNU General Public License for more details.
+//
+//  You should have received a copy of the GNU General Public License
+//  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use super::{Asg, AsgError, FragmentText, IdentKind, Source};
+use crate::{
+    parse::{self, ParseState, Token, Transition, Transitionable},
+    span::UNKNOWN_SPAN,
+    sym::SymbolId,
+};
+use std::fmt::{Debug, Display};
+
+///! Intermediate representation for construction of the
+///!   [abstract semantic graph (ASG)](super) (AIR).
+///!
+///! AIR serves as an abstraction layer between higher-level parsers and the
+///!   aggregate ASG.
+///! It allows parsers to operate as a raw stream of data without having to
+///!   worry about ownership of or references to the ASG,
+///!     and allows for multiple such parsers to be joined.
+///!
+///! AIR is _not_ intended to replace the API of the ASG---it
+///!   is intended as a termination point for the parsing pipeline,
+///!     and as such implements a subset of the ASG's API that is suitable
+///!     for aggregating raw data from source and object files.
+///! Given that it does so little and is so close to the [`Asg`] API,
+///!   one might say that the abstraction is as light as air,
+///!   but that would surely result in face-palming and so we're not going
+///!     air such cringeworthy dad jokes here.
+
+pub type IdentSym = SymbolId;
+pub type DepSym = SymbolId;
+
+/// AIR token.
+///
+/// These tokens mimic a public API for the ASG,
+///   and allow parsers to be completely decoupled from the ASG object that
+///   they will eventually aggregate data into.
+///
+/// This IR is not intended to perform sophisticated manipulation of the
+///   ASG---it
+///     is intended to perform initial aggregation as part of a parsing
+///     phase,
+///       populating the ASG with the raw data that that will be
+///       subsequently analyzed and rewritten.
+#[derive(Debug, PartialEq)]
+pub enum AirToken {
+    /// Declare a resolved identifier.
+    IdentDecl(IdentSym, IdentKind, Source),
+    /// Declare an external identifier that must be resolved before linking.
+    IdentExternDecl(IdentSym, IdentKind, Source),
+    /// Declare that an identifier depends on another for its definition.
+    IdentDep(IdentSym, DepSym),
+    /// Associate a code fragment with an identifier.
+    IdentFragment(IdentSym, FragmentText),
+    /// Root an identifier.
+    IdentRoot(IdentSym),
+}
+
+impl Token for AirToken {
+    fn span(&self) -> crate::span::Span {
+        // TODO: This can be provided once the xmlo files store source
+        //   locations for symbols.
+        UNKNOWN_SPAN
+    }
+}
+
+impl parse::Object for AirToken {}
+
+impl Display for AirToken {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        use AirToken::*;
+
+        match self {
+            IdentDecl(sym, ..) => {
+                write!(f, "declaration of identifier `{sym}`")
+            }
+            IdentExternDecl(sym, ..) => {
+                write!(f, "declaration of external identifier `{sym}`")
+            }
+            IdentDep(isym, dsym) => write!(
+                f,
+                "declaration of identifier dependency `{isym} -> {dsym}`"
+            ),
+            IdentFragment(sym, ..) => {
+                write!(f, "identifier `{sym}` fragment text")
+            }
+            IdentRoot(sym) => write!(f, "rooting of identifier `{sym}`"),
+        }
+    }
+}
+
+/// AIR parser state.
+///
+/// This currently has no parsing state;
+///   all state is stored on the ASG itself,
+///     which is the parsing context.
+#[derive(Debug, PartialEq, Eq, Default)]
+pub enum AirState {
+    #[default]
+    Empty,
+}
+
+impl ParseState for AirState {
+    type Token = AirToken;
+    type Object = ();
+    type Error = AsgError;
+
+    /// Destination [`Asg`] that this parser lowers into.
+    ///
+    /// This ASG will be yielded by [`parse::Parser::finalize`].
+    type Context = Asg;
+
+    fn parse_token(
+        self,
+        tok: Self::Token,
+        asg: &mut Self::Context,
+    ) -> crate::parse::TransitionResult<Self> {
+        use AirState::*;
+        use AirToken::*;
+
+        match (self, tok) {
+            (Empty, IdentDecl(sym, kind, src)) => {
+                asg.declare(sym, kind, src).map(|_| ()).transition(Empty)
+            }
+
+            (Empty, IdentExternDecl(sym, kind, src)) => asg
+                .declare_extern(sym, kind, src)
+                .map(|_| ())
+                .transition(Empty),
+
+            (Empty, IdentDep(sym, dep)) => {
+                asg.add_dep_lookup(sym, dep);
+                Transition(Empty).incomplete()
+            }
+
+            (Empty, IdentFragment(sym, text)) => {
+                asg.set_fragment(sym, text).map(|_| ()).transition(Empty)
+            }
+
+            (Empty, IdentRoot(sym)) => {
+                let obj = asg.lookup_or_missing(sym);
+                asg.add_root(obj);
+
+                Transition(Empty).incomplete()
+            }
+        }
+    }
+
+    fn is_accepting(&self) -> bool {
+        *self == Self::Empty
+    }
+}
+
+impl Display for AirState {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        use AirState::*;
+
+        // This is not terribly useful beyond indicating which parser caused
+        //   an error.
+        match self {
+            Empty => write!(f, "awaiting AIR input for ASG"),
+        }
+    }
+}
+
+// These are tested as if they are another API directly atop of the ASG,
+//   since that is how they are used.
+#[cfg(test)]
+mod test {
+    use std::assert_matches::assert_matches;
+
+    use crate::{
+        asg::{Ident, Object},
+        parse::{ParseError, Parsed},
+    };
+
+    use super::*;
+
+    type Sut = AirState;
+
+    #[test]
+    fn ident_decl() {
+        let sym = "foo".into();
+        let kind = IdentKind::Tpl;
+        let src = Source {
+            src: Some("test/decl".into()),
+            ..Default::default()
+        };
+
+        let toks = vec![AirToken::IdentDecl(sym, kind.clone(), src.clone())]
+            .into_iter();
+        let mut sut = Sut::parse(toks);
+
+        assert_eq!(Some(Ok(Parsed::Incomplete)), sut.next());
+
+        let asg = sut.finalize().unwrap();
+
+        let ident_node =
+            asg.lookup(sym).expect("identifier was not added to graph");
+        let ident = asg.get(ident_node).unwrap();
+
+        assert_eq!(
+            Ok(ident),
+            Ident::declare(sym)
+                .resolve(kind.clone(), src.clone())
+                .map(Object::Ident)
+                .as_ref(),
+        );
+
+        // Re-instantiate the parser and test an error by attempting to
+        //   redeclare the same identifier.
+        let bad_toks = vec![AirToken::IdentDecl(sym, kind, src)].into_iter();
+        let mut sut = Sut::parse_with_context(bad_toks, asg);
+
+        assert_matches!(
+            sut.next(),
+            Some(Err(ParseError::StateError(AsgError::IdentTransition(_)))),
+        );
+    }
+
+    #[test]
+    fn ident_extern_decl() {
+        let sym = "foo".into();
+        let kind = IdentKind::Tpl;
+        let src = Source {
+            src: Some("test/decl-extern".into()),
+            ..Default::default()
+        };
+
+        let toks =
+            vec![AirToken::IdentExternDecl(sym, kind.clone(), src.clone())]
+                .into_iter();
+        let mut sut = Sut::parse(toks);
+
+        assert_eq!(Some(Ok(Parsed::Incomplete)), sut.next());
+
+        let asg = sut.finalize().unwrap();
+
+        let ident_node =
+            asg.lookup(sym).expect("identifier was not added to graph");
+        let ident = asg.get(ident_node).unwrap();
+
+        assert_eq!(
+            Ok(ident),
+            Ident::declare(sym)
+                .extern_(kind, src.clone())
+                .map(Object::Ident)
+                .as_ref(),
+        );
+
+        // Re-instantiate the parser and test an error by attempting to
+        //   redeclare with a different kind.
+        let different_kind = IdentKind::Meta;
+        let bad_toks =
+            vec![AirToken::IdentExternDecl(sym, different_kind, src)]
+                .into_iter();
+        let mut sut = Sut::parse_with_context(bad_toks, asg);
+
+        assert_matches!(
+            sut.next(),
+            Some(Err(ParseError::StateError(AsgError::IdentTransition(_)))),
+        );
+    }
+
+    #[test]
+    fn ident_dep() {
+        let ident = "foo".into();
+        let dep = "dep".into();
+
+        let toks = vec![AirToken::IdentDep(ident, dep)].into_iter();
+        let mut sut = Sut::parse(toks);
+
+        assert_eq!(Some(Ok(Parsed::Incomplete)), sut.next());
+
+        let asg = sut.finalize().unwrap();
+
+        let ident_node = asg
+            .lookup(ident)
+            .expect("identifier was not added to graph");
+        let dep_node = asg.lookup(dep).expect("dep was not added to graph");
+
+        assert!(asg.has_dep(ident_node, dep_node));
+    }
+
+    #[test]
+    fn ident_fragment() {
+        let sym = "frag".into();
+        let kind = IdentKind::Tpl;
+        let src = Source {
+            src: Some("test/frag".into()),
+            ..Default::default()
+        };
+        let frag = "fragment text".into();
+
+        let toks = vec![
+            // Identifier must be declared before it can be given a
+            //   fragment.
+            AirToken::IdentDecl(sym, kind.clone(), src.clone()),
+            AirToken::IdentFragment(sym, frag),
+        ]
+        .into_iter();
+
+        let mut sut = Sut::parse(toks);
+
+        assert_eq!(Some(Ok(Parsed::Incomplete)), sut.next()); // IdentDecl
+        assert_eq!(Some(Ok(Parsed::Incomplete)), sut.next()); // IdentFragment
+
+        let asg = sut.finalize().unwrap();
+
+        let ident_node =
+            asg.lookup(sym).expect("identifier was not added to graph");
+        let ident = asg.get(ident_node).unwrap();
+
+        assert_eq!(
+            Ok(ident),
+            Ident::declare(sym)
+                .resolve(kind.clone(), src.clone())
+                .and_then(|resolved| resolved.set_fragment(frag))
+                .map(Object::Ident)
+                .as_ref(),
+        );
+
+        // Re-instantiate the parser and test an error by attempting to
+        //   re-set the fragment.
+        let bad_toks = vec![AirToken::IdentFragment(sym, frag)].into_iter();
+        let mut sut = Sut::parse_with_context(bad_toks, asg);
+
+        assert_matches!(
+            sut.next(),
+            Some(Err(ParseError::StateError(AsgError::IdentTransition(_)))),
+        );
+    }
+
+    // Adding a root before the identifier exists should add a
+    //   `Ident::Missing`.
+    #[test]
+    fn ident_root_missing() {
+        let sym = "toroot".into();
+
+        let toks = vec![AirToken::IdentRoot(sym)].into_iter();
+        let mut sut = Sut::parse(toks);
+
+        assert_eq!(Some(Ok(Parsed::Incomplete)), sut.next());
+
+        let asg = sut.finalize().unwrap();
+
+        let ident_node = asg
+            .lookup(sym)
+            .expect("identifier was not added to the graph");
+        let ident = asg.get(ident_node).unwrap();
+
+        // The identifier did not previously exist,
+        //   and so a missing node is created as a placeholder.
+        assert_eq!(&Object::Ident(Ident::Missing(sym)), ident);
+
+        // And that missing identifier should be rooted.
+        assert!(asg.is_rooted(ident_node));
+    }
+
+    #[test]
+    fn ident_root_existing() {
+        let sym = "toroot".into();
+        let kind = IdentKind::Tpl;
+        let src = Source {
+            src: Some("test/root-existing".into()),
+            ..Default::default()
+        };
+
+        // Ensure that it won't auto-root based on the kind,
+        //   otherwise we won't be testing the right thing.
+        assert!(!kind.is_auto_root());
+
+        let toks = vec![
+            AirToken::IdentDecl(sym, kind.clone(), src.clone()),
+            AirToken::IdentRoot(sym),
+        ]
+        .into_iter();
+
+        let mut sut = Sut::parse(toks);
+
+        assert_eq!(Some(Ok(Parsed::Incomplete)), sut.next()); // IdentDecl
+        assert_eq!(Some(Ok(Parsed::Incomplete)), sut.next()); // IdentRoot
+
+        let asg = sut.finalize().unwrap();
+
+        let ident_node = asg
+            .lookup(sym)
+            .expect("identifier was not added to the graph");
+        let ident = asg.get(ident_node).unwrap();
+
+        // The previously-declared identifier...
+        assert_eq!(
+            Ok(ident),
+            Ident::declare(sym)
+                .resolve(kind.clone(), src.clone())
+                .map(Object::Ident)
+                .as_ref()
+        );
+
+        // ...should have been subsequently rooted.
+        assert!(asg.is_rooted(ident_node));
+    }
+}

tamer/src/asg/error.rs

@@ -24,24 +24,21 @@ use std::{
     fmt::{self, Display},
 };
 
+use crate::diagnose::{AnnotatedSpan, Diagnostic};
+
 use super::TransitionError;
 
 /// An error from an ASG operation.
 #[derive(Debug, PartialEq)]
 pub enum AsgError {
     /// An object could not change state in the manner requested.
-    ObjectTransition(TransitionError),
-    /// The node was not expected in the current context
-    UnexpectedNode(String),
+    IdentTransition(TransitionError),
 }
 
 impl Display for AsgError {
     fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
         match self {
-            Self::ObjectTransition(err) => Display::fmt(&err, fmt),
-            Self::UnexpectedNode(msg) => {
-                write!(fmt, "unexpected node: {}", msg)
-            }
+            Self::IdentTransition(err) => Display::fmt(&err, fmt),
         }
     }
 }
@@ -49,14 +46,20 @@ impl Display for AsgError {
 impl Error for AsgError {
     fn source(&self) -> Option<&(dyn Error + 'static)> {
         match self {
-            Self::ObjectTransition(err) => err.source(),
-            _ => None,
+            Self::IdentTransition(err) => err.source(),
         }
     }
 }
 
 impl From<TransitionError> for AsgError {
     fn from(err: TransitionError) -> Self {
-        Self::ObjectTransition(err)
+        Self::IdentTransition(err)
+    }
+}
+
+impl Diagnostic for AsgError {
+    fn describe(&self) -> Vec<AnnotatedSpan> {
+        // TODO: This won't be useful until we have spans.
+        vec![]
     }
 }

tamer/src/asg/graph.rs

@@ -67,7 +67,7 @@ type Ix = global::ProgSymSize;
 ///
 /// For more information,
 ///   see the [module-level documentation][self].
-#[derive(Debug, Default)]
+#[derive(Debug)]
 pub struct Asg {
     // TODO: private; see `ld::xmle::lower`.
     /// Directed graph on which objects are stored.
@@ -89,11 +89,18 @@ pub struct Asg {
     root_node: NodeIndex<Ix>,
 }
 
+impl Default for Asg {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl Asg {
     /// Create a new ASG.
     ///
     /// See also [`with_capacity`](Asg::with_capacity).
     pub fn new() -> Self {
+        // TODO: Determine a proper initial capacity.
         Self::with_capacity(0, 0)
     }
 
@@ -167,7 +174,7 @@ impl Asg {
     ///   reference to it.
     ///
     /// See [`Ident::declare`] for more information.
-    fn lookup_or_missing(&mut self, ident: SymbolId) -> ObjectRef {
+    pub(super) fn lookup_or_missing(&mut self, ident: SymbolId) -> ObjectRef {
         self.lookup(ident).unwrap_or_else(|| {
             let index = self.graph.add_node(Some(Ident::declare(ident).into()));
 
@@ -248,6 +255,14 @@ impl Asg {
             .add_edge(self.root_node, identi.into(), Default::default());
     }
 
+    /// Whether an object is rooted.
+    ///
+    /// See [`Asg::add_root`] for more information about roots.
+    #[cfg(test)]
+    pub(super) fn is_rooted(&self, identi: ObjectRef) -> bool {
+        self.graph.contains_edge(self.root_node, identi.into())
+    }
+
     /// Declare a concrete identifier.
     ///
     /// An identifier declaration is similar to a declaration in a header
@@ -594,7 +609,7 @@ mod test {
         let node = sut.declare(sym, IdentKind::Meta, src.clone())?;
         let result = sut.declare(sym, IdentKind::Meta, Source::default());
 
-        assert_matches!(result, Err(AsgError::ObjectTransition(..)));
+        assert_matches!(result, Err(AsgError::IdentTransition(..)));
 
         // The node should have been restored.
         assert_eq!(Some(&src), sut.get_ident(node).unwrap().src());
@@ -639,7 +654,7 @@ mod test {
         let result =
             sut.declare_extern(sym, IdentKind::Worksheet, Source::default());
 
-        assert_matches!(result, Err(AsgError::ObjectTransition(..)));
+        assert_matches!(result, Err(AsgError::IdentTransition(..)));
 
         // The node should have been restored.
         assert_eq!(Some(&src), sut.get_ident(node).unwrap().src());
@@ -701,7 +716,7 @@ mod test {
         let obj = sut.get_ident(node).unwrap();
 
         assert_eq!(sym, obj.name());
-        assert_matches!(result, Err(AsgError::ObjectTransition(..)));
+        assert_matches!(result, Err(AsgError::IdentTransition(..)));
 
         Ok(())
     }

tamer/src/asg/mod.rs

@@ -68,6 +68,8 @@ mod graph;
 mod ident;
 mod object;
 
+pub mod air;
+
 pub use error::AsgError;
 pub use graph::{Asg, AsgResult, IndexType, ObjectRef};
 pub use ident::{

tamer/src/ld/poc.rs

@@ -26,7 +26,10 @@ use super::xmle::{
     XmleSections,
 };
 use crate::{
-    asg::{DefaultAsg, Ident, Object},
+    asg::{
+        air::{AirState, AirToken},
+        Asg, AsgError, DefaultAsg, Ident, Object,
+    },
     diagnose::{AnnotatedSpan, Diagnostic},
     fs::{
         Filesystem, FsCanonicalizer, PathFile, VisitOnceFile,
@@ -34,11 +37,8 @@ use crate::{
     },
     iter::into_iter_while_ok,
     ld::xmle::Sections,
-    obj::xmlo::{
-        AsgBuilder, AsgBuilderError, AsgBuilderState, XmloError, XmloReader,
-    },
-    parse::ParseError,
-    parse::{ParseState, Parsed},
+    obj::xmlo::{self, XmloError, XmloLowerError, XmloReader, XmloToken},
+    parse::{Lower, ParseError, ParseState, Parsed},
     sym::{GlobalSymbolResolve, SymbolId},
     xir::reader::XmlXirReader,
     xir::{
@@ -58,25 +58,23 @@ use std::{
 };
 
 type LinkerAsg = DefaultAsg;
-type LinkerAsgBuilderState = AsgBuilderState<FxBuildHasher>;
 
 pub fn xmle(package_path: &str, output: &str) -> Result<(), TameldError> {
     let mut fs = VisitOnceFilesystem::new();
-    let mut depgraph = LinkerAsg::with_capacity(65536, 65536);
     let escaper = DefaultEscaper::default();
 
-    let state = load_xmlo(
+    let (depgraph, state) = load_xmlo(
         package_path,
         &mut fs,
-        &mut depgraph,
+        LinkerAsg::with_capacity(65536, 65536),
         &escaper,
-        AsgBuilderState::new(),
+        xmlo::LowerContext::default(),
     )?;
 
-    let AsgBuilderState {
-        name,
+    let xmlo::LowerContext {
+        prog_name: name,
         relroot,
-        found: _,
+        ..
     } = state;
 
     let sorted = match sort(&depgraph, Sections::new()) {
@@ -121,15 +119,14 @@ pub fn xmle(package_path: &str, output: &str) -> Result<(), TameldError> {
 // TODO: This needs to be further generalized.
 pub fn graphml(package_path: &str, output: &str) -> Result<(), TameldError> {
     let mut fs = VisitOnceFilesystem::new();
-    let mut depgraph = LinkerAsg::with_capacity(65536, 65536);
     let escaper = DefaultEscaper::default();
 
-    let _ = load_xmlo(
+    let (depgraph, _) = load_xmlo(
         package_path,
         &mut fs,
-        &mut depgraph,
+        LinkerAsg::with_capacity(65536, 65536),
         &escaper,
-        AsgBuilderState::new(),
+        xmlo::LowerContext::default(),
     )?;
 
     // if we move away from petgraph, we will need to abstract this away
@@ -179,37 +176,55 @@ pub fn graphml(package_path: &str, output: &str) -> Result<(), TameldError> {
 fn load_xmlo<'a, P: AsRef<Path>, S: Escaper>(
     path_str: P,
     fs: &mut VisitOnceFilesystem<FsCanonicalizer, FxBuildHasher>,
-    depgraph: &mut LinkerAsg,
+    asg: Asg,
     escaper: &S,
-    state: LinkerAsgBuilderState,
-) -> Result<LinkerAsgBuilderState, TameldError> {
+    state: xmlo::LowerContext,
+) -> Result<(Asg, xmlo::LowerContext), TameldError> {
     let PathFile(path, file, ctx): PathFile<BufReader<fs::File>> =
         match fs.open(path_str)? {
             VisitOnceFile::FirstVisit(file) => file,
-            VisitOnceFile::Visited => return Ok(state),
+            VisitOnceFile::Visited => return Ok((asg, state)),
         };
 
     // TODO: This entire block is a WIP and will be incrementally
     //   abstracted away.
-    let mut state =
-        into_iter_while_ok(XmlXirReader::new(file, escaper, ctx), |toks| {
+    let (mut asg, mut state) = into_iter_while_ok::<_, _, _, TameldError, _>(
+        XmlXirReader::new(file, escaper, ctx),
+        |toks| {
             flat::State::<64>::parse(toks).lower_while_ok::<XmloReader, _, _>(
-                |xirf| {
-                    into_iter_while_ok(xirf, |xmlo_out| {
-                        // TODO: Transitionary---we do not want to filter.
-                        depgraph
-                            .import_xmlo(
-                                xmlo_out.filter_map(|parsed| match parsed {
-                                    Parsed::Incomplete => None,
-                                    Parsed::Object(obj) => Some(Ok(obj)),
-                                }),
-                                state,
-                            )
-                            .map_err(TameldError::from)
-                    })
+                |xmlo| {
+                    let mut iter = xmlo.scan(false, |st, rtok| {
+                        match st {
+                            true => None,
+                            false => {
+                                *st = matches!(rtok, Ok(Parsed::Object(XmloToken::Eoh(..))));
+                                Some(rtok)
+                            },
+                        }
+                    });
+
+                    Lower::<XmloReader, xmlo::LowerState>::lower_with_context_while_ok(
+                        &mut iter,
+                        state,
+                        |air| {
+                            let (_, asg) = Lower::<xmlo::LowerState, AirState>::lower_with_context_while_ok(
+                                    air,
+                                    asg,
+                                    |end| {
+                                        end.fold(
+                                            Result::<(), TameldError>::Ok(()),
+                                            |x, _| x,
+                                        )
+                                    },
+                                )?;
+
+                            Ok(asg)
+                        },
+                    )
                 },
             )
-        })?;
+        },
+    )?;
 
     let mut dir: PathBuf = path.clone();
     dir.pop();
@@ -222,10 +237,10 @@ fn load_xmlo<'a, P: AsRef<Path>, S: Escaper>(
         path_buf.push(str);
         path_buf.set_extension("xmlo");
 
-        state = load_xmlo(path_buf, fs, depgraph, escaper, state)?;
+        (asg, state) = load_xmlo(path_buf, fs, asg, escaper, state)?;
     }
 
-    Ok(state)
+    Ok((asg, state))
 }
 
 fn output_xmle<'a, X: XmleSections<'a>, S: Escaper>(
@@ -264,7 +279,8 @@ pub enum TameldError {
     XirError(XirError),
     XirfParseError(ParseError<XirToken, XirfError>),
     XmloParseError(ParseError<XirfToken, XmloError>),
-    AsgBuilderError(AsgBuilderError),
+    XmloLowerError(ParseError<XmloToken, XmloLowerError>),
+    AirLowerError(ParseError<AirToken, AsgError>),
     XirWriterError(XirWriterError),
     CycleError(Vec<Vec<SymbolId>>),
     Fmt(fmt::Error),
@@ -300,9 +316,15 @@ impl From<ParseError<XirToken, XirfError>> for TameldError {
     }
 }
 
-impl From<AsgBuilderError> for TameldError {
-    fn from(e: AsgBuilderError) -> Self {
-        Self::AsgBuilderError(e)
+impl From<ParseError<XmloToken, XmloLowerError>> for TameldError {
+    fn from(e: ParseError<XmloToken, XmloLowerError>) -> Self {
+        Self::XmloLowerError(e)
+    }
+}
+
+impl From<ParseError<AirToken, AsgError>> for TameldError {
+    fn from(e: ParseError<AirToken, AsgError>) -> Self {
+        Self::AirLowerError(e)
     }
 }
 
@@ -326,7 +348,8 @@ impl Display for TameldError {
             Self::XirError(e) => Display::fmt(e, f),
             Self::XirfParseError(e) => Display::fmt(e, f),
             Self::XmloParseError(e) => Display::fmt(e, f),
-            Self::AsgBuilderError(e) => Display::fmt(e, f),
+            Self::XmloLowerError(e) => Display::fmt(e, f),
+            Self::AirLowerError(e) => Display::fmt(e, f),
             Self::XirWriterError(e) => Display::fmt(e, f),
             Self::CycleError(cycles) => {
                 for cycle in cycles {
@@ -356,7 +379,8 @@ impl Error for TameldError {
             Self::XirError(e) => Some(e),
             Self::XirfParseError(e) => Some(e),
             Self::XmloParseError(e) => Some(e),
-            Self::AsgBuilderError(e) => Some(e),
+            Self::XmloLowerError(e) => Some(e),
+            Self::AirLowerError(e) => Some(e),
             Self::XirWriterError(e) => Some(e),
             Self::CycleError(..) => None,
             Self::Fmt(e) => Some(e),
@@ -370,6 +394,8 @@ impl Diagnostic for TameldError {
             Self::XirError(e) => e.describe(),
             Self::XirfParseError(e) => e.describe(),
             Self::XmloParseError(e) => e.describe(),
+            Self::XmloLowerError(e) => e.describe(),
+            Self::AirLowerError(e) => e.describe(),
 
             // TODO (will fall back to rendering just the error `Display`)
             _ => vec![],

tamer/src/obj/xmlo/lower.rs

@@ -0,0 +1,924 @@
+// Lower `xmlo` object file into the ASG
+//
+//  Copyright (C) 2014-2022 Ryan Specialty Group, LLC.
+//
+//  This file is part of TAME.
+//
+//  This program is free software: you can redistribute it and/or modify
+//  it under the terms of the GNU General Public License as published by
+//  the Free Software Foundation, either version 3 of the License, or
+//  (at your option) any later version.
+//
+//  This program is distributed in the hope that it will be useful,
+//  but WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+//  GNU General Public License for more details.
+//
+//  You should have received a copy of the GNU General Public License
+//  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Lower [`xmlo` IR](crate::obj::xmlo) into [AIR`](crate::asg::air).
+
+use std::{
+    error::Error,
+    fmt::{Debug, Display},
+};
+
+use fxhash::FxHashSet;
+
+use crate::{
+    asg::{air::AirToken, IdentKind, Source},
+    diagnose::{AnnotatedSpan, Diagnostic},
+    obj::xmlo::{SymAttrs, SymType},
+    parse::{ParseState, ParseStatus, Transition, Transitionable},
+    span::Span,
+    sym::SymbolId,
+};
+
+use super::XmloToken;
+
+/// Persistent `xmlo` lowering context to be shared among all `xmlo` files.
+///
+/// TODO: Continue refactoring this into [`LowerState`] and the ASG itself.
+#[derive(Debug)]
+pub struct LowerContext {
+    /// Relative paths to imported packages that have been discovered.
+    ///
+    /// The caller will use these to perform recursive loads.
+    /// This is contained within an [`Option`] so that the caller can `take`
+    ///   ownership over its contents.
+    pub found: Option<FxHashSet<SymbolId>>,
+
+    /// Program name once discovered.
+    ///
+    /// This will be set by the first package encountered.
+    pub prog_name: Option<SymbolId>,
+
+    /// Relative path to project root once discovered.
+    ///
+    /// This will be set by the first package encountered.
+    pub relroot: Option<SymbolId>,
+
+    /// Whether this is the first package encountered.
+    ///
+    /// This defaults to [`true`] and is updated to [`false`] at EOH.
+    first: bool,
+}
+
+impl Default for LowerContext {
+    fn default() -> Self {
+        Self {
+            found: None,
+            prog_name: None,
+            relroot: None,
+            first: true,
+        }
+    }
+}
+
+impl LowerContext {
+    /// Whether this is the first discovered package.
+    #[inline]
+    fn is_first(&self) -> bool {
+        self.first
+    }
+}
+
+type PackageName = SymbolId;
+
+/// State machine for lowering into the [`Asg`](crate::asg::Asg) via
+///   [`AirToken`].
+#[derive(Debug, PartialEq, Eq, Default)]
+pub enum LowerState {
+    #[default]
+    PackageExpected,
+    Package(PackageName, Span),
+    SymDep(PackageName, Span, SymbolId),
+    /// End of header (EOH) reached.
+    Done(Span),
+}
+
+impl ParseState for LowerState {
+    type Token = XmloToken;
+    type Object = AirToken;
+    type Error = XmloLowerError;
+
+    type Context = LowerContext;
+
+    fn parse_token(
+        self,
+        tok: Self::Token,
+        ctx: &mut Self::Context,
+    ) -> crate::parse::TransitionResult<Self> {
+        use LowerState::*;
+
+        match (self, tok) {
+            (PackageExpected, XmloToken::PkgName(name, span)) => {
+                if ctx.is_first() {
+                    ctx.prog_name = Some(name);
+                }
+
+                Transition(Package(name, span)).incomplete()
+            }
+
+            (st @ Package(..), XmloToken::PkgRootPath(relroot, _)) => {
+                if ctx.is_first() {
+                    ctx.relroot = Some(relroot);
+                }
+
+                Transition(st).incomplete()
+            }
+
+            // Eligibility classes are rooted as soon as they are
+            //   encountered on the root node,
+            //     which will result in a missing identifier until its
+            //     definition is encountered later within the same file.
+            // TODO: Let's remove the need for this special root handling
+            //   here.
+            (
+                Package(pkg_name, span),
+                XmloToken::PkgEligClassYields(pkg_elig, _),
+            ) => Transition(Package(pkg_name, span))
+                .ok(AirToken::IdentRoot(pkg_elig)),
+
+            (
+                st @ (PackageExpected | Package(..)),
+                XmloToken::PkgProgramFlag(_),
+            ) => {
+                // TODO: Unused
+                Transition(st).incomplete()
+            }
+
+            (
+                Package(pkg_name, span) | SymDep(pkg_name, span, ..),
+                XmloToken::SymDepStart(sym, _),
+            ) => Transition(SymDep(pkg_name, span, sym)).incomplete(),
+
+            (SymDep(pkg_name, span, sym), XmloToken::Symbol(dep_sym, _)) => {
+                Transition(SymDep(pkg_name, span, sym))
+                    .ok(AirToken::IdentDep(sym, dep_sym))
+            }
+
+            (
+                Package(pkg_name, span),
+                XmloToken::SymDecl(
+                    _sym,
+                    SymAttrs {
+                        src: Some(sym_src), ..
+                    },
+                    _span,
+                ),
+            ) => {
+                ctx.found.get_or_insert(Default::default()).insert(sym_src);
+                Transition(Package(pkg_name, span)).incomplete()
+            }
+
+            (
+                Package(pkg_name, span),
+                XmloToken::SymDecl(sym, attrs, _span),
+            ) => {
+                let extern_ = attrs.extern_;
+
+                // TODO: This attr/source separation is a mess,
+                //   the IR is a mess,
+                //   and spans are not retained.
+                (&attrs)
+                    .try_into()
+                    .and_then(|kindval| {
+                        let mut src: Source = attrs.into();
+
+                        // This used to come from SymAttrs in the old XmloReader.
+                        if src.pkg_name.is_none() {
+                            src.pkg_name.replace(pkg_name);
+                        }
+
+                        // Existing convention is to omit @src of local package
+                        // (in this case, the program being linked)
+                        if ctx.is_first() {
+                            src.pkg_name = None;
+                        }
+
+                        if extern_ {
+                            Ok(ParseStatus::Object(AirToken::IdentExternDecl(
+                                sym, kindval, src,
+                            )))
+                        } else {
+                            Ok(ParseStatus::Object(AirToken::IdentDecl(
+                                sym, kindval, src,
+                            )))
+                        }
+                    })
+                    .transition(Package(pkg_name, span))
+            }
+
+            (
+                Package(pkg_name, span) | SymDep(pkg_name, span, _),
+                XmloToken::Fragment(sym, text, _),
+            ) => Transition(Package(pkg_name, span))
+                .ok(AirToken::IdentFragment(sym, text)),
+
+            // We don't need to read any further than the end of the
+            //   header (symtable, sym-deps, fragments).
+            (Package(..) | SymDep(..), XmloToken::Eoh(span)) => {
+                // It's important to set this _after_ we're done processing,
+                //   otherwise our `first` checks above will be inaccurate.
+                ctx.first = false;
+
+                // Note that this uses `incomplete` because we have nothing
+                //   to yield,
+                //     but we are in fact done.
+                Transition(Done(span)).incomplete()
+            }
+
+            (st, tok) => Transition(st).dead(tok),
+        }
+    }
+
+    fn is_accepting(&self) -> bool {
+        matches!(*self, Self::Done(_))
+    }
+}
+
+impl Display for LowerState {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        use LowerState::*;
+
+        match self {
+            PackageExpected => write!(f, "expecting package definition"),
+            Package(name, _) => {
+                write!(f, "expecting package `/{name}` declarations")
+            }
+            SymDep(pkg_name, _, sym) => {
+                write!(f, "expecting dependency for symbol `/{pkg_name}/{sym}`")
+            }
+            Done(_) => write!(f, "done lowering xmlo into AIR"),
+        }
+    }
+}
+
+impl TryFrom<SymAttrs> for IdentKind {
+    type Error = XmloLowerError;
+
+    /// Attempt to raise [`SymAttrs`] into an [`IdentKind`].
+    ///
+    /// Certain [`IdentKind`] require that certain attributes be present,
+    ///   otherwise the conversion will fail.
+    fn try_from(attrs: SymAttrs) -> Result<Self, Self::Error> {
+        Self::try_from(&attrs)
+    }
+}
+
+impl TryFrom<&SymAttrs> for IdentKind {
+    type Error = XmloLowerError;
+
+    /// Attempt to raise [`SymAttrs`] into an [`IdentKind`].
+    ///
+    /// Certain [`IdentKind`] require that certain attributes be present,
+    ///   otherwise the conversion will fail.
+    fn try_from(attrs: &SymAttrs) -> Result<Self, Self::Error> {
+        let ty = attrs.ty.as_ref().ok_or(Self::Error::MissingType)?;
+
+        macro_rules! ident {
+            ($to:expr) => {
+                Ok($to)
+            };
+            ($to:expr, dim) => {
+                Ok($to(attrs.dim.ok_or(Self::Error::MissingDim)?))
+            };
+            ($to:expr, dtype) => {
+                Ok($to(attrs.dtype.ok_or(Self::Error::MissingDtype)?))
+            };
+            ($to:expr, dim, dtype) => {
+                Ok($to(
+                    attrs.dim.ok_or(Self::Error::MissingDim)?,
+                    attrs.dtype.ok_or(Self::Error::MissingDtype)?,
+                ))
+            };
+        }
+
+        match ty {
+            SymType::Cgen => ident!(Self::Cgen, dim),
+            SymType::Class => ident!(Self::Class, dim),
+            SymType::Const => ident!(Self::Const, dim, dtype),
+            SymType::Func => ident!(Self::Func, dim, dtype),
+            SymType::Gen => ident!(Self::Gen, dim, dtype),
+            SymType::Lparam => ident!(IdentKind::Lparam, dim, dtype),
+            SymType::Param => ident!(IdentKind::Param, dim, dtype),
+            SymType::Rate => ident!(IdentKind::Rate, dtype),
+            SymType::Tpl => ident!(IdentKind::Tpl),
+            SymType::Type => ident!(IdentKind::Type, dtype),
+            SymType::MapHead => ident!(IdentKind::MapHead),
+            SymType::Map => ident!(IdentKind::Map),
+            SymType::MapTail => ident!(IdentKind::MapTail),
+            SymType::RetMapHead => ident!(IdentKind::RetMapHead),
+            SymType::RetMap => ident!(IdentKind::RetMap),
+            SymType::RetMapTail => ident!(IdentKind::RetMapTail),
+            SymType::Meta => ident!(IdentKind::Meta),
+            SymType::Worksheet => ident!(IdentKind::Worksheet),
+        }
+    }
+}
+
+impl From<SymAttrs> for Source {
+    /// Raise Legacy IR [`SymAttrs`].
+    ///
+    /// This simply extracts a subset of fields from the source attributes.
+    fn from(attrs: SymAttrs) -> Self {
+        Source {
+            pkg_name: attrs.pkg_name,
+            src: attrs.src,
+            generated: attrs.generated,
+            parent: attrs.parent,
+            yields: attrs.yields,
+            desc: attrs.desc,
+            from: attrs.from,
+            virtual_: attrs.virtual_,
+            override_: attrs.override_,
+        }
+    }
+}
+
+/// Error populating graph with `xmlo`-derived data.
+///
+/// TODO: Spans are needed!
+#[derive(Debug, PartialEq)]
+pub enum XmloLowerError {
+    /// Symbol type was not provided.
+    MissingType,
+
+    /// Number of symbol dimensions were not provided.
+    MissingDim,
+
+    /// Symbol dtype was not provided.
+    MissingDtype,
+
+    /// Eligibility classification references unknown identifier.
+    ///
+    /// This is generated by the compiler and so should never happen.
+    /// (That's not to say that it won't, but it shouldn't.)
+    BadEligRef(SymbolId),
+}
+
+impl Display for XmloLowerError {
+    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match self {
+            Self::MissingType => write!(fmt, "missing symbol type"),
+            Self::MissingDim => write!(fmt, "missing dim"),
+            Self::MissingDtype => write!(fmt, "missing dtype"),
+            Self::BadEligRef(name) => write!(
+                fmt,
+                "internal error: package elig references nonexistant symbol `{}`",
+                name,
+            ),
+        }
+    }
+}
+
+impl Diagnostic for XmloLowerError {
+    fn describe(&self) -> Vec<AnnotatedSpan> {
+        use XmloLowerError::*;
+
+        match self {
+            // TODO: Missing spans!
+            MissingType | MissingDim | MissingDtype | BadEligRef(_) => vec![],
+        }
+    }
+}
+
+impl Error for XmloLowerError {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        None
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::{
+        asg::{FragmentText, IdentKind},
+        num::{Dim, Dtype},
+        obj::xmlo::{SymAttrs, SymType},
+        parse::Parsed,
+        span::{DUMMY_SPAN, UNKNOWN_SPAN},
+        sym::GlobalSymbolIntern,
+    };
+
+    type Sut = LowerState;
+
+    const S1: Span = DUMMY_SPAN;
+    const S2: Span = S1.offset_add(1).unwrap();
+    const S3: Span = S2.offset_add(1).unwrap();
+    const S4: Span = S3.offset_add(1).unwrap();
+
+    #[test]
+    fn data_from_package_event() {
+        let name = "name".into();
+        let relroot = "some/path".into();
+
+        let toks = vec![
+            XmloToken::PkgName(name, S1),
+            XmloToken::PkgRootPath(relroot, S2),
+            XmloToken::Eoh(S3),
+        ]
+        .into_iter();
+
+        let mut sut = Sut::parse(toks);
+
+        assert_eq!(Some(Ok(Parsed::Incomplete)), sut.next()); // PkgName
+        assert_eq!(Some(Ok(Parsed::Incomplete)), sut.next()); // PkgRootPath
+        assert_eq!(Some(Ok(Parsed::Incomplete)), sut.next()); // Eoh
+
+        let ctx = sut.finalize().unwrap();
+
+        assert_eq!(Some(name), ctx.prog_name);
+        assert_eq!(Some(relroot), ctx.relroot);
+    }
+
+    #[test]
+    fn adds_elig_as_root() {
+        let name = "name-root".into();
+        let elig_sym = "elig".into();
+
+        let toks = vec![
+            XmloToken::PkgName(name, S1),
+            XmloToken::PkgEligClassYields(elig_sym, S2),
+            XmloToken::Eoh(S3),
+        ];
+
+        assert_eq!(
+            Ok(vec![
+                Parsed::Incomplete, // PkgName
+                Parsed::Object(AirToken::IdentRoot(elig_sym)),
+                Parsed::Incomplete, // Eoh
+            ]),
+            Sut::parse(toks.into_iter()).collect(),
+        );
+    }
+
+    #[test]
+    fn adds_sym_deps() {
+        let sym_from = "from".into();
+        let sym_to1 = "to1".into();
+        let sym_to2 = "to2".into();
+
+        let toks = vec![
+            XmloToken::PkgName("name".into(), S1),
+            XmloToken::SymDepStart(sym_from, S2),
+            XmloToken::Symbol(sym_to1, S3),
+            XmloToken::Symbol(sym_to2, S4),
+            XmloToken::Eoh(S1),
+        ];
+
+        assert_eq!(
+            Ok(vec![
+                Parsed::Incomplete, // PkgName
+                Parsed::Incomplete, // SymDepStart
+                Parsed::Object(AirToken::IdentDep(sym_from, sym_to1)),
+                Parsed::Object(AirToken::IdentDep(sym_from, sym_to2)),
+                Parsed::Incomplete, // Eoh
+            ]),
+            Sut::parse(toks.into_iter()).collect(),
+        );
+    }
+
+    #[test]
+    fn sym_decl_with_src_not_added_and_populates_found() {
+        let sym = "sym".into();
+        let src_a = "src_a".into();
+        let src_b = "src_b".into();
+
+        let toks = vec![
+            XmloToken::PkgName("name".into(), S1),
+            XmloToken::SymDecl(
+                sym,
+                SymAttrs {
+                    src: Some(src_a),
+                    ..Default::default()
+                },
+                S2,
+            ),
+            XmloToken::SymDecl(
+                sym,
+                SymAttrs {
+                    src: Some(src_b),
+                    ..Default::default()
+                },
+                S3,
+            ),
+            XmloToken::Eoh(S1),
+        ];
+
+        let mut sut = Sut::parse(toks.into_iter());
+
+        assert_eq!(Some(Ok(Parsed::Incomplete)), sut.next()); // PkgName
+        assert_eq!(Some(Ok(Parsed::Incomplete)), sut.next()); // SymDecl (@src)
+        assert_eq!(Some(Ok(Parsed::Incomplete)), sut.next()); // SymDecl (@src)
+        assert_eq!(Some(Ok(Parsed::Incomplete)), sut.next()); // Eoh
+
+        let ctx = sut.finalize().unwrap();
+        let mut founds = ctx.found.unwrap().into_iter().collect::<Vec<_>>();
+
+        // Just to remove nondeterminism in case the iteration order happens
+        //   to change.
+        founds.sort();
+
+        assert_eq!(vec![src_a, src_b], founds);
+    }
+
+    #[test]
+    fn sym_decl_added_to_graph() {
+        let sym_extern = "sym_extern".into();
+        let sym_non_extern = "sym_non_extern".into();
+        let sym_map = "sym_map".into();
+        let sym_retmap = "sym_retmap".into();
+        let pkg_name = "pkg name".into();
+
+        let toks = vec![
+            XmloToken::PkgName("name".into(), S1),
+            XmloToken::SymDecl(
+                sym_extern,
+                SymAttrs {
+                    pkg_name: Some(pkg_name),
+                    extern_: true,
+                    ty: Some(SymType::Meta),
+                    ..Default::default()
+                },
+                S1,
+            ),
+            XmloToken::SymDecl(
+                sym_non_extern,
+                SymAttrs {
+                    pkg_name: Some(pkg_name),
+                    ty: Some(SymType::Meta),
+                    ..Default::default()
+                },
+                S2,
+            ),
+            XmloToken::SymDecl(
+                sym_map,
+                SymAttrs {
+                    pkg_name: Some(pkg_name),
+                    ty: Some(SymType::Map),
+                    ..Default::default()
+                },
+                S3,
+            ),
+            XmloToken::SymDecl(
+                sym_retmap,
+                SymAttrs {
+                    pkg_name: Some(pkg_name),
+                    ty: Some(SymType::RetMap),
+                    ..Default::default()
+                },
+                S4,
+            ),
+            XmloToken::Eoh(S1),
+        ];
+
+        let mut sut = Sut::parse(toks.into_iter());
+
+        // Note that each of these will have their package names cleared
+        //   since this is considered to be the first package encountered.
+        assert_eq!(Some(Ok(Parsed::Incomplete)), sut.next()); // PkgName
+        assert_eq!(
+            Some(Ok(Parsed::Object(AirToken::IdentExternDecl(
+                sym_extern,
+                IdentKind::Meta,
+                Source {
+                    pkg_name: None,
+                    ..Default::default()
+                }
+            )))),
+            sut.next(),
+        );
+        assert_eq!(
+            Some(Ok(Parsed::Object(AirToken::IdentDecl(
+                sym_non_extern,
+                IdentKind::Meta,
+                Source {
+                    pkg_name: None,
+                    ..Default::default()
+                }
+            )))),
+            sut.next(),
+        );
+        assert_eq!(
+            Some(Ok(Parsed::Object(AirToken::IdentDecl(
+                sym_map,
+                IdentKind::Map,
+                Source {
+                    pkg_name: None,
+                    ..Default::default()
+                }
+            )))),
+            sut.next(),
+        );
+        assert_eq!(
+            Some(Ok(Parsed::Object(AirToken::IdentDecl(
+                sym_retmap,
+                IdentKind::RetMap,
+                Source {
+                    pkg_name: None,
+                    ..Default::default()
+                }
+            )))),
+            sut.next(),
+        );
+        assert_eq!(Some(Ok(Parsed::Incomplete)), sut.next()); // Eoh
+
+        let ctx = sut.finalize().unwrap();
+
+        // Both above symbols were local (no `src`),
+        //   but note that we don't care if it's None or initialized with a
+        //   length of 0.
+        assert!(ctx.found.unwrap_or_default().len() == 0);
+    }
+
+    // See above test, where pkg_name was cleared.
+    #[test]
+    fn sym_decl_pkg_name_retained_if_not_first() {
+        let sym = "sym".into();
+        let pkg_name = "pkg name".into();
+
+        // This is all that's needed to not consider this to be the first
+        //   package,
+        //     so that pkg_name is retained below.
+        let ctx = LowerContext {
+            first: false,
+            ..Default::default()
+        };
+
+        let toks = vec![
+            XmloToken::PkgName(pkg_name, S1),
+            XmloToken::SymDecl(
+                sym,
+                SymAttrs {
+                    pkg_name: Some(pkg_name),
+                    ty: Some(SymType::Meta),
+                    ..Default::default()
+                },
+                UNKNOWN_SPAN,
+            ),
+            XmloToken::Eoh(S1),
+        ];
+
+        assert_eq!(
+            Ok(vec![
+                Parsed::Incomplete, // PkgName
+                Parsed::Object(AirToken::IdentDecl(
+                    sym,
+                    IdentKind::Meta,
+                    Source {
+                        pkg_name: Some(pkg_name),
+                        ..Default::default()
+                    }
+                )),
+                Parsed::Incomplete, // Eoh
+            ]),
+            Sut::parse_with_context(toks.into_iter(), ctx).collect(),
+        );
+    }
+
+    // This used to be set in SymAttrs by XmloReader,
+    //   but that's no longer true with the new reader.
+    #[test]
+    fn sym_decl_pkg_name_set_if_empty_and_not_first() {
+        let sym = "sym".into();
+        let pkg_name = "pkg name".into();
+
+        let ctx = LowerContext {
+            first: false,
+            ..Default::default()
+        };
+
+        let toks = vec![
+            XmloToken::PkgName(pkg_name, S1),
+            XmloToken::SymDecl(
+                sym,
+                SymAttrs {
+                    // No name
+                    ty: Some(SymType::Meta),
+                    ..Default::default()
+                },
+                UNKNOWN_SPAN,
+            ),
+            XmloToken::Eoh(S1),
+        ];
+
+        assert_eq!(
+            Ok(vec![
+                Parsed::Incomplete, // PkgName
+                Parsed::Object(AirToken::IdentDecl(
+                    sym,
+                    IdentKind::Meta,
+                    Source {
+                        pkg_name: Some(pkg_name), // Name inherited
+                        ..Default::default()
+                    },
+                )),
+                Parsed::Incomplete, // Eoh
+            ]),
+            Sut::parse_with_context(toks.into_iter(), ctx).collect(),
+        );
+    }
+
+    #[test]
+    fn ident_kind_conversion_error_propagates() {
+        let sym = "sym".into();
+        let bad_attrs = SymAttrs::default();
+
+        let toks = vec![
+            XmloToken::PkgName("name".into(), S1),
+            XmloToken::SymDecl(sym, bad_attrs, S2),
+            XmloToken::Eoh(S1),
+        ];
+
+        Sut::parse(toks.into_iter())
+            .collect::<Result<Vec<_>, _>>()
+            .expect_err("expected IdentKind conversion error");
+    }
+
+    #[test]
+    fn sets_fragment() {
+        let sym = "sym".into();
+        let frag = FragmentText::from("foo");
+
+        let toks = vec![
+            XmloToken::PkgName("name".into(), S1),
+            XmloToken::Fragment(sym, frag.clone(), S2),
+            XmloToken::Eoh(S1),
+        ];
+
+        assert_eq!(
+            Ok(vec![
+                Parsed::Incomplete, // PkgName
+                Parsed::Object(AirToken::IdentFragment(sym, frag)),
+                Parsed::Incomplete, // Eoh
+            ]),
+            Sut::parse(toks.into_iter()).collect(),
+        );
+    }
+
+    macro_rules! test_kind {
+        ($name:ident, $src:expr => $dest:expr) => {
+            #[test]
+            fn $name() {
+                assert_eq!(
+                    Ok($dest),
+                    SymAttrs {
+                        ty: Some($src),
+                        ..Default::default()
+                    }
+                    .try_into()
+                );
+            }
+        };
+
+        ($name:ident, $src:expr => $dest:expr, dim) => {
+            #[test]
+            fn $name() {
+                let dim = Dim::Vector;
+
+                assert_eq!(
+                    Ok($dest(Dim::Vector)),
+                    SymAttrs {
+                        ty: Some($src),
+                        dim: Some(dim),
+                        ..Default::default()
+                    }
+                    .try_into()
+                );
+
+                // no dim
+                let result = IdentKind::try_from(SymAttrs {
+                    ty: Some($src),
+                    ..Default::default()
+                })
+                .expect_err("must fail when missing dim");
+
+                assert_eq!(XmloLowerError::MissingDim, result);
+            }
+        };
+
+        ($name:ident, $src:expr => $dest:expr, dtype) => {
+            #[test]
+            fn $name() {
+                let dtype = Dtype::Float;
+
+                assert_eq!(
+                    Ok($dest(dtype)),
+                    SymAttrs {
+                        ty: Some($src),
+                        dtype: Some(dtype),
+                        ..Default::default()
+                    }
+                    .try_into()
+                );
+
+                // no dtype
+                let result = IdentKind::try_from(SymAttrs {
+                    ty: Some($src),
+                    ..Default::default()
+                })
+                .expect_err("must fail when missing dtype");
+
+                assert_eq!(XmloLowerError::MissingDtype, result);
+            }
+        };
+
+        ($name:ident, $src:expr => $dest:expr, dim, dtype) => {
+            #[test]
+            fn $name() {
+                let dim = Dim::Vector;
+                let dtype = Dtype::Float;
+
+                assert_eq!(
+                    Ok($dest(Dim::Vector, dtype)),
+                    SymAttrs {
+                        ty: Some($src),
+                        dim: Some(dim),
+                        dtype: Some(dtype),
+                        ..Default::default()
+                    }
+                    .try_into()
+                );
+
+                // no dim
+                let dim_result = IdentKind::try_from(SymAttrs {
+                    ty: Some($src),
+                    dtype: Some(dtype),
+                    ..Default::default()
+                })
+                .expect_err("must fail when missing dim");
+
+                assert_eq!(XmloLowerError::MissingDim, dim_result);
+
+                // no dtype
+                let dtype_result = IdentKind::try_from(SymAttrs {
+                    ty: Some($src),
+                    dim: Some(dim),
+                    ..Default::default()
+                })
+                .expect_err("must fail when missing dtype");
+
+                assert_eq!(XmloLowerError::MissingDtype, dtype_result);
+            }
+        };
+    }
+
+    test_kind!(cgen, SymType::Cgen => IdentKind::Cgen, dim);
+    test_kind!(class, SymType::Class => IdentKind::Class, dim);
+    test_kind!(r#const, SymType::Const => IdentKind::Const, dim, dtype);
+    test_kind!(func, SymType::Func => IdentKind::Func, dim, dtype);
+    test_kind!(gen, SymType::Gen => IdentKind::Gen, dim, dtype);
+    test_kind!(lparam, SymType::Lparam => IdentKind::Lparam, dim, dtype);
+    test_kind!(param, SymType::Param => IdentKind::Param, dim, dtype);
+    test_kind!(rate, SymType::Rate => IdentKind::Rate, dtype);
+    test_kind!(tpl, SymType::Tpl => IdentKind::Tpl);
+    test_kind!(r#type, SymType::Type => IdentKind::Type, dtype);
+    test_kind!(maphead, SymType::MapHead => IdentKind::MapHead);
+    test_kind!(map, SymType::Map => IdentKind::Map);
+    test_kind!(maptail, SymType::MapTail => IdentKind::MapTail);
+    test_kind!(retmaphead, SymType::RetMapHead => IdentKind::RetMapHead);
+    test_kind!(retmap, SymType::RetMap => IdentKind::RetMap);
+    test_kind!(retmaptail, SymType::RetMapTail => IdentKind::RetMapTail);
+    test_kind!(meta, SymType::Meta => IdentKind::Meta);
+    test_kind!(worksheet, SymType::Worksheet => IdentKind::Worksheet);
+
+    #[test]
+    fn source_from_sym_attrs() {
+        let nsym: SymbolId = "name".intern();
+        let ssym: SymbolId = "src".intern();
+        let psym: SymbolId = "parent".intern();
+        let ysym: SymbolId = "yields".intern();
+        let fsym: SymbolId = "from".intern();
+
+        let attrs = SymAttrs {
+            pkg_name: Some(nsym),
+            src: Some(ssym),
+            generated: true,
+            parent: Some(psym),
+            yields: Some(ysym),
+            desc: Some("sym desc".into()),
+            from: Some(fsym),
+            virtual_: true,
+            override_: true,
+            ..Default::default()
+        };
+
+        assert_eq!(
+            Source {
+                pkg_name: Some(nsym),
+                src: Some(ssym),
+                generated: attrs.generated,
+                parent: attrs.parent,
+                yields: attrs.yields,
+                desc: Some("sym desc".into()),
+                from: Some(fsym),
+                virtual_: true,
+                override_: true,
+            },
+            attrs.into(),
+        );
+    }
+}

tamer/src/obj/xmlo/mod.rs

@@ -74,12 +74,14 @@
 //! </package>
 //! ```
 
-mod asg_builder;
 mod error;
 mod ir;
+mod lower;
 mod reader;
 
-pub use asg_builder::{AsgBuilder, AsgBuilderError, AsgBuilderState};
 pub use error::XmloError;
 pub use ir::{SymAttrs, SymType};
+// TODO: Encapsulate LowerContext once it is no longer needed by the caller
+//   in `poc.rs`.
+pub use lower::{LowerContext, LowerState, XmloLowerError};
 pub use reader::{XmloReader, XmloToken};

tamer/src/obj/xmlo/reader.rs

@@ -24,7 +24,7 @@ use crate::{
     num::{Dim, Dtype},
     obj::xmlo::SymType,
     parse::{
-        self, EmptyContext, NoContext, ParseState, Transition,
+        self, EmptyContext, NoContext, ParseState, Token, Transition,
         TransitionResult, Transitionable,
     },
     span::Span,
@@ -44,13 +44,13 @@ use crate::{
 #[derive(Debug, PartialEq, Eq)]
 pub enum XmloToken {
     /// Canonical package name.
-    PkgName(SymbolId),
+    PkgName(SymbolId, Span),
     /// Relative path from package to project root.
-    PkgRootPath(SymbolId),
+    PkgRootPath(SymbolId, Span),
     /// Indicates that the package is a program.
-    PkgProgramFlag,
+    PkgProgramFlag(Span),
     /// Name of package eligibility classification.
-    PkgEligClassYields(SymbolId),
+    PkgEligClassYields(SymbolId, Span),
 
     /// Symbol declaration.
     ///
@@ -86,11 +86,49 @@ pub enum XmloToken {
 
 impl parse::Object for XmloToken {}
 
-/// A [`Result`] with a hard-coded [`XmloError`] error type.
-///
-/// This is the result of every [`XmloReader`] operation that could
-///   potentially fail in error.
-pub type XmloResult<T> = Result<T, XmloError>;
+impl Token for XmloToken {
+    fn span(&self) -> Span {
+        use XmloToken::*;
+
+        match self {
+            // Note that even the spans for the package metadata are
+            //   important since these initial tokens seed
+            //   `Parser::last_span`,
+            //     which is used for early error messages.
+            PkgName(_, span)
+            | PkgRootPath(_, span)
+            | PkgProgramFlag(span)
+            | PkgEligClassYields(_, span)
+            | SymDecl(.., span)
+            | SymDepStart(.., span)
+            | Symbol(.., span)
+            | Fragment(.., span)
+            | Eoh(span) => *span,
+        }
+    }
+}
+
+impl Display for XmloToken {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        use XmloToken::*;
+
+        match self {
+            PkgName(sym, _) => write!(f, "package name `{sym}`"),
+            PkgRootPath(sym, _) => write!(f, "package root path `{sym}`"),
+            PkgProgramFlag(_) => write!(f, "package program flag"),
+            PkgEligClassYields(sym, _) => {
+                write!(f, "package eligibility classification `{sym}`")
+            }
+            SymDecl(sym, ..) => write!(f, "symbol `{sym}` declaration"),
+            SymDepStart(sym, ..) => {
+                write!(f, "beginning of symbol `{sym}` dependency list")
+            }
+            Symbol(sym, ..) => write!(f, "symbol `{sym}`"),
+            Fragment(sym, ..) => write!(f, "symbol `{sym}` code fragment"),
+            Eoh(..) => write!(f, "end of header"),
+        }
+    }
+}
 
 qname_const! {
     QN_DESC: :L_DESC,
@@ -137,7 +175,7 @@ pub enum XmloReader<
     #[default]
     Ready,
     /// Processing `package` attributes.
-    Package,
+    Package(Span),
     /// Expecting a symbol declaration or closing `preproc:symtable`.
     Symtable(Span, SS),
     /// Symbol dependencies are expected next.
@@ -169,33 +207,36 @@ impl<SS: XmloState, SD: XmloState, SF: XmloState> ParseState
         use XmloReader::*;
 
         match (self, tok) {
-            (Ready, Xirf::Open(QN_LV_PACKAGE | QN_PACKAGE, ..)) => {
-                Transition(Package).incomplete()
+            (Ready, Xirf::Open(QN_LV_PACKAGE | QN_PACKAGE, span, ..)) => {
+                Transition(Package(span)).incomplete()
             }
 
             (Ready, tok) => {
                 Transition(Ready).err(XmloError::UnexpectedRoot(tok))
             }
 
-            (Package, Xirf::Attr(Attr(name, value, _))) => {
-                Transition(Package).ok(match name {
-                    QN_NAME => XmloToken::PkgName(value),
-                    QN_UUROOTPATH => XmloToken::PkgRootPath(value),
-                    QN_PROGRAM => XmloToken::PkgProgramFlag,
+            (Package(span), Xirf::Attr(Attr(name, value, aspan))) => {
+                // TODO: These spans do not encompass the entire token for errors,
+                //   which can result in confusing output depending on the context;
+                //     we ought to retain _both_ token- and value-spans.
+                Transition(Package(span)).ok(match name {
+                    QN_NAME => XmloToken::PkgName(value, aspan.1),
+                    QN_UUROOTPATH => XmloToken::PkgRootPath(value, aspan.1),
+                    QN_PROGRAM => XmloToken::PkgProgramFlag(aspan.0), // yes 0
                     QN_ELIG_CLASS_YIELDS => {
-                        XmloToken::PkgEligClassYields(value)
+                        XmloToken::PkgEligClassYields(value, aspan.1)
                     }
                     // Ignore unknown attributes for now to maintain BC,
                     //   since no strict xmlo schema has been defined.
-                    _ => return Transition(Package).incomplete(),
+                    _ => return Transition(Package(span)).incomplete(),
                 })
             }
 
             // Empty package (should we allow this?);
             //   XIRF guarantees a matching closing tag.
-            (Package, Xirf::Close(..)) => Transition(Done).incomplete(),
+            (Package(_), Xirf::Close(..)) => Transition(Done).incomplete(),
 
-            (Package, Xirf::Open(QN_SYMTABLE, span, ..)) => {
+            (Package(_), Xirf::Open(QN_SYMTABLE, span, ..)) => {
                 Transition(Symtable(span, SS::default())).incomplete()
             }
 
@@ -264,7 +305,7 @@ impl<SS: XmloState, SD: XmloState, SF: XmloState> Display
 
         match self {
             Ready => write!(f, "awaiting xmlo input"),
-            Package => write!(f, "processing package attributes"),
+            Package(_) => write!(f, "processing package attributes"),
             Symtable(_, ss) => Display::fmt(ss, f),
             SymDepsExpected => write!(f, "expecting symbol dependency list"),
             SymDeps(_, sd) => Display::fmt(sd, f),

tamer/src/obj/xmlo/reader/test.rs

@@ -83,10 +83,14 @@ fn common_parses_package_attrs(package: QName) {
     assert_eq!(
         Ok(vec![
             Parsed::Incomplete,
-            Parsed::Object(XmloToken::PkgName(name)),
-            Parsed::Object(XmloToken::PkgRootPath(relroot)),
-            Parsed::Object(XmloToken::PkgProgramFlag),
-            Parsed::Object(XmloToken::PkgEligClassYields(elig)),
+            Parsed::Object(XmloToken::PkgName(name, S3)),
+            Parsed::Object(XmloToken::PkgRootPath(relroot, S3)),
+            // Span for the program flag is the attr name,
+            //   rather than the value,
+            //   since the value is just a boolean and does not provide as
+            //     useful of context.
+            Parsed::Object(XmloToken::PkgProgramFlag(S3)),
+            Parsed::Object(XmloToken::PkgEligClassYields(elig, S4)),
             Parsed::Incomplete,
         ]),
         sut.collect(),
@@ -125,7 +129,7 @@ fn ignores_unknown_package_attr() {
     assert_eq!(
         Ok(vec![
             Parsed::Incomplete,
-            Parsed::Object(XmloToken::PkgName(name)),
+            Parsed::Object(XmloToken::PkgName(name, S3)),
             Parsed::Incomplete, // The unknown attribute
             Parsed::Incomplete,
         ]),

tamer/src/parse.rs

@@ -65,6 +65,8 @@ impl<T: Token> From<T> for Span {
 ///     used in the [`Transition`] API to provide greater flexibility.
 pub trait Object: Debug + PartialEq {}
 
+impl Object for () {}
+
 /// An infallible [`Token`] stream.
 ///
 /// If the token stream originates from an operation that could potentially
@@ -730,7 +732,8 @@ impl<S: ParseState, I: TokenStream<S::Token>> Parser<S, I> {
         self.while_ok(|toks| {
             // TODO: This parser is not accessible after error recovery!
             let lower = LS::parse(iter::empty());
-            f(&mut LowerIter { lower, toks })
+            let mut iter = LowerIter { lower, toks };
+            f(&mut iter)
         })
         .map_err(Into::into)
     }
@@ -760,6 +763,62 @@ where
     >,
 }
 
+impl<'a, 'b, S, I, LS> LowerIter<'a, 'b, S, I, LS>
+where
+    S: ParseState,
+    I: Iterator<Item = ParsedResult<S>>,
+    LS: ParseState<Token = S::Object>,
+    <S as ParseState>::Object: Token,
+{
+    /// Consume inner parser and yield its context.
+    #[inline]
+    fn finalize(self) -> Result<LS::Context, ParseError<LS::Token, LS::Error>> {
+        self.lower.finalize().map_err(|(_, e)| e)
+    }
+}
+
+/// Lowering operation from one [`ParseState`] to another.
+pub trait Lower<S, LS>
+where
+    S: ParseState,
+    LS: ParseState<Token = S::Object>,
+    <S as ParseState>::Object: Token,
+{
+    /// Perform a lowering operation between two parsers where the context
+    ///   is both received and returned.
+    ///
+    /// This allows state to be shared among parsers.
+    ///
+    /// See [`ParseState::parse_with_context`] for more information.
+    fn lower_with_context_while_ok<U, E>(
+        &mut self,
+        ctx: LS::Context,
+        f: impl FnOnce(&mut LowerIter<S, Self, LS>) -> Result<U, E>,
+    ) -> Result<(U, LS::Context), E>
+    where
+        Self: Iterator<Item = ParsedResult<S>> + Sized,
+        ParseError<S::Token, S::Error>: Into<E>,
+        ParseError<LS::Token, LS::Error>: Into<E>,
+    {
+        self.while_ok(|toks| {
+            let lower = LS::parse_with_context(iter::empty(), ctx);
+            let mut iter = LowerIter { lower, toks };
+            let val = f(&mut iter)?;
+
+            iter.finalize().map_err(Into::into).map(|ctx| (val, ctx))
+        })
+    }
+}
+
+impl<S, LS, I> Lower<S, LS> for I
+where
+    I: Iterator<Item = ParsedResult<S>> + Sized,
+    S: ParseState,
+    LS: ParseState<Token = S::Object>,
+    <S as ParseState>::Object: Token,
+{
+}
+
 impl<'a, 'b, S, I, LS> Iterator for LowerIter<'a, 'b, S, I, LS>
 where
     S: ParseState,

tamer/src/xir/attr.rs

@@ -61,10 +61,18 @@ impl Attr {
 
 impl Token for Attr {
     fn span(&self) -> Span {
-        // TODO: This may or may not actually represent the span relative to
-        //   a given parser,
-        //     so we may want to accept a context to bias toward.
-        self.2 .1
+        // TODO: This ought to represent the _entire_ token.
+        //   However,
+        //     this is complicated by the closing quote,
+        //     which is not present in _either_ span,
+        //       so we'll need to formalize that first;
+        //         simply adding a single byte offset seems unwise,
+        //           given that this is not responsible for producing the
+        //           spans to begin with;
+        //             I'd prefer help from XIR.
+        match self {
+            Attr(.., (span, _)) => *span,
+        }
     }
 }