From 454f5f4d047e563ab7f091ec9499bdfb8be3966a Mon Sep 17 00:00:00 2001
From: Mike Gerwitz <mike.gerwitz@ryansg.com>
Date: Tue, 13 Jun 2023 23:40:30 -0400
Subject: [PATCH] tamer: Initial clarifying pipeline docs

This provides some initial information to help guide a user to discover how
TAMER works, though either the source code or the generated
documentation.  This will improve over time, since all of the high-level
abstractions are still under development.

DEV-13162
---
 tamer/src/lib.rs            |  5 ++++
 tamer/src/pipeline.rs       | 53 +++++++++++++++++++++++++------------
 tamer/src/pipeline/macro.rs |  1 +
 3 files changed, 42 insertions(+), 17 deletions(-)

diff --git a/tamer/src/lib.rs b/tamer/src/lib.rs
index 471fe037..febfc4b6 100644
--- a/tamer/src/lib.rs
+++ b/tamer/src/lib.rs
@@ -23,6 +23,11 @@
 //!
 //!   - [`tamec`](../tamec), the TAME compiler; and
 //!   - [`tameld`](../tameld), the TAME linker.
+//!
+//! The [`pipeline`] module contains declarative definitions and
+//!   documentation for TAMER's _lowering pipelines_;
+//!     you should start there if you are looking for how a particular
+//!     component of parsing or code generation is integrated.
 
 // Constant functions are still in their infancy as of the time of writing
 //   (October 2021).
diff --git a/tamer/src/pipeline.rs b/tamer/src/pipeline.rs
index 4a078c1e..cb2a5b74 100644
--- a/tamer/src/pipeline.rs
+++ b/tamer/src/pipeline.rs
@@ -128,25 +128,23 @@ use crate::{
 mod r#macro;
 
 lower_pipeline! {
-    /// Load an `xmlo` file represented by `src` into the graph held
-    ///   by `air_ctx`.
-    ///
-    /// Loading an object file will result in opaque objects being added to the
-    ///   graph.
-    ///
-    /// TODO: To re-use this in `tamec` we want to be able to ignore fragments.
-    ///
-    /// TODO: More documentation once this has been further cleaned up.
-    pub load_xmlo -> LoadXmlo
-        |> PartialXirToXirf<4, Text>
-        |> XmloReader
-        |> XmloToAir[xmlo_ctx], until (XmloToken::Eoh(..))
-        |> AirAggregate[air_ctx];
-
     /// Parse a source package into the [ASG](crate::asg) using TAME's XML
     ///   source language.
     ///
-    /// TODO: More documentation once this has been further cleaned up.
+    /// Source XML is represented by [XIR](crate::xir).
+    /// This is parsed into [NIR`](crate::nir),
+    ///   which extracts TAME's high-level source language from the document
+    ///   format (XML).
+    /// NIR is then desugared in various ways,
+    ///   producing a more verbose NIR than what the user originally
+    ///   entered.
+    ///
+    /// NIR is then lowered into [AIR](crate::asg::air),
+    ///   which is then aggregated into the [ASG](crate::asg) to await
+    ///   further processing.
+    /// It is after this point that package imports should be processed and
+    ///   also aggregated into the same ASG so that all needed dependencies
+    ///   definitions are available.
     pub parse_package_xml -> ParsePackageXml
         |> XirToXirf<64, RefinedText>
         |> XirfToNir
@@ -155,10 +153,31 @@ lower_pipeline! {
         |> NirToAir[nir_air_ty]
         |> AirAggregate[air_ctx];
 
+    /// Load an `xmlo` file into the graph held by `air_ctx`.
+    ///
+    /// Loading an object file will result in opaque objects being added to the
+    ///   graph;
+    ///     no sources will be parsed.
+    ///
+    /// To parse sources instead,
+    ///   see [`parse_package_xml`].
+    pub load_xmlo -> LoadXmlo
+        |> PartialXirToXirf<4, Text>
+        |> XmloReader
+        |> XmloToAir[xmlo_ctx], until (XmloToken::Eoh(..))
+        |> AirAggregate[air_ctx];
+
     /// Lower an [`Asg`](crate::asg::Asg)-derived token stream into an
     ///   `xmli` file.
     ///
-    /// TODO: More documentation once this has been further cleaned up.
+    /// After a package has been parsed with [`parse_package_xml`] and
+    ///   further processing has taken place,
+    ///     this pipeline will re-generate TAME sources from the ASG for the
+    ///     purpose of serving as source input to the XSLT-based compiler.
+    /// This allows us to incrementally replace that compiler's
+    ///   functionality by having the XSLT-based system pick up where we
+    ///   leave off,
+    ///     skipping anything that we have already done.
     pub lower_xmli<'a> -> LowerXmli
         |> AsgTreeToXirf<'a>[asg]
         |> XirfAutoClose
diff --git a/tamer/src/pipeline/macro.rs b/tamer/src/pipeline/macro.rs
index 625b466c..ac9500d7 100644
--- a/tamer/src/pipeline/macro.rs
+++ b/tamer/src/pipeline/macro.rs
@@ -171,6 +171,7 @@ macro_rules! lower_pipeline {
 
         lower_pipeline!(
             @pipeline
+            $(#[$meta])*
             $vis $fn$(<$l>)? -> $struct
                 $(|> $lower_name$(<$($lower_t),+>)? $([$ctx])? $(, until ($until))?)*
         );