TAMER: Symbol source data and metadata

tamer/src/ir/asg/base.rs

@@ -19,7 +19,7 @@
 
 use super::graph::{Asg, AsgEdge, AsgError, AsgResult, Node, ObjectRef};
 use super::ident::IdentKind;
-use super::object::{FragmentText, Object};
+use super::object::{FragmentText, Object, Source};
 use crate::sym::Symbol;
 use fixedbitset::FixedBitSet;
 use petgraph::graph::{
@@ -125,13 +125,14 @@ where
         &mut self,
         name: &'i Symbol<'i>,
         kind: IdentKind,
+        src: Source<'i>,
     ) -> AsgResult<ObjectRef<Ix>> {
         // TODO: src check
         if let Some(existing) = self.lookup(name) {
             return Ok(existing);
         }
 
-        let node = self.graph.add_node(Some(Object::Ident(name, kind)));
+        let node = self.graph.add_node(Some(Object::Ident(name, kind, src)));
 
         self.index_identifier(name, node);
 
@@ -172,8 +173,8 @@ where
             .expect("internal error: BaseAsg::set_fragment missing Node data");
 
         let result = match ty {
-            Object::Ident(sym, kind) => {
+            Object::Ident(sym, kind, src) => {
-                Ok(Object::IdentFragment(sym, kind, text))
+                Ok(Object::IdentFragment(sym, kind, src, text))
             }
             _ => {
                 let err = Err(AsgError::BadFragmentDest(format!(
@@ -287,18 +288,47 @@ mod test {
         let syma = Symbol::new_dummy(SymbolIndex::from_u32(5), "syma");
         let symb = Symbol::new_dummy(SymbolIndex::from_u32(1), "symab");
 
-        let nodea = sut.declare(&syma, IdentKind::Meta)?;
+        let nodea = sut.declare(
-        let nodeb = sut.declare(&symb, IdentKind::Worksheet)?;
+            &syma,
+            IdentKind::Meta,
+            Source {
+                desc: Some("a".to_string()),
+                ..Default::default()
+            },
+        )?;
+
+        let nodeb = sut.declare(
+            &symb,
+            IdentKind::Worksheet,
+            Source {
+                desc: Some("b".to_string()),
+                ..Default::default()
+            },
+        )?;
 
         assert_ne!(nodea, nodeb);
 
         assert_eq!(
-            Some(&Object::Ident(&syma, IdentKind::Meta)),
+            Some(&Object::Ident(
+                &syma,
+                IdentKind::Meta,
+                Source {
+                    desc: Some("a".to_string()),
+                    ..Default::default()
+                },
+            )),
             sut.get(nodea),
         );
 
         assert_eq!(
-            Some(&Object::Ident(&symb, IdentKind::Worksheet)),
+            Some(&Object::Ident(
+                &symb,
+                IdentKind::Worksheet,
+                Source {
+                    desc: Some("b".to_string()),
+                    ..Default::default()
+                },
+            )),
             sut.get(nodeb),
         );
 
@@ -310,7 +340,14 @@ mod test {
         let mut sut = Sut::with_capacity(0, 0);
 
         let sym = Symbol::new_dummy(SymbolIndex::from_u32(1), "lookup");
-        let node = sut.declare(&sym, IdentKind::Meta)?;
+        let node = sut.declare(
+            &sym,
+            IdentKind::Meta,
+            Source {
+                generated: true,
+                ..Default::default()
+            },
+        )?;
 
         assert_eq!(Some(node), sut.lookup(&sym));
 
@@ -335,10 +372,11 @@ mod test {
         let mut sut = Sut::with_capacity(0, 0);
 
         let sym = Symbol::new_dummy(SymbolIndex::from_u32(1), "symdup");
-        let node = sut.declare(&sym, IdentKind::Meta)?;
+        let node = sut.declare(&sym, IdentKind::Meta, Source::default())?;
 
         // Same declaration a second time
-        let redeclare = sut.declare(&sym, IdentKind::Meta)?;
+        let redeclare =
+            sut.declare(&sym, IdentKind::Meta, Source::default())?;
 
         assert_eq!(node, redeclare);
         Ok(())
@@ -349,7 +387,11 @@ mod test {
         let mut sut = Sut::with_capacity(0, 0);
 
         let sym = Symbol::new_dummy(SymbolIndex::from_u32(1), "tofrag");
-        let node = sut.declare(&sym, IdentKind::Meta)?;
+        let src = Source {
+            generated: true,
+            ..Default::default()
+        };
+        let node = sut.declare(&sym, IdentKind::Meta, src.clone())?;
 
         let fragment = "a fragment".to_string();
         let node_with_frag = sut.set_fragment(node, fragment.clone())?;
@@ -362,7 +404,7 @@ mod test {
         );
 
         assert_eq!(
-            Some(&Object::IdentFragment(&sym, IdentKind::Meta, fragment)),
+            Some(&Object::IdentFragment(&sym, IdentKind::Meta, src, fragment)),
             sut.get(node)
         );
 
@@ -374,7 +416,7 @@ mod test {
         let mut sut = Sut::with_capacity(0, 0);
 
         let sym = Symbol::new_dummy(SymbolIndex::from_u32(1), "sym");
-        let node = sut.declare(&sym, IdentKind::Meta)?;
+        let node = sut.declare(&sym, IdentKind::Meta, Source::default())?;
         let fragment = "orig fragment".to_string();
 
         sut.set_fragment(node, fragment.clone())?;
@@ -391,7 +433,12 @@ mod test {
 
         // Make sure we didn't leave the node in an inconsistent state
         assert_eq!(
-            Some(&Object::IdentFragment(&sym, IdentKind::Meta, fragment)),
+            Some(&Object::IdentFragment(
+                &sym,
+                IdentKind::Meta,
+                Default::default(),
+                fragment
+            )),
             sut.get(node)
         );
 
@@ -405,8 +452,8 @@ mod test {
         let sym = Symbol::new_dummy(SymbolIndex::from_u32(1), "sym");
         let dep = Symbol::new_dummy(SymbolIndex::from_u32(1), "dep");
 
-        let symnode = sut.declare(&sym, IdentKind::Meta)?;
+        let symnode = sut.declare(&sym, IdentKind::Meta, Source::default())?;
-        let depnode = sut.declare(&dep, IdentKind::Meta)?;
+        let depnode = sut.declare(&dep, IdentKind::Meta, Source::default())?;
 
         sut.add_dep(symnode, depnode);
         assert!(sut.has_dep(symnode, depnode));

tamer/src/ir/asg/graph.rs

@@ -18,7 +18,7 @@
 //! Abstract graph as the basis for concrete ASGs.
 
 use super::ident::IdentKind;
-use super::object::{FragmentText, Object};
+use super::object::{FragmentText, Object, Source};
 use crate::sym::Symbol;
 use petgraph::graph::{IndexType, NodeIndex};
 use std::result::Result;
@@ -65,6 +65,7 @@ pub trait Asg<'i, Ix: IndexType> {
         &mut self,
         name: &'i Symbol<'i>,
         kind: IdentKind,
+        src: Source<'i>,
     ) -> AsgResult<ObjectRef<Ix>>;
 
     /// Declare an abstract identifier.

tamer/src/ir/asg/ident.rs

@@ -150,6 +150,18 @@ impl<'i> TryFrom<SymAttrs<'i>> for IdentKind {
     /// Certain [`IdentKind`] require that certain attributes be present,
     ///   otherwise the conversion will fail.
     fn try_from(attrs: SymAttrs<'i>) -> Result<Self, Self::Error> {
+        Self::try_from(&attrs)
+    }
+}
+
+impl<'i> TryFrom<&SymAttrs<'i>> for IdentKind {
+    type Error = &'static str;
+
+    /// Attempt to raise [`SymAttrs`] into an [`IdentKind`].
+    ///
+    /// Certain [`IdentKind`] require that certain attributes be present,
+    ///   otherwise the conversion will fail.
+    fn try_from(attrs: &SymAttrs<'i>) -> Result<Self, Self::Error> {
         let ty = attrs.ty.as_ref().ok_or("missing symbol type")?;
 
         macro_rules! ident {
@@ -205,6 +217,27 @@ impl<'i> TryFrom<SymAttrs<'i>> for IdentKind {
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 pub struct Dim(u8);
 
+/// Underlying datatype of identifier.
+///
+/// TODO: This will always be 0≤n≤9, so let's introduce a newtype for it.
+impl AsRef<str> for Dim {
+    fn as_ref(&self) -> &str {
+        match self.0 {
+            0 => &"0",
+            1 => &"1",
+            2 => &"2",
+            3 => &"3",
+            4 => &"4",
+            5 => &"5",
+            6 => &"6",
+            7 => &"7",
+            8 => &"8",
+            9 => &"9",
+            _ => unreachable!(),
+        }
+    }
+}
+
 /// Underlying datatype of identifier.
 pub type DataType = SymDtype;
 
@@ -213,6 +246,16 @@ mod test {
     use super::*;
     use std::convert::TryInto;
 
+    #[test]
+    fn dim_to_str() {
+        // we'll just test high and low
+        let low: &str = Dim(0).as_ref();
+        let high: &str = Dim(9).as_ref();
+
+        assert_eq!("0", low);
+        assert_eq!("9", high);
+    }
+
     macro_rules! test_kind {
         ($name:ident, $src:expr => $dest:expr) => {
             #[test]

tamer/src/ir/asg/mod.rs

@@ -59,7 +59,7 @@
 //!
 //! ```
 //! use tamer::global;
-//! use tamer::ir::asg::{Asg, DefaultAsg, IdentKind, Object};
+//! use tamer::ir::asg::{Asg, DefaultAsg, IdentKind, Object, Source};
 //! use tamer::sym::{Interner, DefaultInterner};
 //!
 //! # fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -71,7 +71,7 @@
 //! let identa_sym = interner.intern("identa");
 //! let identb_sym = interner.intern("identb");
 //!
-//! let identa = asg.declare(identa_sym, IdentKind::Meta)?;
+//! let identa = asg.declare(identa_sym, IdentKind::Meta, Source::default())?;
 //! let identb = asg.declare_extern(identb_sym, IdentKind::Meta)?;
 //!
 //! assert_eq!(
@@ -103,7 +103,7 @@
 //!
 //! ```
 //! # use tamer::global;
-//! # use tamer::ir::asg::{Asg, DefaultAsg, IdentKind, Object, FragmentText};
+//! # use tamer::ir::asg::{Asg, DefaultAsg, IdentKind, Object, FragmentText, Source};
 //! # use tamer::sym::{Interner, DefaultInterner};
 //! #
 //! # fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -111,13 +111,16 @@
 //! # let interner = DefaultInterner::new();
 //! #
 //! // Fragments can be attached to resolved identifiers.
-//! let ident = asg.declare(interner.intern("ident"), IdentKind::Meta)?;
+//! let ident = asg.declare(
+//!     interner.intern("ident"), IdentKind::Meta, Source::default()
+//! )?;
 //! asg.set_fragment(ident, FragmentText::from("test fragment"))?;
 //!
 //! assert_eq!(
 //!     Some(&Object::IdentFragment(
 //!         interner.intern("ident"),
 //!         IdentKind::Meta,
+//!         Source::default(),
 //!         FragmentText::from("test fragment"),
 //!     )),
 //!     asg.get(ident),
@@ -138,7 +141,7 @@ mod object;
 
 pub use graph::{Asg, AsgResult, ObjectRef};
 pub use ident::IdentKind;
-pub use object::{FragmentText, Object};
+pub use object::{FragmentText, Object, Source};
 
 /// Default concrete ASG implementation.
 pub type DefaultAsg<'i, Ix> = base::BaseAsg<'i, Ix>;

tamer/src/ir/asg/object.rs

@@ -21,6 +21,7 @@
 //!  See [`super`] for available exports._
 
 use super::ident::IdentKind;
+use crate::ir::legacyir::SymAttrs;
 use crate::sym::Symbol;
 
 /// Type of object.
@@ -38,15 +39,13 @@ use crate::sym::Symbol;
 ///   through [`Asg`][super::Asg]'s public API,
 ///     as it represents the absence of an object at that node within the
 ///     ASG.
-///
-/// TODO: Source location (span; see Rustc).
 #[derive(Debug, PartialEq)]
 pub enum Object<'i> {
     /// A resolved identifier.
     ///
     /// This represents an identifier that has been declared with certain
     ///   type information.
-    Ident(&'i Symbol<'i>, IdentKind),
+    Ident(&'i Symbol<'i>, IdentKind, Source<'i>),
 
     /// An identifier that has not yet been resolved.
     ///
@@ -63,7 +62,7 @@ pub enum Object<'i> {
     /// They are produced by the compiler and it is the job of the
     ///   [linker][crate::ld] to put them into the correct order for the
     ///   final executable.
-    IdentFragment(&'i Symbol<'i>, IdentKind, FragmentText),
+    IdentFragment(&'i Symbol<'i>, IdentKind, Source<'i>, FragmentText),
 
     /// The empty node (default value for indexer).
     ///
@@ -75,3 +74,87 @@ pub enum Object<'i> {
 ///
 /// This represents the text associated with an identifier.
 pub type FragmentText = String;
+
+/// Metadata about the source of an object.
+///
+/// This contains information from the symbol table that does not belong on
+///   [`IdentKind`],
+///     since that stores _type_ information.
+///
+/// TODO: This does not currently store byte offsets within the source file
+///   since the original XSLT-based compiler did not have that capability;
+///     this will provide that information in the future.
+#[derive(Debug, Default, PartialEq, Clone)]
+pub struct Source<'i> {
+    /// The identifier from which this one is derived.
+    ///
+    /// See [`IdentKind`] for more information on parents.
+    /// For example,
+    ///   a [`IdentKind::Cgen`] always has a parent [`IdentKind::Class`].
+    pub parent: Option<&'i Symbol<'i>>,
+
+    /// Child identifier associated with this identifier.
+    ///
+    /// For [`IdentKind::Class`],
+    ///   this represents an associated [`IdentKind::Cgen`].
+    pub yields: Option<&'i Symbol<'i>>,
+
+    /// User-friendly identifier description.
+    ///
+    /// This is used primarily by [`IdentKind::Class`] and
+    ///   [`IdentKind::Gen`].
+    pub desc: Option<String>,
+
+    /// Whether this identifier was generated by the compiler.
+    ///
+    /// A generated identifier is representative of an internal
+    ///   implementation detail that should remain encapsulated from the
+    ///   user and is subject to change over time.
+    ///
+    /// Identifiers created by templates are not considered to be generated.
+    pub generated: bool,
+}
+
+impl<'i> From<SymAttrs<'i>> for Source<'i> {
+    /// Raise Legacy IR [`SymAttrs`].
+    ///
+    /// This simply extracts a subset of fields from the source attributes.
+    fn from(attrs: SymAttrs<'i>) -> Self {
+        Source {
+            generated: attrs.generated,
+            parent: attrs.parent,
+            yields: attrs.yields,
+            desc: attrs.desc,
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::sym::SymbolIndex;
+
+    #[test]
+    fn source_from_sym_attrs() {
+        let psym = Symbol::new_dummy(SymbolIndex::from_u32(1), "parent");
+        let ysym = Symbol::new_dummy(SymbolIndex::from_u32(2), "yields");
+
+        let attrs = SymAttrs {
+            generated: true,
+            parent: Some(&psym),
+            yields: Some(&ysym),
+            desc: Some("sym desc".to_string()),
+            ..Default::default()
+        };
+
+        assert_eq!(
+            Source {
+                generated: attrs.generated,
+                parent: attrs.parent,
+                yields: attrs.yields,
+                desc: Some("sym desc".to_string()),
+            },
+            attrs.into(),
+        );
+    }
+}

tamer/src/ir/legacyir.rs

@@ -104,6 +104,33 @@ pub struct SymAttrs<'i> {
     /// The linker (see [`crate::ld`]) is responsible for ensuring that the
     ///   extern is satisfied and properly located in the final executable.
     pub extern_: bool,
+
+    /// The identifier from which this one is derived.
+    ///
+    /// For example,
+    ///   [`SymType::Cgen`] has a parent [`SymType::Class`] and
+    ///   [`SymType::Gen`] has a parent [`SymType::Rate`].
+    pub parent: Option<&'i Symbol<'i>>,
+
+    /// Whether this identifier was generated by the compiler.
+    ///
+    /// A generated identifier is representative of an internal
+    ///   implementation detail that should remain encapsulated from the
+    ///   user and is subject to change over time.
+    ///
+    /// Identifiers created by templates are not considered to be generated.
+    pub generated: bool,
+
+    /// Child identifier associated with this identifier.
+    ///
+    /// For [`SymType::Class`],
+    ///   this represents an associated [`SymType::Cgen`].
+    pub yields: Option<&'i Symbol<'i>>,
+
+    /// User-friendly identifier description.
+    ///
+    /// This is used primarily by [`SymType::Class`] and [`SymType::Gen`].
+    pub desc: Option<String>,
 }
 
 /// Legacy symbol types.

tamer/src/ld/poc.rs

@@ -135,10 +135,12 @@ fn load_xmlo<'a, 'i, I: Interner<'i>>(
 
                 let owned = attrs.src.is_none();
 
-                let kind = attrs.try_into().map_err(|err| {
+                let kind = (&attrs).try_into().map_err(|err| {
                     format!("sym `{}` attrs error: {}", sym, err)
                 });
 
+                let src = attrs.into();
+
                 // TODO: should probably track these down in the XSLT linker...
                 match kind {
                     Ok(kindval) => {
@@ -148,7 +150,7 @@ fn load_xmlo<'a, 'i, I: Interner<'i>>(
                                 || sym.starts_with(":map:")
                                 || sym.starts_with(":retmap:"));
 
-                        let node = depgraph.declare(sym, kindval)?;
+                        let node = depgraph.declare(sym, kindval, src)?;
 
                         if link_root {
                             roots.push(node);

tamer/src/obj/xmlo/reader.rs

@@ -392,6 +392,28 @@ impl<'i, B: BufRead, I: Interner<'i>> XmloReader<'i, B, I> {
                     sym_attrs.extern_ = &*attr.value == b"true";
                 }
 
+                b"preproc:generated" => {
+                    sym_attrs.generated = &*attr.value == b"true";
+                }
+
+                b"parent" => {
+                    sym_attrs.parent = Some(unsafe {
+                        interner.intern_utf8_unchecked(&attr.value)
+                    });
+                }
+
+                b"yields" => {
+                    sym_attrs.yields = Some(unsafe {
+                        interner.intern_utf8_unchecked(&attr.value)
+                    });
+                }
+
+                b"desc" => {
+                    sym_attrs.desc = Some(unsafe {
+                        String::from_utf8_unchecked(attr.value.to_vec())
+                    });
+                }
+
                 // As this reader evolves, we may wish to provide an error
                 // for unknown attributes so that we can be sure that we've
                 // handled them all.
@@ -1411,6 +1433,21 @@ mod test {
             ..Default::default()
         }
 
+        parent: [parent="foo"] => SymAttrs {
+            parent: Some(interner.intern("foo")),
+            ..Default::default()
+        }
+
+        yields: [yields="yield"] => SymAttrs {
+            yields: Some(interner.intern("yield")),
+            ..Default::default()
+        }
+
+        desc: [desc="Description"] => SymAttrs {
+            desc: Some("Description".to_string()),
+            ..Default::default()
+        }
+
         // Multiple attributes at once
         multi: [src="foo", type="class", dim="1", dtype="float", extern="true"]
             => SymAttrs {
@@ -1419,9 +1456,48 @@ mod test {
                 dim: Some(1),
                 dtype: Some(SymDtype::Float),
                 extern_: true,
+                ..Default::default()
             }
     }
 
+    // can't be tested using the above
+    #[test]
+    fn generated_true() -> XmloResult<()> {
+        let stub_data: &[u8] = &[];
+        let interner = DefaultInterner::new();
+        let mut sut = Sut::new(stub_data, &interner);
+
+        // See xmlo_tests macro for explanation
+        sut.seen_root = true;
+
+        sut.reader.next_event = Some(Box::new(|_, _| {
+            Ok(XmlEvent::Empty(MockBytesStart::new(
+                b"preproc:sym",
+                Some(MockAttributes::new(vec![
+                    MockAttribute::new(b"name", b"generated_true"),
+                    MockAttribute::new(b"preproc:generated", b"true"),
+                ])),
+            )))
+        }));
+
+        let result = sut.read_event()?;
+
+        let expected_attrs = SymAttrs {
+            generated: true,
+            ..Default::default()
+        };
+
+        assert_eq!(
+            XmloEvent::SymDecl(
+                interner.intern("generated_true"),
+                expected_attrs
+            ),
+            result
+        );
+
+        Ok(())
+    }
+
     #[test]
     fn fails_on_non_ascii_dim() {
         let stub_data: &[u8] = &[];