employer
/
tame
1
0
Fork 0
Code Packages Releases Activity

TAMER: Symbol source data and metadata

master
Mike Gerwitz 2020-01-13 15:15:38 -05:00
parent bcc2ab1221
commit 85a4934db5
8 changed files with 311 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
tamer/src/ir/asg/base.rs
View File

@ -19,7 +19,7 @@
use super::graph::{Asg, AsgEdge, AsgError, AsgResult, Node, ObjectRef};
use super::ident::IdentKind;
use super::object::{FragmentText, Object};
use super::object::{FragmentText, Object, Source};
use crate::sym::Symbol;
use fixedbitset::FixedBitSet;
use petgraph::graph::{
@ -125,13 +125,14 @@ where
&mut self,
name: &'i Symbol<'i>,
kind: IdentKind,
src: Source<'i>,
) -> AsgResult<ObjectRef<Ix>> {
// TODO: src check
if let Some(existing) = self.lookup(name) {
return Ok(existing);
}
let node = self.graph.add_node(Some(Object::Ident(name, kind)));
let node = self.graph.add_node(Some(Object::Ident(name, kind, src)));
self.index_identifier(name, node);
@ -172,8 +173,8 @@ where
.expect("internal error: BaseAsg::set_fragment missing Node data");
let result = match ty {
Object::Ident(sym, kind) => {
Ok(Object::IdentFragment(sym, kind, text))
Object::Ident(sym, kind, src) => {
Ok(Object::IdentFragment(sym, kind, src, text))
}
_ => {
let err = Err(AsgError::BadFragmentDest(format!(
@ -287,18 +288,47 @@ mod test {
let syma = Symbol::new_dummy(SymbolIndex::from_u32(5), "syma");
let symb = Symbol::new_dummy(SymbolIndex::from_u32(1), "symab");
let nodea = sut.declare(&syma, IdentKind::Meta)?;
let nodeb = sut.declare(&symb, IdentKind::Worksheet)?;
let nodea = sut.declare(
&syma,
IdentKind::Meta,
Source {
desc: Some("a".to_string()),
..Default::default()
},
)?;
let nodeb = sut.declare(
&symb,
IdentKind::Worksheet,
Source {
desc: Some("b".to_string()),
..Default::default()
},
)?;
assert_ne!(nodea, nodeb);
assert_eq!(
Some(&Object::Ident(&syma, IdentKind::Meta)),
Some(&Object::Ident(
&syma,
IdentKind::Meta,
Source {
desc: Some("a".to_string()),
..Default::default()
},
)),
sut.get(nodea),
);
assert_eq!(
Some(&Object::Ident(&symb, IdentKind::Worksheet)),
Some(&Object::Ident(
&symb,
IdentKind::Worksheet,
Source {
desc: Some("b".to_string()),
..Default::default()
},
)),
sut.get(nodeb),
);
@ -310,7 +340,14 @@ mod test {
let mut sut = Sut::with_capacity(0, 0);
let sym = Symbol::new_dummy(SymbolIndex::from_u32(1), "lookup");
let node = sut.declare(&sym, IdentKind::Meta)?;
let node = sut.declare(
&sym,
IdentKind::Meta,
Source {
generated: true,
..Default::default()
},
)?;
assert_eq!(Some(node), sut.lookup(&sym));
@ -335,10 +372,11 @@ mod test {
let mut sut = Sut::with_capacity(0, 0);
let sym = Symbol::new_dummy(SymbolIndex::from_u32(1), "symdup");
let node = sut.declare(&sym, IdentKind::Meta)?;
let node = sut.declare(&sym, IdentKind::Meta, Source::default())?;
// Same declaration a second time
let redeclare = sut.declare(&sym, IdentKind::Meta)?;
let redeclare =
sut.declare(&sym, IdentKind::Meta, Source::default())?;
assert_eq!(node, redeclare);
Ok(())
@ -349,7 +387,11 @@ mod test {
let mut sut = Sut::with_capacity(0, 0);
let sym = Symbol::new_dummy(SymbolIndex::from_u32(1), "tofrag");
let node = sut.declare(&sym, IdentKind::Meta)?;
let src = Source {
generated: true,
..Default::default()
};
let node = sut.declare(&sym, IdentKind::Meta, src.clone())?;
let fragment = "a fragment".to_string();
let node_with_frag = sut.set_fragment(node, fragment.clone())?;
@ -362,7 +404,7 @@ mod test {
);
assert_eq!(
Some(&Object::IdentFragment(&sym, IdentKind::Meta, fragment)),
Some(&Object::IdentFragment(&sym, IdentKind::Meta, src, fragment)),
sut.get(node)
);
@ -374,7 +416,7 @@ mod test {
let mut sut = Sut::with_capacity(0, 0);
let sym = Symbol::new_dummy(SymbolIndex::from_u32(1), "sym");
let node = sut.declare(&sym, IdentKind::Meta)?;
let node = sut.declare(&sym, IdentKind::Meta, Source::default())?;
let fragment = "orig fragment".to_string();
sut.set_fragment(node, fragment.clone())?;
@ -391,7 +433,12 @@ mod test {
// Make sure we didn't leave the node in an inconsistent state
assert_eq!(
Some(&Object::IdentFragment(&sym, IdentKind::Meta, fragment)),
Some(&Object::IdentFragment(
&sym,
IdentKind::Meta,
Default::default(),
fragment
)),
sut.get(node)
);
@ -405,8 +452,8 @@ mod test {
let sym = Symbol::new_dummy(SymbolIndex::from_u32(1), "sym");
let dep = Symbol::new_dummy(SymbolIndex::from_u32(1), "dep");
let symnode = sut.declare(&sym, IdentKind::Meta)?;
let depnode = sut.declare(&dep, IdentKind::Meta)?;
let symnode = sut.declare(&sym, IdentKind::Meta, Source::default())?;
let depnode = sut.declare(&dep, IdentKind::Meta, Source::default())?;
sut.add_dep(symnode, depnode);
assert!(sut.has_dep(symnode, depnode));

3
tamer/src/ir/asg/graph.rs
View File

@ -18,7 +18,7 @@
//! Abstract graph as the basis for concrete ASGs.
use super::ident::IdentKind;
use super::object::{FragmentText, Object};
use super::object::{FragmentText, Object, Source};
use crate::sym::Symbol;
use petgraph::graph::{IndexType, NodeIndex};
use std::result::Result;
@ -65,6 +65,7 @@ pub trait Asg<'i, Ix: IndexType> {
&mut self,
name: &'i Symbol<'i>,
kind: IdentKind,
src: Source<'i>,
) -> AsgResult<ObjectRef<Ix>>;
/// Declare an abstract identifier.

43
tamer/src/ir/asg/ident.rs
View File

@ -150,6 +150,18 @@ impl<'i> TryFrom<SymAttrs<'i>> for IdentKind {
/// Certain [`IdentKind`] require that certain attributes be present,
/// otherwise the conversion will fail.
fn try_from(attrs: SymAttrs<'i>) -> Result<Self, Self::Error> {
Self::try_from(&attrs)
}
}
impl<'i> TryFrom<&SymAttrs<'i>> for IdentKind {
type Error = &'static str;
/// Attempt to raise [`SymAttrs`] into an [`IdentKind`].
///
/// Certain [`IdentKind`] require that certain attributes be present,
/// otherwise the conversion will fail.
fn try_from(attrs: &SymAttrs<'i>) -> Result<Self, Self::Error> {
let ty = attrs.ty.as_ref().ok_or("missing symbol type")?;
macro_rules! ident {
@ -205,6 +217,27 @@ impl<'i> TryFrom<SymAttrs<'i>> for IdentKind {
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
pub struct Dim(u8);
/// Underlying datatype of identifier.
///
/// TODO: This will always be 0≤n≤9, so let's introduce a newtype for it.
impl AsRef<str> for Dim {
fn as_ref(&self) -> &str {
match self.0 {
0 => &"0",
1 => &"1",
2 => &"2",
3 => &"3",
4 => &"4",
5 => &"5",
6 => &"6",
7 => &"7",
8 => &"8",
9 => &"9",
_ => unreachable!(),
}
}
}
/// Underlying datatype of identifier.
pub type DataType = SymDtype;
@ -213,6 +246,16 @@ mod test {
use super::*;
use std::convert::TryInto;
#[test]
fn dim_to_str() {
// we'll just test high and low
let low: &str = Dim(0).as_ref();
let high: &str = Dim(9).as_ref();
assert_eq!("0", low);
assert_eq!("9", high);
}
macro_rules! test_kind {
($name:ident, $src:expr => $dest:expr) => {
#[test]

13
tamer/src/ir/asg/mod.rs
View File

@ -59,7 +59,7 @@
//!
//! ```
//! use tamer::global;
//! use tamer::ir::asg::{Asg, DefaultAsg, IdentKind, Object};
//! use tamer::ir::asg::{Asg, DefaultAsg, IdentKind, Object, Source};
//! use tamer::sym::{Interner, DefaultInterner};
//!
//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
@ -71,7 +71,7 @@
//! let identa_sym = interner.intern("identa");
//! let identb_sym = interner.intern("identb");
//!
//! let identa = asg.declare(identa_sym, IdentKind::Meta)?;
//! let identa = asg.declare(identa_sym, IdentKind::Meta, Source::default())?;
//! let identb = asg.declare_extern(identb_sym, IdentKind::Meta)?;
//!
//! assert_eq!(
@ -103,7 +103,7 @@
//!
//! ```
//! # use tamer::global;
//! # use tamer::ir::asg::{Asg, DefaultAsg, IdentKind, Object, FragmentText};
//! # use tamer::ir::asg::{Asg, DefaultAsg, IdentKind, Object, FragmentText, Source};
//! # use tamer::sym::{Interner, DefaultInterner};
//! #
//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
@ -111,13 +111,16 @@
//! # let interner = DefaultInterner::new();
//! #
//! // Fragments can be attached to resolved identifiers.
//! let ident = asg.declare(interner.intern("ident"), IdentKind::Meta)?;
//! let ident = asg.declare(
//! interner.intern("ident"), IdentKind::Meta, Source::default()
//! )?;
//! asg.set_fragment(ident, FragmentText::from("test fragment"))?;
//!
//! assert_eq!(
//! Some(&Object::IdentFragment(
//! interner.intern("ident"),
//! IdentKind::Meta,
//! Source::default(),
//! FragmentText::from("test fragment"),
//! )),
//! asg.get(ident),
@ -138,7 +141,7 @@ mod object;
pub use graph::{Asg, AsgResult, ObjectRef};
pub use ident::IdentKind;
pub use object::{FragmentText, Object};
pub use object::{FragmentText, Object, Source};
/// Default concrete ASG implementation.
pub type DefaultAsg<'i, Ix> = base::BaseAsg<'i, Ix>;

91
tamer/src/ir/asg/object.rs
View File

@ -21,6 +21,7 @@
//! See [`super`] for available exports._
use super::ident::IdentKind;
use crate::ir::legacyir::SymAttrs;
use crate::sym::Symbol;
/// Type of object.
@ -38,15 +39,13 @@ use crate::sym::Symbol;
/// through [`Asg`][super::Asg]'s public API,
/// as it represents the absence of an object at that node within the
/// ASG.
///
/// TODO: Source location (span; see Rustc).
#[derive(Debug, PartialEq)]
pub enum Object<'i> {
/// A resolved identifier.
///
/// This represents an identifier that has been declared with certain
/// type information.
Ident(&'i Symbol<'i>, IdentKind),
Ident(&'i Symbol<'i>, IdentKind, Source<'i>),
/// An identifier that has not yet been resolved.
///
@ -63,7 +62,7 @@ pub enum Object<'i> {
/// They are produced by the compiler and it is the job of the
/// [linker][crate::ld] to put them into the correct order for the
/// final executable.
IdentFragment(&'i Symbol<'i>, IdentKind, FragmentText),
IdentFragment(&'i Symbol<'i>, IdentKind, Source<'i>, FragmentText),
/// The empty node (default value for indexer).
///
@ -75,3 +74,87 @@ pub enum Object<'i> {
///
/// This represents the text associated with an identifier.
pub type FragmentText = String;
/// Metadata about the source of an object.
///
/// This contains information from the symbol table that does not belong on
/// [`IdentKind`],
/// since that stores _type_ information.
///
/// TODO: This does not currently store byte offsets within the source file
/// since the original XSLT-based compiler did not have that capability;
/// this will provide that information in the future.
#[derive(Debug, Default, PartialEq, Clone)]
pub struct Source<'i> {
/// The identifier from which this one is derived.
///
/// See [`IdentKind`] for more information on parents.
/// For example,
/// a [`IdentKind::Cgen`] always has a parent [`IdentKind::Class`].
pub parent: Option<&'i Symbol<'i>>,
/// Child identifier associated with this identifier.
///
/// For [`IdentKind::Class`],
/// this represents an associated [`IdentKind::Cgen`].
pub yields: Option<&'i Symbol<'i>>,
/// User-friendly identifier description.
///
/// This is used primarily by [`IdentKind::Class`] and
/// [`IdentKind::Gen`].
pub desc: Option<String>,
/// Whether this identifier was generated by the compiler.
///
/// A generated identifier is representative of an internal
/// implementation detail that should remain encapsulated from the
/// user and is subject to change over time.
///
/// Identifiers created by templates are not considered to be generated.
pub generated: bool,
}
impl<'i> From<SymAttrs<'i>> for Source<'i> {
/// Raise Legacy IR [`SymAttrs`].
///
/// This simply extracts a subset of fields from the source attributes.
fn from(attrs: SymAttrs<'i>) -> Self {
Source {
generated: attrs.generated,
parent: attrs.parent,
yields: attrs.yields,
desc: attrs.desc,
}
}
}
#[cfg(test)]
mod test {
use super::*;
use crate::sym::SymbolIndex;
#[test]
fn source_from_sym_attrs() {
let psym = Symbol::new_dummy(SymbolIndex::from_u32(1), "parent");
let ysym = Symbol::new_dummy(SymbolIndex::from_u32(2), "yields");
let attrs = SymAttrs {
generated: true,
parent: Some(&psym),
yields: Some(&ysym),
desc: Some("sym desc".to_string()),
..Default::default()
};
assert_eq!(
Source {
generated: attrs.generated,
parent: attrs.parent,
yields: attrs.yields,
desc: Some("sym desc".to_string()),
},
attrs.into(),
);
}
}

27
tamer/src/ir/legacyir.rs
View File

@ -104,6 +104,33 @@ pub struct SymAttrs<'i> {
/// The linker (see [`crate::ld`]) is responsible for ensuring that the
/// extern is satisfied and properly located in the final executable.
pub extern_: bool,
/// The identifier from which this one is derived.
///
/// For example,
/// [`SymType::Cgen`] has a parent [`SymType::Class`] and
/// [`SymType::Gen`] has a parent [`SymType::Rate`].
pub parent: Option<&'i Symbol<'i>>,
/// Whether this identifier was generated by the compiler.
///
/// A generated identifier is representative of an internal
/// implementation detail that should remain encapsulated from the
/// user and is subject to change over time.
///
/// Identifiers created by templates are not considered to be generated.
pub generated: bool,
/// Child identifier associated with this identifier.
///
/// For [`SymType::Class`],
/// this represents an associated [`SymType::Cgen`].
pub yields: Option<&'i Symbol<'i>>,
/// User-friendly identifier description.
///
/// This is used primarily by [`SymType::Class`] and [`SymType::Gen`].
pub desc: Option<String>,
}
/// Legacy symbol types.

6
tamer/src/ld/poc.rs
View File

@ -135,10 +135,12 @@ fn load_xmlo<'a, 'i, I: Interner<'i>>(
let owned = attrs.src.is_none();
let kind = attrs.try_into().map_err(|err| {
let kind = (&attrs).try_into().map_err(|err| {
format!("sym `{}` attrs error: {}", sym, err)
});
let src = attrs.into();
// TODO: should probably track these down in the XSLT linker...
match kind {
Ok(kindval) => {
@ -148,7 +150,7 @@ fn load_xmlo<'a, 'i, I: Interner<'i>>(
|| sym.starts_with(":map:")
|| sym.starts_with(":retmap:"));
let node = depgraph.declare(sym, kindval)?;
let node = depgraph.declare(sym, kindval, src)?;
if link_root {
roots.push(node);

76
tamer/src/obj/xmlo/reader.rs
View File

@ -392,6 +392,28 @@ impl<'i, B: BufRead, I: Interner<'i>> XmloReader<'i, B, I> {
sym_attrs.extern_ = &*attr.value == b"true";
}
b"preproc:generated" => {
sym_attrs.generated = &*attr.value == b"true";
}
b"parent" => {
sym_attrs.parent = Some(unsafe {
interner.intern_utf8_unchecked(&attr.value)
});
}
b"yields" => {
sym_attrs.yields = Some(unsafe {
interner.intern_utf8_unchecked(&attr.value)
});
}
b"desc" => {
sym_attrs.desc = Some(unsafe {
String::from_utf8_unchecked(attr.value.to_vec())
});
}
// As this reader evolves, we may wish to provide an error
// for unknown attributes so that we can be sure that we've
// handled them all.
@ -1411,6 +1433,21 @@ mod test {
..Default::default()
}
parent: [parent="foo"] => SymAttrs {
parent: Some(interner.intern("foo")),
..Default::default()
}
yields: [yields="yield"] => SymAttrs {
yields: Some(interner.intern("yield")),
..Default::default()
}
desc: [desc="Description"] => SymAttrs {
desc: Some("Description".to_string()),
..Default::default()
}
// Multiple attributes at once
multi: [src="foo", type="class", dim="1", dtype="float", extern="true"]
=> SymAttrs {
@ -1419,9 +1456,48 @@ mod test {
dim: Some(1),
dtype: Some(SymDtype::Float),
extern_: true,
..Default::default()
}
}
// can't be tested using the above
#[test]
fn generated_true() -> XmloResult<()> {
let stub_data: &[u8] = &[];
let interner = DefaultInterner::new();
let mut sut = Sut::new(stub_data, &interner);
// See xmlo_tests macro for explanation
sut.seen_root = true;
sut.reader.next_event = Some(Box::new(|_, _| {
Ok(XmlEvent::Empty(MockBytesStart::new(
b"preproc:sym",
Some(MockAttributes::new(vec![
MockAttribute::new(b"name", b"generated_true"),
MockAttribute::new(b"preproc:generated", b"true"),
])),
)))
}));
let result = sut.read_event()?;
let expected_attrs = SymAttrs {
generated: true,
..Default::default()
};
assert_eq!(
XmloEvent::SymDecl(
interner.intern("generated_true"),
expected_attrs
),
result
);
Ok(())
}
#[test]
fn fails_on_non_ascii_dim() {
let stub_data: &[u8] = &[];