employer
/
tame
1
0
Fork 0
Code Packages Releases Activity

[DEV-7504] Add GraphML generation 

Merge branch 'jira-7504'

* jira-7504:
  [DEV-7504] Update RELEASES.md to make it less technical
  [DEV-7504] Add cypher script for post-graph import
  [DEV-7504] Add make target for "graphml"
  [DEV-7504] Add GraphML generation
master
Joseph Frazer 2020-05-13 08:05:57 -04:00
parent 18d87a6b00 c72409d617
commit 45b7adbf54
10 changed files with 252 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
RELEASES.md
View File

@ -16,9 +16,20 @@ commits that introduce the changes. To make a new release, run
NEXT
====
This release adds GraphML output for linked objects to allow us to
inspect the graph.
Linker
------
- Add `--emit` oprion to `tamer/src/bin/tameld.rs` that allows us to specify
the type of output we want.
- Minor refactoring.
Miscellaneous
-------------
- Added `make` target to build linked GraphML files.
- Updated `make *.xmle` target to explicitly state it is emitting `xmle`.
- Added Cypher script to use in Neo4J after a GraphML file is imported.
- `RELEASES.md`
- Add missing link to semver.org.
- Fix `tame-core` heading, which was erroneously Org-mode-styled.

6
build-aux/Makefile.am
View File

@ -117,6 +117,10 @@ program-ui: ui/package.strip.js ui/Program.js ui/html/index.phtml
%.xmli: %.xml
$(path_tame)/tamer/target/release/tamec --emit xmlo -o $@ $<
%.graphml: %.xmlo
$(TAME_TS)
$(path_tame)/tamer/target/release/tameld --emit graphml -o $@ $<
# Individual dependencies appear in suppliers.mk (see below)
%.xmlo: %.xmli $(path_tame)/.rev-xmlo
$(TAME_TS)
@ -136,7 +140,7 @@ standalones: $(dest_standalone)
strip: $(dest_standalone_strip) ui/package.strip.js
%.xmle: %.xmlo $(path_tame)/.rev-xmle
$(TAME_TS)
$(path_tame)/tamer/target/release/tameld -o $@ $<
$(path_tame)/tamer/target/release/tameld --emit xmle -o $@ $<
%.js: %.xmle
$(TAME_TS)
$(TAME) standalone $< $@

17
tamer/Cargo.lock generated
View File

@ -127,6 +127,15 @@ dependencies = [
"indexmap 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "petgraph-graphml"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"petgraph 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
"xml-rs 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "predicates"
version = "1.0.4"
@ -247,6 +256,7 @@ dependencies = [
"getopts 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"petgraph 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
"petgraph-graphml 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"predicates 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
"quick-xml 0.18.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -274,6 +284,11 @@ name = "unicode-xid"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "xml-rs"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
[metadata]
"checksum aho-corasick 0.7.10 (registry+https://github.com/rust-lang/crates.io-index)" = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada"
"checksum assert_cmd 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b7ac5c260f75e4e4ba87b7342be6edcecbcb3eb6741a0507fda7ad115845cc65"
@ -294,6 +309,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum normalize-line-endings 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be"
"checksum num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096"
"checksum petgraph 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "29c127eea4a29ec6c85d153c59dc1213f33ec74cead30fe4730aecc88cc1fd92"
"checksum petgraph-graphml 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f1a24b19072d6f11cec958e21af7f80d45ee135e9d459b7703510aaf0550b47b"
"checksum predicates 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "347a1b6f0b21e636bc9872fb60b83b8e185f6f5516298b8238699f7f9a531030"
"checksum predicates-core 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "06075c3a3e92559ff8929e7a280684489ea27fe44805174c3ebd9328dcb37178"
"checksum predicates-tree 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8e63c4859013b38a76eca2414c64911fba30def9e3202ac461a2d22831220124"
@ -311,3 +327,4 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum treeline 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41"
"checksum unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479"
"checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c"
"checksum xml-rs 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b07db065a5cf61a7e4ba64f29e67db906fb1787316516c4e6e5ff0fea1efcd8a"

1
tamer/Cargo.toml
View File

@ -34,4 +34,5 @@ quick-xml = ">= 0.17.0"
getopts = "0.2"
exitcode = "1.1.2"
lazy_static = ">= 1.4.0"
petgraph-graphml = ">= 2.0.1"

104
tamer/src/bin/tameld.rs
View File

@ -33,27 +33,41 @@ use tamer::ld::poc;
/// Types of commands
enum Command {
Link(String, String),
Link(String, String, Emit),
Usage,
}
/// Ways to emit the linked objects
enum Emit {
/// The typical desired `Emit`
///
/// Outputs the linked object files in a format that can be used in an
/// application.
Xmle,
/// Used for exploring the linked graph
Graphml,
}
/// Entrypoint for the linker
pub fn main() -> Result<(), Box<dyn Error>> {
let args: Vec<String> = env::args().collect();
let program = &args[0];
let opts = get_opts();
let usage = opts.usage(&format!("Usage: {} -o OUTPUT FILE", program));
let usage =
opts.usage(&format!("Usage: {} [OPTIONS] -o OUTPUT FILE", program));
match parse_options(opts, args) {
Ok(Command::Link(input, output)) => match poc::main(&input, &output) {
Err(e) => {
eprintln!("error: {}", e);
eprintln!("fatal: failed to link `{}`", output);
Ok(Command::Link(input, output, emit)) => match emit {
Emit::Xmle => poc::xmle(&input, &output),
Emit::Graphml => poc::graphml(&input, &output),
}
.or_else(|e| {
eprintln!("error: {}", e);
eprintln!("fatal: failed to link `{}`", output);
std::process::exit(1);
}
ok => ok,
},
std::process::exit(1);
}),
Ok(Command::Usage) => {
println!("{}", usage);
std::process::exit(exitcode::OK);
@ -77,6 +91,12 @@ fn get_opts() -> Options {
let mut opts = Options::new();
opts.optopt("o", "output", "set output file name", "NAME");
opts.optflag("h", "help", "print this help menu");
opts.optopt(
"",
"emit",
"set the output to be emitted",
"--emit xmle|graphml",
);
opts
}
@ -107,7 +127,16 @@ fn parse_options(opts: Options, args: Vec<String>) -> Result<Command, Fail> {
}
};
Ok(Command::Link(input, output))
let emit = match matches.opt_str("emit") {
Some(m) => match &m[..] {
"xmle" => Emit::Xmle,
"graphml" => Emit::Graphml,
em => return Err(Fail::ArgumentMissing(format!("--emit {}", em))),
},
None => Emit::Xmle,
};
Ok(Command::Link(input, output, emit))
}
#[cfg(test)]
@ -207,6 +236,29 @@ mod test {
}
}
#[test]
fn parse_options_valid_long_emit_invalid() {
let opts = get_opts();
let result = parse_options(
opts,
vec![
String::from("program"),
String::from("foo"),
String::from("--output"),
String::from("bar"),
String::from("--emit"),
String::from("foo"),
],
);
match result {
Err(Fail::ArgumentMissing(message)) => {
assert_eq!("--emit foo", message);
}
_ => panic!("Extra option not caught"),
}
}
#[test]
fn parse_options_valid() {
let opts = get_opts();
@ -221,7 +273,7 @@ mod test {
);
match result {
Ok(Command::Link(infile, outfile)) => {
Ok(Command::Link(infile, outfile, Emit::Xmle)) => {
assert_eq!("foo", infile);
assert_eq!("bar", outfile);
}
@ -239,11 +291,37 @@ mod test {
String::from("foo"),
String::from("--output"),
String::from("bar"),
String::from("--emit"),
String::from("xmle"),
],
);
match result {
Ok(Command::Link(infile, outfile)) => {
Ok(Command::Link(infile, outfile, Emit::Xmle)) => {
assert_eq!("foo", infile);
assert_eq!("bar", outfile);
}
_ => panic!("Unexpected result"),
}
}
#[test]
fn parse_options_valid_long_emit_graphml() {
let opts = get_opts();
let result = parse_options(
opts,
vec![
String::from("program"),
String::from("foo"),
String::from("--output"),
String::from("bar"),
String::from("--emit"),
String::from("graphml"),
],
);
match result {
Ok(Command::Link(infile, outfile, Emit::Graphml)) => {
assert_eq!("foo", infile);
assert_eq!("bar", outfile);
}

5
tamer/src/ir/asg/base.rs
View File

@ -97,6 +97,11 @@ where
}
}
/// Get the underlying Graph
pub fn into_inner(self) -> DiGraph<Node<O>, AsgEdge, Ix> {
self.graph
}
/// Index the provided symbol `name` as representing the identifier `node`.
///
/// This index permits `O(1)` identifier lookups.

63
tamer/src/ir/asg/ident.rs
View File

@ -145,6 +145,31 @@ pub enum IdentKind {
Worksheet,
}
impl AsRef<str> for IdentKind {
fn as_ref(&self) -> &'static str {
match self {
Self::Cgen(_) => "cgen",
Self::Class(_) => "class",
Self::Const(_, _) => "const",
Self::Func(_, _) => "func",
Self::Gen(_, _) => "gen",
Self::Lparam(_, _) => "lparam",
Self::Param(_, _) => "param",
Self::Rate(_) => "rate",
Self::Tpl => "tpl",
Self::Type(_) => "type",
Self::MapHead => "map:head",
Self::Map => "map",
Self::MapTail => "map:tail",
Self::RetMapHead => "retmap:head",
Self::RetMap => "retmap",
Self::RetMapTail => "retmap:tail",
Self::Meta => "meta",
Self::Worksheet => "worksheet",
}
}
}
impl std::fmt::Display for IdentKind {
/// Format identifier type for display to the user.
///
@ -152,31 +177,33 @@ impl std::fmt::Display for IdentKind {
/// new type system,
/// so for now this just uses a syntax similar to Rust.
fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
let name = self.as_ref();
match self {
Self::Cgen(dim) => {
write!(fmt, "cgen[{}; {}]", DataType::Boolean, dim)
write!(fmt, "{}[{}; {}]", name, DataType::Boolean, dim)
}
Self::Class(dim) => {
write!(fmt, "class[{}; {}]", DataType::Boolean, dim)
write!(fmt, "{}[{}; {}]", name, DataType::Boolean, dim)
}
Self::Const(dim, dtype) => {
write!(fmt, "{}[{}; {}]", name, dtype, dim)
}
Self::Func(dim, dtype) => {
write!(fmt, "{}[{}; {}]", name, dtype, dim)
}
Self::Gen(dim, dtype) => {
write!(fmt, "{}[{}; {}]", name, dtype, dim)
}
Self::Const(dim, dtype) => write!(fmt, "const[{}; {}]", dtype, dim),
Self::Func(dim, dtype) => write!(fmt, "func[{}; {}]", dtype, dim),
Self::Gen(dim, dtype) => write!(fmt, "gen[{}; {}]", dtype, dim),
Self::Lparam(dim, dtype) => {
write!(fmt, "lparam[{}; {}]", dtype, dim)
write!(fmt, "{}[{}; {}]", name, dtype, dim)
}
Self::Param(dim, dtype) => write!(fmt, "param[{}; {}]", dtype, dim),
Self::Rate(dtype) => write!(fmt, "rate[{}; 0]", dtype),
Self::Tpl => write!(fmt, "tpl"),
Self::Type(dtype) => write!(fmt, "type[{}]", dtype),
Self::MapHead => write!(fmt, "map:head"),
Self::Map => write!(fmt, "map"),
Self::MapTail => write!(fmt, "map:tail"),
Self::RetMapHead => write!(fmt, "retmap:head"),
Self::RetMap => write!(fmt, "retmap"),
Self::RetMapTail => write!(fmt, "retmap:tail"),
Self::Meta => write!(fmt, "meta"),
Self::Worksheet => write!(fmt, "worksheet"),
Self::Param(dim, dtype) => {
write!(fmt, "{}[{}; {}]", name, dtype, dim)
}
Self::Rate(dtype) => write!(fmt, "{}[{}; 0]", name, dtype),
Self::Type(dtype) => write!(fmt, "{}[{}]", name, dtype),
_ => write!(fmt, "{}", name),
}
}
}

9
tamer/src/ir/asg/object.rs
View File

@ -135,6 +135,15 @@ pub trait IdentObjectData<'i> {
fn as_ident(&self) -> Option<&IdentObject<'i>>;
}
impl<'i> std::fmt::Display for IdentObject<'i> {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self.name() {
Some(n) => write!(f, "{}", n),
_ => write!(f, "missing"),
}
}
}
impl<'i> IdentObjectData<'i> for IdentObject<'i> {
fn name(&self) -> Option<&'i Symbol<'i>> {
match self {

58
tamer/src/ld/poc.rs
View File

@ -32,6 +32,7 @@ use crate::obj::xmle::writer::XmleWriter;
use crate::obj::xmlo::{AsgBuilder, AsgBuilderState, XmloReader};
use crate::sym::{DefaultInterner, Interner, Symbol};
use fxhash::FxBuildHasher;
use petgraph_graphml::GraphMl;
use std::error::Error;
use std::fs;
use std::io::BufReader;
@ -42,7 +43,7 @@ type LinkerAsg<'i> = DefaultAsg<'i, IdentObject<'i>, global::ProgIdentSize>;
type LinkerAsgBuilderState<'i> =
AsgBuilderState<'i, FxBuildHasher, global::ProgIdentSize>;
pub fn main(package_path: &str, output: &str) -> Result<(), Box<dyn Error>> {
pub fn xmle(package_path: &str, output: &str) -> Result<(), Box<dyn Error>> {
let mut fs = VisitOnceFilesystem::new();
let mut depgraph = LinkerAsg::with_capacity(65536, 65536);
let interner = DefaultInterner::new();
@ -96,8 +97,6 @@ pub fn main(package_path: &str, output: &str) -> Result<(), Box<dyn Error>> {
Err(e) => return Err(e.into()),
};
//println!("Sorted ({}): {:?}", sorted.len(), sorted);
output_xmle(
&depgraph,
&interner,
@ -110,6 +109,59 @@ pub fn main(package_path: &str, output: &str) -> Result<(), Box<dyn Error>> {
Ok(())
}
pub fn graphml(package_path: &str, output: &str) -> Result<(), Box<dyn Error>> {
let mut fs = VisitOnceFilesystem::new();
let mut depgraph = LinkerAsg::with_capacity(65536, 65536);
let interner = DefaultInterner::new();
let _ = load_xmlo(
package_path,
&mut fs,
&mut depgraph,
&interner,
AsgBuilderState::new(),
)?;
// if we move away from petgraph, we will need to abstract this away
let g = depgraph.into_inner();
let graphml =
GraphMl::new(&g)
.pretty_print(true)
.export_node_weights(Box::new(|node| {
// eprintln!("{:?}", node);
let (name, kind, generated) = match node {
Some(n) => {
let generated = match n.src() {
Some(src) => src.generated,
None => false,
};
(
format!("{}", n),
n.kind().unwrap().as_ref(),
format!("{}", generated),
)
}
None => (
String::from("missing"),
"missing",
format!("{}", false),
),
};
vec![
("label".into(), name.into()),
("kind".into(), kind.into()),
("generated".into(), generated.into()),
]
}));
fs::write(output, graphml.to_string())?;
Ok(())
}
fn load_xmlo<'a, 'i, I: Interner<'i>, P: AsRef<Path>>(
path_str: P,
fs: &mut VisitOnceFilesystem<FsCanonicalizer, FxBuildHasher>,

13
tamer/tests/tameld.rs
View File

@ -67,6 +67,19 @@ fn link_input_file_does_not_exist() -> Result<(), Box<dyn std::error::Error>> {
Ok(())
}
#[test]
fn link_invalid_emit() -> Result<(), Box<dyn std::error::Error>> {
let mut cmd = Command::cargo_bin("tameld")?;
cmd.arg("foobar");
cmd.arg("--emit").arg("notgood");
cmd.arg("-o").arg("tests/data/test-output.xmle");
cmd.assert()
.failure()
.stderr(predicate::str::contains("--emit notgood"));
Ok(())
}
#[test]
fn link_empty_input_file() -> Result<(), Box<dyn std::error::Error>> {
let mut cmd = Command::cargo_bin("tameld")?;