// Proof-of-concept TAME linker // // Copyright (C) 2014-2019 Ryan Specialty Group, LLC. // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program. If not, see . //! **This is a poorly-written proof of concept; do not use!** It has been //! banished to its own file to try to make that more clear. use crate::obj::xmlo::reader::{XmloEvent, XmloReader}; use crate::sym::{DefaultInterner, Interner}; use fixedbitset::FixedBitSet; use petgraph::graph::{DiGraph, EdgeIndex, Neighbors, NodeIndex}; use petgraph::visit::{DfsPostOrder, GraphBase, IntoNeighbors, Visitable}; use std::collections::hash_map::{Entry, Iter}; use std::collections::{HashMap, HashSet}; use std::error::Error; use std::fs; use std::io::BufReader; use std::ops::{Deref, Index}; use std::rc::Rc; // The term "sym" is used throughout because it's easier to search for that // in source code than "symbol", which is a generic term with many different // meanings. // if mutability is needed: //#[derive(Debug)] //struct SymRecord { // data: SymData, // // // the idea is to keep the index encapsulated so that nothing else can // // ever hold a reference to it, ensuring that it's freed when the node // // is removed // index: Rc>>, //} #[derive(Debug)] struct SymData { name: Rc, } type DepGraphNode = SymEntry; type DepGraphEdge = (); struct DepGraph { graph: DiGraph, // serves as both a string internment system and graph indexer index: HashMap, SymRef>, // if removals are permitted: //index: HashMap, Weak>>>, } // This encapsulates the underlying Graph to enforce certain // assumptions. For example, we do not permit removing nodes because that // would invalidate the NodeIndex reference in the index, which would then // require workarounds like the commented-out code above and below. // // While Petgraph's use of indexes to represent graph and edge references // makes it easy to bypass the borrow checker, it does just that---it's no // different than a pointer reference (albeit guaranteed to safely reference // a node rather than an arbitrary memory location) that can change out from // under you at any moment. As such, much of the planning that went into // this was determining how to best mitigate that. // // The linker has certain needs that may differ as the compiler evolves, so // it may be desirable to permit deletions in the future. In the meantime, // if a node needs to be deleted, we can simply remove all edges from it and // possibly mark it in a way that states it was removed. // // This graph uses a separate map to serve a dual role: a string internment // system and an indexer by symbol name. This will have to evolve in the // future as the graph ends up containing more stuff. // // This is currently called a dependency graph, since that's what we're // using it for, but in the future the compiler will also use it as an IR, // so this will likely be renamed. impl DepGraph { fn new() -> Self { Self { // TODO: with_capacity graph: DiGraph::new(), index: HashMap::new(), } } fn declare(&mut self, name: &str) -> SymRef { match self.index.entry(name.into()) { Entry::Occupied(o) => *o.get(), Entry::Vacant(v) => { let entry = SymEntry::MissingSym { name: Rc::clone(v.key()), }; let index = SymRef(self.graph.add_node(entry)); v.insert(index); index } } } // will not duplicate dependencies if they already exist fn declare_dep(&mut self, symbol: SymRef, dep: SymRef) -> () { self.graph.update_edge(*symbol, *dep, ()); } fn lookup(&self, name: &str) -> Option { self.index.get(name.into()).map(|index| *index) } fn index_iter(&self) -> Iter, SymRef> { self.index.iter() } // POC when removals were permitted: //fn add_symbol(&mut self, sym: SymData) -> NodeIndex { // let name = Rc::clone(&sym.name); // let record = SymRecord { data: sym, index: Rc::new(RefCell::new(None)) }; // let index = self.graph.add_node(record); // let index = Rc::downgrade(&self.graph[index].index); // self.graph[index].index.replace(Some(index)); // self.index.insert(name, index); // index //} } impl GraphBase for DepGraph { type NodeId = NodeIndex; type EdgeId = EdgeIndex; } impl Visitable for DepGraph { type Map = FixedBitSet; fn visit_map(&self) -> Self::Map { self.graph.visit_map() } fn reset_map(&self, map: &mut Self::Map) { self.graph.reset_map(map) } } impl<'a> IntoNeighbors for &'a DepGraph { type Neighbors = Neighbors<'a, DepGraphEdge>; fn neighbors(self, n: Self::NodeId) -> Self::Neighbors { self.graph.neighbors(n) } } impl Index for DepGraph { type Output = DepGraphNode; fn index(&self, index: SymRef) -> &Self::Output { &self.graph[*index] } } // TODO: we may not to allow this; using SymRef could be a means to // guarantee that a lookup has occurred and that it actually exists. We // don't need this if we set NodeId = SymRef in GraphBase, but that requires // implementing other traits as well. impl Index for DepGraph { type Output = DepGraphNode; fn index(&self, index: NodeIndex) -> &Self::Output { &self.graph[index] } } #[derive(Debug, Clone, Copy, PartialEq)] struct SymRef(NodeIndex); impl From for NodeIndex { fn from(symref: SymRef) -> Self { *symref } } impl From for SymRef { fn from(index: NodeIndex) -> Self { Self(index) } } impl Deref for SymRef { type Target = NodeIndex; fn deref(&self) -> &Self::Target { &self.0 } } #[derive(Debug, PartialEq)] enum SymEntry { MissingSym { name: Rc }, } pub fn main() -> Result<(), Box> { let mut pkgs_seen = HashSet::::new(); let mut fragments = HashMap::<&str, String>::new(); let mut depgraph = DepGraph::new(); let interner = DefaultInterner::new(); let package_path = std::env::args().nth(1).expect("Missing argument"); let abs_path = fs::canonicalize(package_path).unwrap(); println!("WARNING: This is proof-of-concept; do not use!"); load_xmlo( &abs_path.to_str().unwrap().to_string(), &mut pkgs_seen, &mut fragments, &mut depgraph, &interner, )?; // println!( // "Graph {:?}", // depgraph // .graph // .raw_nodes() // .iter() // .map(|node| &node.weight) // .collect::>() // ); let sorted = sort_deps(&depgraph); println!("Sorted ({}): {:?}", sorted.len(), sorted); Ok(()) } fn load_xmlo<'a, 'i, I: Interner<'i>>( path_str: &'a str, pkgs_seen: &mut HashSet, fragments: &mut HashMap<&'i str, String>, depgraph: &mut DepGraph, interner: &'i I, ) -> Result<(), Box> { let path = fs::canonicalize(path_str)?; let path_str = path.to_str().unwrap(); if !pkgs_seen.insert(path_str.to_string()) { return Ok(()); } //println!("processing {}", path_str); let mut found = HashSet::<&str>::new(); let file = fs::File::open(&path)?; let reader = BufReader::new(file); let mut xmlo = XmloReader::new(reader, interner); loop { match xmlo.read_event()? { XmloEvent::SymDeps(sym, deps) => { // TODO: API needs to expose whether a symbol is already // known so that we can warn on them // // note: using from_utf8_unchecked here did _not_ improve // performance let sym_node = depgraph.declare(sym); for dep_sym in deps { let dep_node = depgraph.declare(dep_sym); depgraph.declare_dep(sym_node, dep_node); } } XmloEvent::SymDecl(_sym, attrs) => { if let Some(sym_src) = attrs.src { found.insert(sym_src); } } XmloEvent::Fragment(sym, text) => { fragments.insert(sym, text); } // We don't need to read any further than the end of the // header (symtable, sym-deps, fragments) XmloEvent::Eoh => break, } } let mut dir = path.clone(); dir.pop(); for relpath in found.iter() { let mut path_buf = dir.clone(); path_buf.push(relpath); path_buf.set_extension("xmlo"); //println!("Trying {:?}", path_buf); let path_abs = path_buf.canonicalize().unwrap(); let path = path_abs.to_str().unwrap(); load_xmlo(path, pkgs_seen, fragments, depgraph, interner)?; } Ok(()) } fn sort_deps(depgraph: &DepGraph) -> Vec<&SymEntry> { // @type=meta, @preproc:elig-class-yields // @type={ret}map{,:head,:tail} let roots = discover_roots(depgraph); // This is technically a topological sort, but functions have // cycles. Once we have more symbol metadata, we can filter them out // and actually invoke toposort. let mut dfs = DfsPostOrder::empty(&depgraph); let mut sorted = Vec::new(); // TODO: we'll be processing various roots separately for index in roots { dfs.stack.push(*index); } while let Some(index) = dfs.next(&depgraph) { sorted.push(&depgraph[index]); } sorted } fn discover_roots(depgraph: &DepGraph) -> Vec { // TODO: filter_map let mut map_syms = depgraph .index_iter() .filter(|(key, _)| { key.starts_with(":map:") || key.starts_with(":retmap:") }) .map(|(_, value)| *value) .collect::>(); let mut roots = vec!["___yield", "___worksheet"] .iter() .filter_map(|sym| depgraph.lookup(sym)) .collect::>(); roots.append(&mut map_syms); //println!( // "found roots: {:?}", // roots // .iter() // .map(|index| &depgraph.graph[*index]) // .collect::>() //); roots } #[cfg(test)] mod test { #[test] fn placeholder() {} }