employer
/
tame
1
0
Fork 0
Code Packages Releases Activity

tamer: Re-use prior AirAggreagteCtx for subsequent parsers 

A new AirAggregate parser is utilized for each package import.  This
prevents us from moving the index from `Asg` onto `AirAggregateCtx` because
the index would be dropped between each import.

This allows re-using that context and solves for problems that result from
attempting to do so, as explained in the new
`resume_previous_parsing_context` test case.

But, it's now clear that there's a missing abstraction, and that reasoning
about this problem at the topmost level of the compiler/linker in terms of
internal parsing details like "context" is not appropriate.  What we're
doing is suspending parsing and resuming it later on for another package,
aggregating into the same destination (ASG + index).  An abstraction ought
to be formed in terms of that.

DEV-13162
main
Mike Gerwitz 2023-05-19 13:02:07 -04:00
parent 92214c7e05
commit 7857460c1d
7 changed files with 163 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
tamer/src/asg/air.rs
View File

@ -158,11 +158,6 @@ impl ParseState for AirAggregate {
type Error = AsgError;
type Context = AirAggregateCtx;
/// Destination [`Asg`] that this parser lowers into.
///
/// This ASG will be yielded by [`crate::parse::Parser::finalize`].
type PubContext = Asg;
fn parse_token(
self,
tok: Self::Token,
@ -172,12 +167,15 @@ impl ParseState for AirAggregate {
use AirAggregate::*;
match (self, tok.into()) {
// Initialize the parser with the graph root.
// The graph may contain multiple roots in the future to support
// cross-version analysis.
(Uninit, tok) => Transition(Root(ctx.asg_mut().root(tok.span())))
.incomplete()
.with_lookahead(tok),
// Initialize the parser with the graph root,
// or continue with a previous context that has already been
// initialized.
// See `asg::air::test::resume_previous_parsing_context` for an
// explanation of why this is important.
(Uninit, tok) => {
let oi_root = ctx.asg_ref().root(tok.span());
ctx.stack().continue_or_init(|| Root(oi_root), tok)
}
(st, AirTodo(Todo(_))) => Transition(st).incomplete(),
@ -765,6 +763,15 @@ impl AirAggregateCtx {
oi_ident
}
/// Consume the context and yield the inner [`Asg`].
///
/// This indicates that all package parsing has been completed and that
/// the ASG contains complete information about the program sources
/// for the requested compilation unit.
pub fn finish(self) -> Asg {
self.asg
}
}
/// Property of identifier scope within a given environment.
@ -901,12 +908,6 @@ impl AsMut<AirStack> for AirAggregateCtx {
}
}
impl From<AirAggregateCtx> for Asg {
fn from(ctx: AirAggregateCtx) -> Self {
ctx.asg
}
}
impl From<Asg> for AirAggregateCtx {
fn from(asg: Asg) -> Self {
Self {

99
tamer/src/asg/air/test.rs
View File

@ -380,7 +380,7 @@ fn pkg_is_rooted() {
let mut sut = Sut::parse(toks.into_iter());
assert!(sut.all(|x| x.is_ok()));
let asg = sut.finalize().unwrap().into_context();
let asg = sut.finalize().unwrap().into_context().finish();
let oi_root = asg.root(S3);
let pkg = oi_root
@ -570,7 +570,7 @@ fn pkg_import_canonicalized_against_current_pkg() {
let mut sut = Sut::parse(toks.into_iter());
assert!(sut.all(|x| x.is_ok()));
let asg = sut.finalize().unwrap().into_context();
let asg = sut.finalize().unwrap().into_context().finish();
let import = asg
.root(S1)
@ -621,6 +621,97 @@ fn pkg_doc() {
);
}
// Package imports will trigger parsing,
// but the intent is to re-use the previous parsing context so that we can
// continue to accumulate into the same graph along with the same scope
// index.
#[test]
fn resume_previous_parsing_context() {
let name_foo = SPair("foo".into(), S2);
let name_bar = SPair("bar".into(), S5);
let name_baz = SPair("baz".into(), S6);
let kind = IdentKind::Tpl;
let src = Source::default();
// We're going to test with opaque objects as if we are the linker.
// This is the first parse.
#[rustfmt::skip]
let toks = vec![
// The first package will reference an identifier from another
// package.
PkgStart(S1, SPair("/pkg-a".into(), S1)),
IdentDep(name_foo, name_bar),
PkgEnd(S3),
];
let ctx = air_ctx_from_toks(toks);
// We consumed the parser above and retrieved its context.
// This is the token stream for the second parser,
// which will re-use the above context.
#[rustfmt::skip]
let toks = vec![
// This package will define that identifier,
// which should also find the identifier having been placed into
// the global environment.
PkgStart(S4, SPair("/pkg-b".into(), S4)),
IdentDecl(name_bar, kind.clone(), src.clone()),
// This is a third identifier that is unique to this package.
// This is intended to catch the following situation,
// where `P` is the ParseState and `S` is the stack.
//
// 1. P:Uninit S:[]
// 2. P:Root S:[]
// 3. P:Pkg S:[Root]
// ---- next parser ---
// 4. P:Uninit S:[Root]
// 5. P:Root S:[Root] <-- new Root
// 6. P:Pkg S:[Root, Root]
// ^ ^
// `-----\
// Would try to index at oi_root
// _twice_, which would panic.
//
// AirAggregate is designed to resume from the top of the stack
// when initializing to avoid this scenario.
// So here's what it's expected to do instead:
//
// [...]
// ---- next parser ---
// 4. P:Uninit S:[Root]
// 5. P:Root S:[] <-- pop existing Root
// 6. P:Pkg S:[Root]
IdentDecl(name_baz, kind.clone(), src),
PkgEnd(S7),
];
// We _resume_ parsing with the previous context.
let mut sut = Sut::parse_with_context(toks.into_iter(), ctx);
assert!(sut.all(|x| x.is_ok()));
// The ASG should have been constructed from _both_ of the previous
// individual parsers,
// having used the shared context.
let ctx = sut.finalize().unwrap().into_private_context();
// Both should have been added to the same graph.
let oi_foo = root_lookup(&ctx, name_foo).expect("missing foo");
let oi_bar = root_lookup(&ctx, name_bar).expect("missing bar");
assert!(oi_foo.has_edge_to(ctx.asg_ref(), oi_bar));
// And it should have been resolved via the _second_ package,
// which is parsed separately,
// as part of the same graph and with the same indexed identifiers.
// If there were not a shared index between the two parsers,
// then it would have retained an original `Missing` Ident and created
// a new resolved one.
assert_eq!(Some(&kind), oi_bar.resolve(ctx.asg_ref()).kind());
}
/////// Tests above; plumbing begins below ///////
/// Parse using [`Sut`] when the test does not care about the outer package.
pub fn parse_as_pkg_body<I: IntoIterator<Item = Air>>(
toks: I,
@ -644,7 +735,7 @@ where
I::IntoIter: Debug,
{
// Equivalent to `into_{private_=>}context` in this function.
air_ctx_from_pkg_body_toks(toks).into()
air_ctx_from_pkg_body_toks(toks).finish()
}
pub(super) fn air_ctx_from_pkg_body_toks<I: IntoIterator<Item = Air>>(
@ -664,7 +755,7 @@ where
I::IntoIter: Debug,
{
// Equivalent to `into_{private_=>}context` in this function.
air_ctx_from_toks(toks).into()
air_ctx_from_toks(toks).finish()
}
/// Create and yield a new [`Asg`] from an [`Air`] token stream.

2
tamer/src/asg/graph/visit/ontree/test.rs
View File

@ -45,7 +45,7 @@ where
let mut parser = AirAggregate::parse(toks.into_iter());
assert!(parser.all(|x| x.is_ok()));
let asg = &parser.finalize().unwrap().into_context();
let asg = &parser.finalize().unwrap().into_context().finish();
tree_reconstruction(asg)
.map(|TreeWalkRel(rel, depth)| {

4
tamer/src/asg/graph/visit/topo/test.rs
View File

@ -67,7 +67,7 @@ where
let mut parser = AirAggregate::parse(toks.into_iter());
assert!(parser.all(|x| x.is_ok()));
let asg = &parser.finalize().unwrap().into_context();
let asg = &parser.finalize().unwrap().into_context().finish();
let oi_root = asg.root(UNKNOWN_SPAN);
topo_report_only(
@ -261,7 +261,7 @@ fn omits_unreachable() {
let mut parser = AirAggregate::parse(toks.into_iter());
assert!(parser.all(|x| x.is_ok()));
let asg = &parser.finalize().unwrap().into_context();
let asg = &parser.finalize().unwrap().into_context().finish();
let oi_pkg = asg
.root(UNKNOWN_SPAN)

11
tamer/src/bin/tamec.rs
View File

@ -40,7 +40,7 @@ use std::{
};
use tamer::{
asg::{
air::{Air, AirAggregate},
air::{Air, AirAggregate, AirAggregateCtx},
AsgError, DefaultAsg,
},
diagnose::{
@ -156,9 +156,9 @@ fn compile<R: Reporter>(
// TODO: Determine a good default capacity once we have this populated
// and can come up with some heuristics.
let asg = DefaultAsg::with_capacity(1024, 2048);
let air_ctx: AirAggregateCtx = DefaultAsg::with_capacity(1024, 2048).into();
let (_, asg) = Lower::<
let (_, air_ctx) = Lower::<
ParsedObject<UnknownToken, XirToken, XirError>,
XirToXirf<64, RefinedText>,
_,
@ -169,7 +169,7 @@ fn compile<R: Reporter>(
Lower::<InterpolateNir, NirToAir, _>::lower(nir, |air| {
Lower::<NirToAir, AirAggregate, _>::lower_with_context(
air,
asg,
air_ctx,
|end| {
end.fold(Ok(()), |x, result| match result {
Ok(_) => x,
@ -190,11 +190,12 @@ fn compile<R: Reporter>(
false => {
#[cfg(feature = "wip-asg-derived-xmli")]
{
let asg = air_ctx.finish();
derive_xmli(asg, fout, &escaper)
}
#[cfg(not(feature = "wip-asg-derived-xmli"))]
{
let _ = asg; // unused_variables
let _ = air_ctx; // unused_variables
Ok(())
}
}

29
tamer/src/ld/poc.rs
View File

@ -27,8 +27,8 @@ use super::xmle::{
};
use crate::{
asg::{
air::{Air, AirAggregate},
Asg, AsgError, DefaultAsg,
air::{Air, AirAggregate, AirAggregateCtx},
AsgError, DefaultAsg,
},
diagnose::{AnnotatedSpan, Diagnostic},
fs::{
@ -67,10 +67,10 @@ pub fn xmle(package_path: &str, output: &str) -> Result<(), TameldError> {
let mut fs = VisitOnceFilesystem::new();
let escaper = DefaultEscaper::default();
let (depgraph, state) = load_xmlo(
let (air_ctx, state) = load_xmlo(
package_path,
&mut fs,
LinkerAsg::with_capacity(65536, 65536),
LinkerAsg::with_capacity(65536, 65536).into(),
&escaper,
XmloAirContext::default(),
)?;
@ -81,7 +81,8 @@ pub fn xmle(package_path: &str, output: &str) -> Result<(), TameldError> {
..
} = state;
let sorted = sort(&depgraph, Sections::new())?;
let asg = air_ctx.finish();
let sorted = sort(&asg, Sections::new())?;
output_xmle(
sorted,
@ -97,14 +98,14 @@ pub fn xmle(package_path: &str, output: &str) -> Result<(), TameldError> {
fn load_xmlo<P: AsRef<Path>, S: Escaper>(
path_str: P,
fs: &mut VisitOnceFilesystem<FsCanonicalizer, FxBuildHasher>,
asg: Asg,
air_ctx: AirAggregateCtx,
escaper: &S,
state: XmloAirContext,
) -> Result<(Asg, XmloAirContext), TameldError> {
) -> Result<(AirAggregateCtx, XmloAirContext), TameldError> {
let PathFile(path, file, ctx): PathFile<BufReader<fs::File>> =
match fs.open(path_str)? {
VisitOnceFile::FirstVisit(file) => file,
VisitOnceFile::Visited => return Ok((asg, state)),
VisitOnceFile::Visited => return Ok((air_ctx, state)),
};
let src = &mut lowerable(XmlXirReader::new(file, escaper, ctx))
@ -112,7 +113,7 @@ fn load_xmlo<P: AsRef<Path>, S: Escaper>(
// TODO: This entire block is a WIP and will be incrementally
// abstracted away.
let (mut asg, mut state) = Lower::<
let (mut air_ctx, mut state) = Lower::<
ParsedObject<UnknownToken, XirToken, XirError>,
PartialXirToXirf<4, Text>,
_,
@ -131,10 +132,10 @@ fn load_xmlo<P: AsRef<Path>, S: Escaper>(
&mut iter,
state,
|air| {
let (_, asg) =
let (_, air_ctx) =
Lower::<XmloToAir, AirAggregate, _>::lower_with_context(
air,
asg,
air_ctx,
|end| {
for result in end {
let _ = result?;
@ -144,7 +145,7 @@ fn load_xmlo<P: AsRef<Path>, S: Escaper>(
},
)?;
Ok::<_, TameldError>(asg)
Ok::<_, TameldError>(air_ctx)
},
)
})
@ -160,10 +161,10 @@ fn load_xmlo<P: AsRef<Path>, S: Escaper>(
path_buf.push(relpath.lookup_str());
path_buf.set_extension("xmlo");
(asg, state) = load_xmlo(path_buf, fs, asg, escaper, state)?;
(air_ctx, state) = load_xmlo(path_buf, fs, air_ctx, escaper, state)?;
}
Ok((asg, state))
Ok((air_ctx, state))
}
fn output_xmle<'a, X: XmleSections<'a>, S: Escaper>(

27
tamer/src/parse/state.rs
View File

@ -718,6 +718,9 @@ impl<S: ClosedParseState, const MAX_DEPTH: usize> StateStack<S, MAX_DEPTH> {
/// If there is no state to return to on the stack,
/// then it is assumed that we have received more input than expected
/// after having completed a full parse.
///
/// If a missing state is _not_ an error condition,
/// see [`Self::continue_or_init`] instead.
pub fn ret_or_dead(
&mut self,
deadst: S,
@ -726,7 +729,7 @@ impl<S: ClosedParseState, const MAX_DEPTH: usize> StateStack<S, MAX_DEPTH> {
let Self(stack) = self;
// This should certainly never happen unless there is a bug in the
// `ele_parse!` parser-generator,
// parser,
// since it means that we're trying to return to a caller that
// does not exist.
match stack.pop() {
@ -735,6 +738,28 @@ impl<S: ClosedParseState, const MAX_DEPTH: usize> StateStack<S, MAX_DEPTH> {
}
}
/// Attempt to resume a computation atop of the stack,
/// or initialize with a new [`ParseState`] if the stack is empty.
///
/// This can be thought of like invoking a stored continuation,
/// as if with `call-with-current-continuation` in Scheme.
/// It is fully the responsibility of the caller to ensure that all
/// necessary state is captured or is otherwise able to be restored in
/// such a way that the computation can be resumed.
///
/// If a missing state is an error condition,
/// see [`Self::ret_or_dead`] instead.
pub fn continue_or_init(
&mut self,
init: impl FnOnce() -> S,
lookahead: impl Token + Into<S::Token>,
) -> TransitionResult<S> {
let Self(stack) = self;
let st = stack.pop().unwrap_or_else(init);
Transition(st).incomplete().with_lookahead(lookahead)
}
/// Iterate through each [`ClosedParseState`] held on the stack.
pub fn iter(&self) -> std::slice::Iter<'_, S> {
let Self(stack) = self;