From 4510de38ed70eb16db421b69acfae28cd085bbe1 Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Wed, 10 May 2023 11:16:07 -0400 Subject: [PATCH] tamer: asg::air::pkg: Extract AirPkgAggregate from AirAggregate This is something I've wanted to do for some time, but the system is becoming hard enough to reason about (with some attempted future changes) that I require the consistency afforded by this change. It's not entirely done---as noted by the TODO for `UnnamedPkg`---but it's close, and then `AirAggregate` will just be a delegating superstate, like `ele_parse!`. Importantly, this also puts a package parser on the stack, which will work better with the stack-based scoping system being developed. It will also make it easier to fall back to a base case that I had really wanted to avoid, and will have more information on in the future: root indexing for a shared global environment for package-level identifiers. (Imports are still package-scoped, but only in appearance, by contributing to the global environment of the compilation unit during import. Well, it doesn't do that yet. The XSLT compiler works in that way.) DEV-13162 --- tamer/src/asg/air.rs | 241 ++++++++++++--------------------------- tamer/src/asg/air/ir.rs | 8 ++ tamer/src/asg/air/pkg.rs | 238 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 321 insertions(+), 166 deletions(-) create mode 100644 tamer/src/asg/air/pkg.rs diff --git a/tamer/src/asg/air.rs b/tamer/src/asg/air.rs index 1bfa3418..fd6cf9da 100644 --- a/tamer/src/asg/air.rs +++ b/tamer/src/asg/air.rs @@ -40,10 +40,8 @@ use super::{ Asg, AsgError, Expr, Ident, ObjectIndex, }; use crate::{ - diagnose::Annotate, - diagnostic_todo, parse::{prelude::*, StateStack}, - span::{Span, UNKNOWN_SPAN}, + span::Span, sym::SymbolId, }; use std::fmt::{Debug, Display}; @@ -53,8 +51,10 @@ mod ir; pub use ir::Air; mod expr; +mod pkg; mod tpl; use expr::AirExprAggregate; +use pkg::AirPkgAggregate; use tpl::AirTplAggregate; pub type IdentSym = SymbolId; @@ -67,12 +67,8 @@ pub enum AirAggregate { #[default] Empty, - /// Package definition or declaration started, - /// but the name is not yet known. - UnnamedPkg(Span), - - /// Expecting a package-level token. - Toplevel(ObjectIndex), + /// Parsing a package. + Pkg(AirPkgAggregate), /// Parsing an expression. /// @@ -95,11 +91,8 @@ impl Display for AirAggregate { match self { Empty => write!(f, "awaiting AIR input for ASG"), - UnnamedPkg(_) => { - write!(f, "expecting canonical package name") - } - Toplevel(_) => { - write!(f, "expecting package header or an expression") + Pkg(pkg) => { + write!(f, "defining a package: {pkg}") } PkgExpr(expr) => { write!(f, "defining a package expression: {expr}") @@ -111,6 +104,12 @@ impl Display for AirAggregate { } } +impl From for AirAggregate { + fn from(st: AirPkgAggregate) -> Self { + Self::Pkg(st) + } +} + impl From for AirAggregate { fn from(st: AirExprAggregate) -> Self { Self::PkgExpr(st) @@ -139,95 +138,34 @@ impl ParseState for AirAggregate { tok: Self::Token, ctx: &mut Self::Context, ) -> crate::parse::TransitionResult { - use ir::{ - AirBind::*, AirDoc::*, AirIdent::*, AirPkg::*, AirSubsets::*, - AirTodo::*, - }; + use ir::{AirBind::BindIdent, AirSubsets::*, AirTodo::*}; use AirAggregate::*; + use AirPkgAggregate::{Toplevel, UnnamedPkg}; - // TODO: Seems to be about time for refactoring this... match (self, tok.into()) { (st, AirTodo(Todo(_))) => Transition(st).incomplete(), - (Empty, AirPkg(PkgStart(span))) => { - Transition(UnnamedPkg(span)).incomplete() - } - - ( - st @ (UnnamedPkg(_) | Toplevel(_) | PkgExpr(_) | PkgTpl(_)), - AirPkg(PkgStart(span)), - ) => { - // This should always be available in this context. - let first_span = - ctx.pkg_oi().map(|oi| oi.span()).unwrap_or(UNKNOWN_SPAN); - - Transition(st).err(AsgError::NestedPkgStart(span, first_span)) - } - - // Packages are identified by canonical paths relative to the - // project root. - (UnnamedPkg(span), AirBind(BindIdent(name))) => { - match ctx.begin_pkg(span, name) { - Ok(oi_pkg) => Transition(Toplevel(oi_pkg)).incomplete(), - Err(e) => Transition(UnnamedPkg(span)).err(e), - } - } - - // TODO: Remove; transitionary (no package name required) - (UnnamedPkg(span), tok) => { - match ctx.begin_pkg(span, SPair("/TODO".into(), span)) { - Ok(oi_pkg) => Transition(Toplevel(oi_pkg)) + // TODO: Remove this kluge; transitionary (no package name required) + (Pkg(UnnamedPkg(span)), tok) + if !matches!(tok, AirBind(BindIdent(..))) => + { + match ctx.pkg_begin(span, SPair("/TODO".into(), span)) { + Ok(oi_pkg) => Transition(Pkg(Toplevel(oi_pkg))) .incomplete() .with_lookahead(tok), - Err(e) => Transition(UnnamedPkg(span)).err(e), + Err(e) => Transition(Pkg(UnnamedPkg(span))).err(e), } } - (Toplevel(oi_pkg), AirBind(BindIdent(rename))) => { - // TODO: `unwrap_or` is just until canonical name is - // unconditionally available just to avoid the possibility - // of a panic. - let name = oi_pkg.resolve(ctx.asg_mut()).canonical_name(); - - Transition(Toplevel(oi_pkg)) - .err(AsgError::PkgRename(name, rename)) + (st @ (Empty | PkgExpr(..) | PkgTpl(..)), tok @ AirPkg(..)) => { + ctx.ret_or_transfer(st, tok, AirPkgAggregate::new()) } + (Pkg(pkg), AirPkg(etok)) => ctx.proxy(pkg, etok), + (Pkg(pkg), AirBind(etok)) => ctx.proxy(pkg, etok), + (Pkg(pkg), AirIdent(etok)) => ctx.proxy(pkg, etok), + (Pkg(pkg), AirDoc(etok)) => ctx.proxy(pkg, etok), - // No expression was started. - (Toplevel(oi_pkg), AirPkg(PkgEnd(span))) => { - oi_pkg.close(ctx.asg_mut(), span); - Transition(Empty).incomplete() - } - - (Toplevel(oi_pkg), tok @ AirDoc(DocIndepClause(..))) => { - diagnostic_todo!( - vec![ - oi_pkg.note("for this package"), - tok.internal_error( - "this package description is not yet supported" - ) - ], - "package-level short description is not yet supported by TAMER", - ) - } - - (Toplevel(oi_pkg), AirDoc(DocText(text))) => { - oi_pkg.append_doc_text(ctx.asg_mut(), text); - Transition(Toplevel(oi_pkg)).incomplete() - } - - // Package import - (Toplevel(oi_pkg), AirBind(RefIdent(pathspec))) => oi_pkg - .import(ctx.asg_mut(), pathspec) - .map(|_| ()) - .transition(Toplevel(oi_pkg)), - - // Note: We unfortunately can't match on `AirExpr | AirBind | ...` - // and delegate in the same block - // (without having to duplicate type checks and then handle - // unreachable paths) - // because of the different inner types. - (st @ (Toplevel(_) | PkgTpl(_)), tok @ AirExpr(..)) => { + (st @ (Pkg(_) | PkgTpl(_)), tok @ AirExpr(..)) => { ctx.ret_or_transfer(st, tok, AirExprAggregate::new()) } (PkgExpr(expr), AirExpr(etok)) => ctx.proxy(expr, etok), @@ -235,103 +173,62 @@ impl ParseState for AirAggregate { (PkgExpr(expr), AirDoc(etok)) => ctx.proxy(expr, etok), // Template parsing. - (st @ (Toplevel(_) | PkgExpr(_)), tok @ AirTpl(..)) => { + (st @ (Pkg(_) | PkgExpr(_)), tok @ AirTpl(..)) => { ctx.ret_or_transfer(st, tok, AirTplAggregate::new()) } (PkgTpl(tplst), AirTpl(ttok)) => ctx.proxy(tplst, ttok), (PkgTpl(tplst), AirBind(ttok)) => ctx.proxy(tplst, ttok), (PkgTpl(tplst), AirDoc(ttok)) => ctx.proxy(tplst, ttok), - (Empty, AirPkg(PkgEnd(span))) => { - Transition(Empty).err(AsgError::InvalidPkgEndContext(span)) - } - - (st @ (PkgExpr(_) | PkgTpl(_)), AirPkg(PkgEnd(span))) => { - match st.active_is_accepting(ctx) { - true => { - ctx.stack().ret_or_dead(Empty, AirPkg(PkgEnd(span))) - } - false => { - Transition(st).err(AsgError::InvalidPkgEndContext(span)) - } - } - } - ( Empty, tok @ (AirExpr(..) | AirBind(..) | AirTpl(..) | AirDoc(..)), ) => Transition(Empty).err(AsgError::PkgExpected(tok.span())), - (Toplevel(oi_pkg), AirIdent(IdentDecl(name, kind, src))) => { - let asg = ctx.asg_mut(); - let oi_root = asg.root(name); - - asg.lookup_or_missing(oi_root, name) - .declare(asg, name, kind, src) - .map(|_| ()) - .transition(Toplevel(oi_pkg)) - } - - (Toplevel(oi_pkg), AirIdent(IdentExternDecl(name, kind, src))) => { - let asg = ctx.asg_mut(); - let oi_root = asg.root(name); - - asg.lookup_or_missing(oi_root, name) - .declare_extern(asg, name, kind, src) - .map(|_| ()) - .transition(Toplevel(oi_pkg)) - } - - (Toplevel(oi_pkg), AirIdent(IdentDep(name, dep))) => { - let asg = ctx.asg_mut(); - let oi_root = asg.root(dep); - - let oi_from = asg.lookup_or_missing(oi_root, name); - let oi_to = asg.lookup_or_missing(oi_root, dep); - oi_from.add_opaque_dep(ctx.asg_mut(), oi_to); - - Transition(Toplevel(oi_pkg)).incomplete() - } - - (Toplevel(oi_pkg), AirIdent(IdentFragment(name, text))) => { - let asg = ctx.asg_mut(); - let oi_root = asg.root(name); - - asg.lookup_or_missing(oi_root, name) - .set_fragment(asg, text) - .map(|_| ()) - .transition(Toplevel(oi_pkg)) - } - - (Toplevel(oi_pkg), AirIdent(IdentRoot(name))) => { - let asg = ctx.asg_mut(); - asg.root(name).root_ident(asg, name); - - Transition(Toplevel(oi_pkg)).incomplete() - } - (st @ (Empty | PkgExpr(..) | PkgTpl(..)), AirIdent(tok)) => { Transition(st).err(AsgError::UnexpectedOpaqueIdent(tok.name())) } } } - fn is_accepting(&self, _: &Self::Context) -> bool { - matches!(self, Self::Empty) + fn is_accepting(&self, ctx: &Self::Context) -> bool { + ctx.stack_ref().iter().all(|st| st.active_is_accepting(ctx)) + && self.active_is_accepting(ctx) } } impl AirAggregate { - /// Whether the active parser is in an accepting state. + /// Whether the active parser is completed with active parsing. + /// + /// This method is used to determine whether control ought to be + /// transferred to a new child parser. /// /// If a child parser is active, /// then its [`ParseState::is_accepting`] will be consulted. + fn active_is_complete(&self, ctx: &::Context) -> bool { + use AirAggregate::*; + + match self { + // We can't be done with something we're not doing. + // This is necessary to start the first child parser. + Empty => false, + + Pkg(st) => st.is_accepting(ctx), + PkgExpr(st) => st.is_accepting(ctx), + PkgTpl(st) => st.is_accepting(ctx), + } + } + + // Whether the parser is in an accepting state. fn active_is_accepting(&self, ctx: &::Context) -> bool { use AirAggregate::*; match self { + // This must not recurse on `AirAggregate::is_accepting`, + // otherwise it'll be mutually recursive. Empty => true, - UnnamedPkg(_) | Toplevel(_) => self.is_accepting(ctx), + + Pkg(st) => st.is_accepting(ctx), PkgExpr(st) => st.is_accepting(ctx), PkgTpl(st) => st.is_accepting(ctx), } @@ -346,8 +243,10 @@ impl AirAggregate { fn active_rooting_oi(&self) -> Option> { match self { AirAggregate::Empty => None, - AirAggregate::UnnamedPkg(_) => None, - AirAggregate::Toplevel(pkg_oi) => Some((*pkg_oi).into()), + + // Packages always serve as roots for identifiers + // (that is their entire purpose). + AirAggregate::Pkg(pkgst) => pkgst.active_pkg_oi().map(Into::into), // Expressions never serve as roots for identifiers; // this will always fall through to the parent context. @@ -409,6 +308,11 @@ impl AirAggregateCtx { stack } + fn stack_ref(&self) -> &AirStack { + let Self(_, stack, _) = self; + stack + } + /// Return control to the parser atop of the stack if `st` is an /// accepting state, /// otherwise transfer control to a new parser `to`. @@ -448,7 +352,7 @@ impl AirAggregateCtx { ) -> TransitionResult { let st_super = st.into(); - if st_super.active_is_accepting(self) { + if st_super.active_is_complete(self) { // TODO: dead state or error self.stack().ret_or_dead(AirAggregate::Empty, tok) } else { @@ -475,7 +379,7 @@ impl AirAggregateCtx { } /// Create a new rooted package and record it as the active package. - fn begin_pkg( + fn pkg_begin( &mut self, start: Span, name: SPair, @@ -489,6 +393,12 @@ impl AirAggregateCtx { Ok(oi_pkg) } + /// Indicate that there is no longer any active package. + fn pkg_clear(&mut self) { + let Self(_, _, pkg) = self; + pkg.take(); + } + /// The active package if any. fn pkg_oi(&self) -> Option> { match self { @@ -525,7 +435,7 @@ impl AirAggregateCtx { // unreachable. // There should be no parent frame and so this will fail to find // a value. - AirAggregate::UnnamedPkg(_) | AirAggregate::Toplevel(_) => None, + AirAggregate::Pkg(_) => None, // Expressions may always contain other expressions, // and so this method should not be consulted in such a @@ -552,8 +462,7 @@ impl AirAggregateCtx { stack.iter().rev().find_map(|st| match st { AirAggregate::Empty => None, - AirAggregate::UnnamedPkg(_) => None, - AirAggregate::Toplevel(pkg_oi) => Some((*pkg_oi).into()), + AirAggregate::Pkg(pkg_st) => pkg_st.active_pkg_oi().map(Into::into), AirAggregate::PkgExpr(exprst) => { exprst.active_expr_oi().map(Into::into) } diff --git a/tamer/src/asg/air/ir.rs b/tamer/src/asg/air/ir.rs index 9e61fe3c..8dcb2842 100644 --- a/tamer/src/asg/air/ir.rs +++ b/tamer/src/asg/air/ir.rs @@ -816,6 +816,14 @@ sum_ir! { } } + /// Package definitions also capable of being loaded from object files. + /// + /// It is assumed that tokens that may appear as the body of a package, + /// with the exception of [`AirIdent`], + /// will preempt the package parser, + /// and so are not included here. + pub sum enum AirLoadablePkg = AirPkg | AirBind | AirIdent | AirDoc; + /// Expressions that are able to be bound to identifiers. /// /// This is the primary token set when parsing packages, diff --git a/tamer/src/asg/air/pkg.rs b/tamer/src/asg/air/pkg.rs new file mode 100644 index 00000000..f479d25f --- /dev/null +++ b/tamer/src/asg/air/pkg.rs @@ -0,0 +1,238 @@ +// ASG IR package parsing +// +// Copyright (C) 2014-2023 Ryan Specialty, LLC. +// +// This file is part of TAME. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! AIR package parser. +//! +//! See the [parent module](super) for more information. + +use super::{ + super::{graph::object::Pkg, AsgError, ObjectIndex}, + ir::AirLoadablePkg, + AirAggregate, AirAggregateCtx, +}; +use crate::{ + diagnose::Annotate, diagnostic_todo, parse::prelude::*, span::Span, +}; + +/// Package parsing with support for loaded identifiers. +/// +/// This supports non-nested package definitions of source files, +/// as well as declaring opaque identifiers loaded from object files via +/// [`AirIdent`](super::ir::AirIdent). +#[derive(Debug, PartialEq)] +pub enum AirPkgAggregate { + /// Ready for an expression; + /// expression stack is empty. + Ready, + + /// Package definition or declaration started, + /// but the name is not yet known. + UnnamedPkg(Span), + + /// Expecting a package-level token. + Toplevel(ObjectIndex), +} + +impl Display for AirPkgAggregate { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + use AirPkgAggregate::*; + + match self { + Ready => { + write!(f, "expecting package definition") + } + UnnamedPkg(_) => { + write!(f, "expecting canonical package name") + } + Toplevel(_) => { + write!(f, "expecting package header or an expression") + } + } + } +} + +impl ParseState for AirPkgAggregate { + type Token = AirLoadablePkg; + type Object = (); + type Error = AsgError; + type Context = AirAggregateCtx; + type Super = AirAggregate; + + fn parse_token( + self, + tok: Self::Token, + ctx: &mut Self::Context, + ) -> crate::parse::TransitionResult { + use super::ir::{AirBind::*, AirDoc::*, AirIdent::*, AirPkg::*}; + use AirLoadablePkg::*; + use AirPkgAggregate::*; + + match (self, tok) { + (Ready, AirPkg(PkgStart(span))) => { + if let Some(first_span) = ctx.pkg_oi().map(|oi| oi.span()) { + Transition(Ready) + .err(AsgError::NestedPkgStart(span, first_span)) + } else { + Transition(UnnamedPkg(span)).incomplete() + } + } + + (Toplevel(oi_pkg), AirPkg(PkgStart(span))) => { + Transition(Toplevel(oi_pkg)) + .err(AsgError::NestedPkgStart(span, oi_pkg.span())) + } + + // Packages are identified by canonical paths relative to the + // project root. + (UnnamedPkg(span), AirBind(BindIdent(name))) => { + match ctx.pkg_begin(span, name) { + Ok(oi_pkg) => Transition(Toplevel(oi_pkg)).incomplete(), + Err(e) => Transition(UnnamedPkg(span)).err(e), + } + } + + (Toplevel(oi_pkg), AirBind(BindIdent(rename))) => { + let name = oi_pkg.resolve(ctx.asg_mut()).canonical_name(); + + Transition(Toplevel(oi_pkg)) + .err(AsgError::PkgRename(name, rename)) + } + + (Toplevel(oi_pkg), AirPkg(PkgEnd(span))) => { + oi_pkg.close(ctx.asg_mut(), span); + ctx.pkg_clear(); + Transition(Ready).incomplete() + } + + (Toplevel(oi_pkg), tok @ AirDoc(DocIndepClause(..))) => { + diagnostic_todo!( + vec![ + oi_pkg.note("for this package"), + tok.internal_error( + "this package description is not yet supported" + ) + ], + "package-level short description is not yet supported by TAMER", + ) + } + + (Toplevel(oi_pkg), AirDoc(DocText(text))) => { + oi_pkg.append_doc_text(ctx.asg_mut(), text); + Transition(Toplevel(oi_pkg)).incomplete() + } + + // Package import + (Toplevel(oi_pkg), AirBind(RefIdent(pathspec))) => oi_pkg + .import(ctx.asg_mut(), pathspec) + .map(|_| ()) + .transition(Toplevel(oi_pkg)), + + (Toplevel(oi_pkg), AirIdent(IdentDecl(name, kind, src))) => { + let asg = ctx.asg_mut(); + let oi_root = asg.root(name); + + asg.lookup_or_missing(oi_root, name) + .declare(asg, name, kind, src) + .map(|_| ()) + .transition(Toplevel(oi_pkg)) + } + + (Toplevel(oi_pkg), AirIdent(IdentExternDecl(name, kind, src))) => { + let asg = ctx.asg_mut(); + let oi_root = asg.root(name); + + asg.lookup_or_missing(oi_root, name) + .declare_extern(asg, name, kind, src) + .map(|_| ()) + .transition(Toplevel(oi_pkg)) + } + + (Toplevel(oi_pkg), AirIdent(IdentDep(name, dep))) => { + let asg = ctx.asg_mut(); + let oi_root = asg.root(dep); + + let oi_from = asg.lookup_or_missing(oi_root, name); + let oi_to = asg.lookup_or_missing(oi_root, dep); + oi_from.add_opaque_dep(ctx.asg_mut(), oi_to); + + Transition(Toplevel(oi_pkg)).incomplete() + } + + (Toplevel(oi_pkg), AirIdent(IdentFragment(name, text))) => { + let asg = ctx.asg_mut(); + let oi_root = asg.root(name); + + asg.lookup_or_missing(oi_root, name) + .set_fragment(asg, text) + .map(|_| ()) + .transition(Toplevel(oi_pkg)) + } + + (Toplevel(oi_pkg), AirIdent(IdentRoot(name))) => { + let asg = ctx.asg_mut(); + asg.root(name).root_ident(asg, name); + + Transition(Toplevel(oi_pkg)).incomplete() + } + + (Ready, AirPkg(PkgEnd(span))) => { + Transition(Ready).err(AsgError::InvalidPkgEndContext(span)) + } + + // TODO: See superstate + (UnnamedPkg(span), tok) => { + diagnostic_todo!( + vec![ + span.note("for this package"), + tok.internal_error( + "package name expected before this token" + ), + ], + "package name expected", + ) + } + + // Token may refer to a parent context. + (st @ Ready, tok @ (AirBind(..) | AirIdent(..) | AirDoc(..))) => { + Transition(st).dead(tok) + } + } + } + + fn is_accepting(&self, _: &Self::Context) -> bool { + matches!(self, Self::Ready) + } +} + +impl AirPkgAggregate { + pub fn new() -> Self { + Self::Ready + } + + /// The [`ObjectIndex`] of the package being parsed, + /// if any. + pub fn active_pkg_oi(&self) -> Option> { + use AirPkgAggregate::*; + + match self { + Ready | UnnamedPkg(_) => None, + Toplevel(oi_pkg) => Some(*oi_pkg), + } + } +}