employer
/
tame
1
0
Fork 0
Code Packages Releases Activity

tamer: obj::xmlo::reader: Begin conversion to ParseState 

This begins to transition XmloReader into a ParseState.  Unlike previous
changes where ParseStates were composed into a single ParseState, this is
instead a lowering operation that will take the output of one Parser and
provide it to another.

The mess in ld::poc (...which still needs to be refactored and removed)
shows the concept, which will be abstracted away.  This won't actually get
to the ASG in order to test that that this works with the
wip-xmlo-xir-reader flag on (development hasn't gotten that far yet), but
since it type-checks, it should conceptually work.

Wiring lowering operations together is something that I've been dreading for
months, but my approach of only abstracting after-the-fact has helped to
guide a sane approach for this.  For some definition of "sane".

It's also worth noting that AsgBuilder will too become a ParseState
implemented as another lowering operation, so:

  XIR -> XIRF -> XMLO -> ASG

These steps will all be streaming, with iteration happening only at the
topmost level.  For this reason, it's important that ASG not be responsible
for doing that pull, and further we should propagate Parsed::Incomplete
rather than filtering it out and looping an indeterminate number of times
outside of the toplevel.

One final note: the choice of 64 for the maximum depth is entirely
arbitrary and should be more than generous; it'll be finalized at some point
in the future once I actually evaluate what maximum depth is reasonable
based on how the system is used, with some added growing room.

DEV-10863
main
Mike Gerwitz 2022-03-22 13:56:43 -04:00
parent f6957ff028
commit b4a7591357
6 changed files with 100 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
tamer/src/ld/poc.rs
View File

@ -25,9 +25,6 @@ use super::xmle::{
xir::lower_iter,
XmleSections,
};
use crate::sym::SymbolId;
use crate::sym::{GlobalSymbolIntern, GlobalSymbolResolve};
use crate::xir::writer::XmlWriter;
use crate::{
asg::{Asg, DefaultAsg, IdentObject},
xir::DefaultEscaper,
@ -43,13 +40,19 @@ use crate::{
obj::xmlo::{AsgBuilder, AsgBuilderState, XmloReader},
xir::Escaper,
};
use crate::{parse::ParseState, sym::SymbolId};
use crate::{parse::Parsed, xir::writer::XmlWriter};
use crate::{
sym::{GlobalSymbolIntern, GlobalSymbolResolve},
xir::flat,
};
use fxhash::FxBuildHasher;
use petgraph_graphml::GraphMl;
use std::error::Error;
use std::fs;
use std::io::Write;
use std::io::{BufReader, BufWriter};
use std::path::{Path, PathBuf};
use std::{error::Error, iter};
type LinkerAsg = DefaultAsg<IdentObject>;
type LinkerAsgBuilderState = AsgBuilderState<FxBuildHasher>;
@ -195,10 +198,32 @@ fn load_xmlo<'a, P: AsRef<Path>, S: Escaper>(
use crate::iter::into_iter_while_ok;
use crate::xir::reader::XmlXirReader;
// TODO: This entire block is a WIP and will be incrementally
// abstracted away.
into_iter_while_ok(XmlXirReader::new(file, escaper), |toks| {
let xmlo: XmloReader<_> = toks.into();
depgraph.import_xmlo(xmlo, state)
})??
into_iter_while_ok(flat::State::<64>::parse(toks), |xirf| {
let mut xmlo = XmloReader::parse(iter::empty());
let foo = xirf.map(|parsed| match parsed {
Parsed::Incomplete => Ok(Parsed::Incomplete),
Parsed::Object(obj) => {
let item: flat::Object = obj;
xmlo.feed_tok(item)
}
});
into_iter_while_ok(foo, |xmlo_out| {
// TODO: Transitionary---we do not want to filter.
depgraph.import_xmlo(
xmlo_out.filter_map(|parsed| match parsed {
Parsed::Incomplete => None,
Parsed::Object(obj) => Some(Ok(obj)),
}),
state,
)
})
})
})????
}
};

2
tamer/src/obj/xmlo/error.rs
View File

@ -34,7 +34,7 @@ use std::fmt::Display;
/// This drastically simplifies the reader and [`Result`] chaining.
///
/// TODO: These errors provide no context (byte offset).
#[derive(Debug, PartialEq)]
#[derive(Debug, PartialEq, Eq)]
pub enum XmloError {
/// XML parsing error (legacy, quick-xml).
XmlError(XmlError),

69
tamer/src/obj/xmlo/reader.rs
View File

@ -44,68 +44,45 @@ mod new {
//! This module will be merged into [`super`] once complete;
//! it exists to make feature-flagging less confusing and error-prone.
use super::{XmloError, XmloEvent, XmloResult};
use super::{XmloError, XmloEvent};
use crate::parse::{ParseState, Transition};
use crate::sym::st::*;
use crate::xir::{Token, TokenStream};
use crate::xir::flat::Object as Xirf;
qname_const! {
QN_LV_PACKAGE: L_LV:L_PACKAGE,
QN_PACKAGE: :L_PACKAGE,
}
#[derive(Debug)]
pub struct XmloReader<I: TokenStream> {
reader: I,
seen_root: bool,
#[derive(Debug, Default, PartialEq, Eq)]
pub enum XmloReader {
#[default]
Ready,
}
impl<I> XmloReader<I>
where
I: TokenStream,
{
pub fn from_reader(reader: I) -> Self {
Self {
reader,
seen_root: false,
}
}
impl ParseState for XmloReader {
type Token = Xirf;
type Object = XmloEvent;
type Error = XmloError;
pub fn read_event(&mut self) -> XmloResult<XmloEvent> {
let token = self.reader.next().ok_or(XmloError::UnexpectedEof)?;
fn parse_token(
self,
tok: Self::Token,
) -> crate::parse::TransitionResult<Self> {
use XmloReader::Ready;
if !self.seen_root {
match token {
Token::Open(QN_LV_PACKAGE | QN_PACKAGE, _) => {
todo!("PackageAttrs removed");
}
_ => return Err(XmloError::UnexpectedRoot),
match (self, tok) {
(Ready, Xirf::Open(QN_LV_PACKAGE | QN_PACKAGE, _, _)) => {
todo!("PackageAttrs removed");
}
}
match token {
todo => todo!("read_event: {:?}", todo),
(Ready, _) => Transition(Ready).err(XmloError::UnexpectedRoot),
}
}
}
impl<I> Iterator for XmloReader<I>
where
I: TokenStream,
{
type Item = XmloResult<XmloEvent>;
fn next(&mut self) -> Option<Self::Item> {
Some(self.read_event())
}
}
impl<I> From<I> for XmloReader<I>
where
I: TokenStream,
{
fn from(toks: I) -> Self {
Self::from_reader(toks)
fn is_accepting(&self) -> bool {
// TODO
todo!("XmloReader::is_accepting")
}
}
}

67
tamer/src/obj/xmlo/reader/test.rs
View File

@ -20,23 +20,33 @@
use std::assert_matches::assert_matches;
use super::*;
use crate::{convert::ExpectInto, span::DUMMY_SPAN as DS, xir::Token};
use crate::{
convert::ExpectInto,
parse::{ParseError, ParseState},
span::DUMMY_SPAN as DS,
xir::{
attr::Attr,
flat::{Depth, Object as Xirf},
},
};
type Sut<B> = XmloReader<B>;
#[test]
fn fail_unexpected_eof() {
let mut sut = Sut::from_reader([].into_iter());
assert_matches!(sut.next(), Some(Err(XmloError::UnexpectedEof)));
}
type Sut = XmloReader;
#[test]
fn fails_on_invalid_root() {
let mut sut = Sut::from_reader(
[Token::Open("not-a-valid-package-node".unwrap_into(), DS)].into_iter(),
let mut sut = Sut::parse(
[Xirf::Open(
"not-a-valid-package-node".unwrap_into(),
DS,
Depth(0),
)]
.into_iter(),
);
assert_matches!(sut.next(), Some(Err(XmloError::UnexpectedRoot)));
assert_matches!(
sut.next(),
Some(Err(ParseError::StateError(XmloError::UnexpectedRoot)))
);
}
//#[test]
@ -45,17 +55,25 @@ fn _parses_package_attrs() {
let relroot = "../../".into();
let elig = "elig-class-yields".into();
let mut sut = Sut::from_reader(
let mut sut = Sut::parse(
[
Token::Open("package".unwrap_into(), DS),
Token::AttrName("name".unwrap_into(), DS),
Token::AttrValue(name, DS),
Token::AttrName("__rootpath".unwrap_into(), DS),
Token::AttrValue(relroot, DS),
Token::AttrName("program".unwrap_into(), DS),
Token::AttrValue(crate::sym::st::raw::L_TRUE, DS),
Token::AttrName(("preproc", "elig-class-yields").unwrap_into(), DS),
Token::AttrValue(elig, DS),
Xirf::Open("package".unwrap_into(), DS, Depth(0)),
Xirf::Attr(Attr::new("name".unwrap_into(), name, (DS, DS))),
Xirf::Attr(Attr::new(
"__rootpath".unwrap_into(),
relroot,
(DS, DS),
)),
Xirf::Attr(Attr::new(
"program".unwrap_into(),
crate::sym::st::raw::L_TRUE,
(DS, DS),
)),
Xirf::Attr(Attr::new(
("preproc", "elig-class-yields").unwrap_into(),
elig,
(DS, DS),
)),
]
.into_iter(),
);
@ -70,11 +88,10 @@ fn _parses_package_attrs() {
fn _parses_package_attrs_with_ns_prefix() {
let name = "pkgrootns".into();
let mut sut = Sut::from_reader(
let mut sut = Sut::parse(
[
Token::Open(("lv", "package").unwrap_into(), DS),
Token::AttrName("name".unwrap_into(), DS),
Token::AttrValue(name, DS),
Xirf::Open(("lv", "package").unwrap_into(), DS, Depth(0)),
Xirf::Attr(Attr::new("name".unwrap_into(), name, (DS, DS))),
]
.into_iter(),
);

2
tamer/src/parse.rs
View File

@ -262,7 +262,7 @@ impl<S: ParseState, I: TokenStream<S::Token>> Parser<S, I> {
/// [`Iterator`] implementation.
/// The pull system also uses this method to provided data to the
/// parser.
fn feed_tok(&mut self, tok: S::Token) -> ParsedResult<S> {
pub fn feed_tok(&mut self, tok: S::Token) -> ParsedResult<S> {
// Store the most recently encountered Span for error
// reporting in case we encounter an EOF.
self.last_span = Some(tok.span());

2
tamer/src/xir/flat.rs
View File

@ -56,7 +56,7 @@ use std::{error::Error, fmt::Display};
/// Tag nesting depth
/// (`0` represents the root).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Depth(usize);
pub struct Depth(pub usize);
impl Display for Depth {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {