593 lines
21 KiB
Rust
593 lines
21 KiB
Rust
// XIR reader
|
||
//
|
||
// Copyright (C) 2014-2022 Ryan Specialty Group, LLC.
|
||
//
|
||
// This file is part of TAME.
|
||
//
|
||
// This program is free software: you can redistribute it and/or modify
|
||
// it under the terms of the GNU General Public License as published by
|
||
// the Free Software Foundation, either version 3 of the License, or
|
||
// (at your option) any later version.
|
||
//
|
||
// This program is distributed in the hope that it will be useful,
|
||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
// GNU General Public License for more details.
|
||
//
|
||
// You should have received a copy of the GNU General Public License
|
||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||
|
||
//! Parse XML files into a XIR [`Token`] stream.
|
||
//!
|
||
//! This uses [`quick_xml`] as the parser.
|
||
|
||
use super::{
|
||
error::SpanlessError, CloseSpan, DefaultEscaper, Error, Escaper, OpenSpan,
|
||
Token,
|
||
};
|
||
use crate::{
|
||
parse::{ParseError, Parsed, ParsedObject, ParsedResult},
|
||
span::Context,
|
||
sym::{st::raw::WS_EMPTY, GlobalSymbolInternBytes},
|
||
};
|
||
use quick_xml::{
|
||
self,
|
||
events::{
|
||
attributes::Attributes, BytesDecl, BytesStart, Event as QuickXmlEvent,
|
||
},
|
||
Error as QuickXmlError,
|
||
};
|
||
use std::{borrow::Cow, collections::VecDeque, io::BufRead, result};
|
||
|
||
pub type Result<T> = result::Result<T, Error>;
|
||
|
||
/// Parse XML into a XIR [`Token`] stream.
|
||
///
|
||
/// This reader is intended to be used as an [`Iterator`].
|
||
///
|
||
/// The underlying reader produces events in chunks that are far too
|
||
/// large for XIR,
|
||
/// so most [`Token`]s retrieved via this call are buffered.
|
||
/// Parsing takes place when that buffer is exhausted and the next event
|
||
/// is requested from the underlying reader
|
||
/// (see [`XmlXirReader::refill_buf`]).
|
||
/// Errors can only occur during parsing,
|
||
/// and will never occur on buffered tokens.
|
||
///
|
||
/// [`None`] is returned only on EOF,
|
||
/// not on error.
|
||
pub struct XmlXirReader<'s, B, S = DefaultEscaper>
|
||
where
|
||
B: BufRead,
|
||
S: Escaper,
|
||
{
|
||
/// Inner parser.
|
||
reader: quick_xml::Reader<B>,
|
||
|
||
/// Parsing context for reader.
|
||
ctx: Context,
|
||
|
||
/// Buffer for [`quick_xml::Reader`].
|
||
readbuf: Vec<u8>,
|
||
|
||
/// [`Token`] buffer populated upon receiving a new event from
|
||
/// `reader`.
|
||
///
|
||
/// This buffer serves [`Iterator::next`] requests until it is
|
||
/// depleted,
|
||
/// after which [`XmlXirReader::refill_buf`] requests another token
|
||
/// from `reader`.
|
||
tokbuf: VecDeque<Token>,
|
||
|
||
/// System for unescaping string data.
|
||
escaper: &'s S,
|
||
}
|
||
|
||
impl<'s, B: BufRead, S: Escaper> XmlXirReader<'s, B, S> {
|
||
pub fn new(reader: B, escaper: &'s S, ctx: Context) -> Self {
|
||
let mut reader = quick_xml::Reader::from_reader(reader);
|
||
|
||
// XIR must support mismatched tags so that we are able to represent
|
||
// and reconstruct malformed inputs.
|
||
// XIRT will handle mismatch errors itself.
|
||
reader.check_end_names(false);
|
||
|
||
Self {
|
||
reader,
|
||
ctx,
|
||
readbuf: Vec::new(),
|
||
// This capacity is largely arbitrary,
|
||
// but [`Token`]s are small enough that it likely does not
|
||
// matter much.
|
||
tokbuf: VecDeque::with_capacity(32),
|
||
|
||
escaper,
|
||
}
|
||
}
|
||
|
||
/// Parse using the underlying [`quick_xml::Reader`] and populate the
|
||
/// [`Token`] buffer.
|
||
///
|
||
/// This is intended to be invoked once the buffer has been depleted by
|
||
/// [`XmlXirReader::next`].
|
||
pub fn refill_buf(&mut self) -> Option<Result<Token>> {
|
||
// Clear any previous buffer to free unneeded data.
|
||
self.tokbuf.clear();
|
||
self.readbuf.clear();
|
||
|
||
let ctx = self.ctx;
|
||
let prev_pos = self.reader.buffer_position();
|
||
|
||
match self.reader.read_event(&mut self.readbuf) {
|
||
// TODO: To provide better spans and error messages,
|
||
// we need to map specific types of errors.
|
||
// But we don't encounter much of anything here with how we make
|
||
// use of quick-xml.
|
||
Err(inner) => Some(Err({
|
||
let span = ctx.span_or_zz(prev_pos, 0);
|
||
SpanlessError::from(inner).with_span(span)
|
||
})),
|
||
|
||
Ok(ev) => match ev {
|
||
// This is the only time we'll consider the iterator to be
|
||
// done.
|
||
QuickXmlEvent::Eof => None,
|
||
|
||
QuickXmlEvent::Empty(ele) => Some(
|
||
Self::parse_element_open(
|
||
&self.escaper,
|
||
&mut self.tokbuf,
|
||
ele,
|
||
prev_pos,
|
||
ctx,
|
||
true,
|
||
)
|
||
.and_then(|open| {
|
||
let new_pos = self.reader.buffer_position();
|
||
|
||
// `<tag ... />`
|
||
// ||
|
||
let span = ctx.span_or_zz(new_pos - 2, 2);
|
||
|
||
// Tag is self-closing, but this does not yet
|
||
// handle whitespace before the `/`
|
||
// (as indicated in the span above).
|
||
self.tokbuf.push_front(Token::Close(
|
||
None,
|
||
CloseSpan::empty(span),
|
||
));
|
||
|
||
Ok(open)
|
||
}),
|
||
),
|
||
|
||
QuickXmlEvent::Start(ele) => Some(Self::parse_element_open(
|
||
&self.escaper,
|
||
&mut self.tokbuf,
|
||
ele,
|
||
prev_pos,
|
||
ctx,
|
||
false,
|
||
)),
|
||
|
||
QuickXmlEvent::End(ele) => Some({
|
||
// Only whitespace is permitted following the element
|
||
// name,
|
||
// so we can simply take the delta of the buffer pos.
|
||
//
|
||
// </foo >
|
||
// [------] name + '<' + '/' + " >"
|
||
let pos_delta = self.reader.buffer_position() - prev_pos;
|
||
let span = ctx.span_or_zz(prev_pos, pos_delta);
|
||
let name_len = ele.name().len();
|
||
|
||
ele.name()
|
||
.try_into()
|
||
.map_err(Error::from_with_span(span))
|
||
.and_then(|qname| {
|
||
Ok(Token::Close(
|
||
Some(qname),
|
||
CloseSpan(
|
||
span,
|
||
name_len.try_into().unwrap_or(0),
|
||
),
|
||
))
|
||
})
|
||
}),
|
||
|
||
// quick_xml emits a useless text event if the first byte is
|
||
// a '<'.
|
||
QuickXmlEvent::Text(bytes) if bytes.escaped().is_empty() => {
|
||
self.refill_buf()
|
||
}
|
||
|
||
// quick_xml _escapes_ the unescaped CData before handing it
|
||
// off to us,
|
||
// which is a complete waste since we'd just have to
|
||
// unescape it again.
|
||
QuickXmlEvent::CData(bytes) => todo!("CData: {:?}", bytes),
|
||
|
||
QuickXmlEvent::Text(bytes) => Some({
|
||
// <text>foo bar</text>
|
||
// [-----]
|
||
let span = ctx.span_or_zz(prev_pos, bytes.len());
|
||
|
||
bytes
|
||
.intern_utf8()
|
||
.map_err(Into::into)
|
||
.and_then(|sym| self.escaper.unescape(sym))
|
||
.map_err(Error::from_with_span(span))
|
||
.and_then(|unesc| Ok(Token::Text(unesc, span)))
|
||
}),
|
||
|
||
// Comments are _not_ returned escaped.
|
||
QuickXmlEvent::Comment(bytes) => Some({
|
||
// <!-- foo -->
|
||
// [----------] " foo " + "<!--" + "-->"
|
||
let span = ctx.span_or_zz(prev_pos, bytes.len() + 7);
|
||
|
||
bytes
|
||
.intern_utf8()
|
||
.map_err(Error::from_with_span(span))
|
||
.and_then(|comment| Ok(Token::Comment(comment, span)))
|
||
}),
|
||
|
||
// TODO: This must appear in the prologue.
|
||
QuickXmlEvent::Decl(decl) => {
|
||
match Self::validate_decl(&decl, prev_pos, ctx) {
|
||
Err(x) => Some(Err(x)),
|
||
Ok(()) => self.refill_buf(),
|
||
}
|
||
}
|
||
|
||
// We do not support processor instructions or doctypes.
|
||
// TODO: Convert this into an error/warning?
|
||
// Previously `xml-stylesheet` was present in some older
|
||
// source files and may linger for a bit after cleanup.
|
||
QuickXmlEvent::PI(..) | QuickXmlEvent::DocType(..) => {
|
||
self.refill_buf()
|
||
}
|
||
},
|
||
}
|
||
}
|
||
|
||
/// Validate an that an XML declaration contains expected values.
|
||
///
|
||
/// A declaration looks like `<?xml version="1.0" encoding="utf-8"?>`,
|
||
/// where `@encoding` is optional but `@version` is not.
|
||
/// It may also contain `@standalone`,
|
||
/// but we do not check for that.
|
||
///
|
||
/// We expect version 1.0 and UTF-8 encoding.
|
||
/// Failing when these expectations are voilated helps to ensure that
|
||
/// people unfamiliar with the system do not have expectations that
|
||
/// are going to be unmet,
|
||
/// which may result in subtle (or even serious) problems.
|
||
fn validate_decl(decl: &BytesDecl, pos: usize, ctx: Context) -> Result<()> {
|
||
// Starts after `<?`, which we want to include.
|
||
let decl_ptr = decl.as_ptr() as usize - 2 + pos;
|
||
|
||
// Fallback span that covers the entire declaration.
|
||
let decl_span = ctx.span_or_zz(pos, decl.len() + 4);
|
||
|
||
let ver =
|
||
&decl.version().map_err(Error::from_with_span(decl_span))?[..];
|
||
|
||
// NB: `quick-xml` docs state that `version` returns the quotes,
|
||
// but it does not.
|
||
if ver != b"1.0" {
|
||
// <?xml version="X.Y"?>
|
||
// [-]
|
||
let ver_pos = (ver.as_ptr() as usize) - decl_ptr;
|
||
let span = ctx.span_or_zz(ver_pos, ver.len());
|
||
|
||
Err(Error::UnsupportedXmlVersion(
|
||
ver.intern_utf8().map_err(Error::from_with_span(span))?,
|
||
span,
|
||
))?
|
||
}
|
||
|
||
if let Some(enc) = decl.encoding() {
|
||
match &enc.map_err(Error::from_with_span(decl_span))?[..] {
|
||
b"utf-8" | b"UTF-8" => (),
|
||
invalid => {
|
||
let enc_pos = (invalid.as_ptr() as usize) - decl_ptr;
|
||
let span = ctx.span_or_zz(enc_pos, invalid.len());
|
||
|
||
Err(Error::UnsupportedEncoding(
|
||
invalid
|
||
.intern_utf8()
|
||
.map_err(Error::from_with_span(span))?,
|
||
span,
|
||
))?
|
||
}
|
||
}
|
||
}
|
||
|
||
Ok(())
|
||
}
|
||
|
||
/// Parse opening element and its attributes into a XIR [`Token`]
|
||
/// stream.
|
||
///
|
||
/// The opening element is returned rather than being added to the token
|
||
/// buffer,
|
||
/// since the intent is to provide that token immediately.
|
||
fn parse_element_open(
|
||
escaper: &'s S,
|
||
tokbuf: &mut VecDeque<Token>,
|
||
ele: BytesStart,
|
||
pos: usize,
|
||
ctx: Context,
|
||
empty_tag: bool,
|
||
) -> Result<Token> {
|
||
// Starts after the opening tag `<`, so adjust.
|
||
let addr = ele.as_ptr() as usize - 1;
|
||
let len = ele.name().len();
|
||
|
||
match ele.name().last() {
|
||
None => {
|
||
// TODO: QName should be self-validating. Move this.
|
||
return Err(Error::InvalidQName(
|
||
WS_EMPTY,
|
||
// <>
|
||
// | where QName should be
|
||
ctx.span_or_zz(pos + 1, 0),
|
||
));
|
||
}
|
||
|
||
// Quick-and-dirty guess as to whether they may have missed the
|
||
// element name and included an attribute instead,
|
||
// which quick-xml does not check for.
|
||
Some(b'"' | b'\'') => {
|
||
return Err({
|
||
// <foo="bar" ...>
|
||
// [-------]
|
||
let span = ctx.span_or_zz(pos + 1, len);
|
||
|
||
Error::InvalidQName(
|
||
ele.name()
|
||
.intern_utf8()
|
||
.map_err(Error::from_with_span(span))?,
|
||
span,
|
||
)
|
||
});
|
||
}
|
||
|
||
_ => (),
|
||
};
|
||
|
||
// `ele` contains every byte up to the [self-]closing tag.
|
||
ele.name()
|
||
.try_into()
|
||
.map_err(Error::from_with_span(ctx.span_or_zz(pos + 1, len)))
|
||
.and_then(|qname| {
|
||
// The whitespace check is to handle input like this:
|
||
// <foo />
|
||
// ^ whitespace making `attributes_raw().len` > 0
|
||
let has_attrs = ele
|
||
.attributes_raw()
|
||
.iter()
|
||
.find(|b| !Self::is_whitespace(**b))
|
||
.is_some();
|
||
|
||
let attr_len = ele.attributes_raw().len();
|
||
|
||
// The tail is anything following the last byte of the QName
|
||
// in a non-empty tag with no attributes.
|
||
// For example:
|
||
// <foo > <foo> <foo bar="baz">
|
||
// ~~~~ tail ~ tail (no tail)
|
||
let tail = if has_attrs {
|
||
let found = Self::parse_attrs(
|
||
escaper,
|
||
tokbuf,
|
||
ele.attributes(),
|
||
addr - pos, // offset relative to _beginning_ of buf
|
||
pos,
|
||
ctx,
|
||
)?;
|
||
|
||
// Given this input, quick-xml ignores the bytes entirely:
|
||
// <foo bar>
|
||
// [--] missing `="value"`
|
||
if !found {
|
||
return Err(Error::AttrValueExpected(
|
||
None,
|
||
ctx.span_or_zz(pos + ele.len() + 1, 0),
|
||
));
|
||
}
|
||
|
||
// (empty) (open)
|
||
// <foo bar="baz" /> <foo bar="baz"></foo>
|
||
// | |
|
||
// zero-length '>'
|
||
let attr_end_span = ctx.span_or_zz(
|
||
pos + 1 + len + attr_len,
|
||
if empty_tag { 0 } else { 1 },
|
||
);
|
||
|
||
tokbuf.push_front(Token::AttrEnd(attr_end_span));
|
||
|
||
// No tail because of attributes.
|
||
0
|
||
} else {
|
||
match empty_tag {
|
||
// Empty tag cannot have a tail.
|
||
true => 0,
|
||
// The "attributes" buffer represents whitespace,
|
||
// so the tail is the number of bytes of
|
||
// whitespace plus the closing '>' tag delimiter.
|
||
false => attr_len + 1,
|
||
}
|
||
};
|
||
|
||
// <tag ... /> <tag/>
|
||
// [--] name + '<' [--] name + '<'
|
||
//
|
||
// <tag >...</tag> <tag ...>...</tag>
|
||
// [-----] name + '<' + " >" [--] name + '<'
|
||
// ~~~ tail
|
||
let span = ctx.span_or_zz(pos, len + 1 + tail);
|
||
|
||
// The first token will be immediately returned
|
||
// via the Iterator.
|
||
Ok(Token::Open(
|
||
qname,
|
||
OpenSpan(span, len.try_into().unwrap_or(0)),
|
||
))
|
||
})
|
||
}
|
||
|
||
/// Whether the byte represents XML whitespace.
|
||
///
|
||
/// This is quick-xml's whitespace predicate,
|
||
/// and corresponds to the
|
||
/// [nonterminal `S` in the XML specification][xmlspec-s].
|
||
///
|
||
/// [xmlspec-s]: https://www.w3.org/TR/xml/#NT-S
|
||
fn is_whitespace(b: u8) -> bool {
|
||
match b {
|
||
b' ' | b'\r' | b'\n' | b'\t' => true,
|
||
_ => false,
|
||
}
|
||
}
|
||
|
||
/// Parse attributes into a XIR [`Token`] stream.
|
||
///
|
||
/// The order of attributes will be maintained.
|
||
///
|
||
/// This does not yet handle whitespace between attributes,
|
||
/// or around `=`.
|
||
///
|
||
/// Note About Pointer Arithmetic
|
||
/// =============================
|
||
/// `ele_ptr` is expected to be a pointer to the buffer containing the
|
||
/// bytes read from the source file.
|
||
/// Attributes reference this buffer,
|
||
/// so we can use pointer arithmetic to determine the offset within
|
||
/// the buffer relative to the node.
|
||
/// This works because the underlying buffer is a `Vec`,
|
||
/// which is contiguous in memory.
|
||
///
|
||
/// However, since this is a `Vec`,
|
||
/// it is important that the address be retrieved _after_ quick-xml
|
||
/// read events,
|
||
/// otherwise the buffer may be expanded and will be reallocated.
|
||
fn parse_attrs<'a>(
|
||
escaper: &'s S,
|
||
tokbuf: &mut VecDeque<Token>,
|
||
mut attrs: Attributes<'a>,
|
||
ele_ptr: usize,
|
||
ele_pos: usize,
|
||
ctx: Context,
|
||
) -> Result<bool> {
|
||
let mut found = false;
|
||
|
||
// Disable checks to allow duplicate attributes;
|
||
// XIR does not enforce this,
|
||
// because it needs to accommodate semantically invalid XML for
|
||
// later analysis.
|
||
for result in attrs.with_checks(false) {
|
||
found = true;
|
||
|
||
let attr = result.map_err(|e| match e {
|
||
QuickXmlError::NoEqAfterName(pos) => {
|
||
// TODO: quick-xml doesn't give us the name,
|
||
// but we should discover it.
|
||
Error::AttrValueExpected(
|
||
None,
|
||
ctx.span_or_zz(ele_pos + pos, 0),
|
||
)
|
||
}
|
||
|
||
QuickXmlError::UnquotedValue(pos) => {
|
||
// TODO: name and span length
|
||
Error::AttrValueUnquoted(
|
||
None,
|
||
ctx.span_or_zz(ele_pos + pos, 0),
|
||
)
|
||
}
|
||
|
||
// fallback
|
||
e => Error::from_with_span(ctx.span_or_zz(ele_pos, 0))(e),
|
||
})?;
|
||
|
||
let keyoffset = attr.key.as_ptr() as usize;
|
||
let name_offset = keyoffset - ele_ptr;
|
||
|
||
// Accommodates zero-length values (e.g. `key=""`) with a
|
||
// zero-length span at the location the value _would_ be.
|
||
let valoffset = match attr.value {
|
||
Cow::Borrowed(b) => b.as_ptr() as usize,
|
||
|
||
// This should never happen since we have a reference to the
|
||
// underlying buffer.
|
||
Cow::Owned(_) => unreachable!(
|
||
"internal error: unexpected owned attribute value"
|
||
),
|
||
};
|
||
|
||
let value_offset = valoffset - ele_ptr;
|
||
|
||
let span_name = ctx.span_or_zz(name_offset, attr.key.len());
|
||
let span_value = ctx.span_or_zz(value_offset, attr.value.len());
|
||
|
||
// The name must be parsed as a QName.
|
||
let name = attr
|
||
.key
|
||
.try_into()
|
||
.map_err(Error::from_with_span(span_name))?;
|
||
|
||
// The attribute value,
|
||
// having just been read from XML,
|
||
// must have been escaped to be parsed properly.
|
||
// If it parsed but it's not technically escaped according to
|
||
// the spec,
|
||
// that's okay as long as we can read it again,
|
||
// but we probably should still throw an error if we
|
||
// encounter such a situation.
|
||
let value = escaper
|
||
.unescape(
|
||
attr.value
|
||
.as_ref()
|
||
.intern_utf8()
|
||
.map_err(Error::from_with_span(span_value))?,
|
||
)
|
||
.map_err(Error::from_with_span(span_value))?
|
||
.into();
|
||
|
||
tokbuf.push_front(Token::AttrName(name, span_name));
|
||
tokbuf.push_front(Token::AttrValue(value, span_value));
|
||
}
|
||
|
||
Ok(found)
|
||
}
|
||
}
|
||
|
||
impl<'s, B, S> Iterator for XmlXirReader<'s, B, S>
|
||
where
|
||
B: BufRead,
|
||
S: Escaper,
|
||
{
|
||
type Item = ParsedResult<ParsedObject<Token, Error>>;
|
||
|
||
/// Produce the next XIR [`Token`] from the input.
|
||
///
|
||
/// For more information on how this reader operates,
|
||
/// see [`XmlXirReader`].
|
||
fn next(&mut self) -> Option<Self::Item> {
|
||
self.tokbuf
|
||
.pop_back()
|
||
.map(|tok| Ok(Parsed::Object(tok)))
|
||
.or_else(|| {
|
||
self.refill_buf().map(|result| {
|
||
result.map(Parsed::Object).map_err(ParseError::StateError)
|
||
})
|
||
})
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod test;
|