From 92eb991df33e87ad8e1d1b432efb7171dfc2702b Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Wed, 17 May 2023 11:14:00 -0400 Subject: [PATCH] tamer: benches: Remove asg and asg_lower_xmle microbenchmarks These benchmarks were useful as TAMER was in its infancy and I was trying to gain an intuition for working with Rust. But they are now out of date, and there are better ways to measure TAMER's performance, including running it on real-world data (which wasn't possible previously) and through profiling tools like Valgrind. With that said, these types of benchmarks _would_ be useful for helping to dig down into improvements that could be made, at a glance. The problem is, they aren't testing anything new, and they're also testing something I'm about to extract from `Asg`. It is not worth the ongoing maintenance cost. So benchmarks may be reintroduced in the future if they are found to be valuable. DEV-13162 --- tamer/benches/asg.rs | 491 -------------------------------- tamer/benches/asg_lower_xmle.rs | 121 -------- 2 files changed, 612 deletions(-) delete mode 100644 tamer/benches/asg.rs delete mode 100644 tamer/benches/asg_lower_xmle.rs diff --git a/tamer/benches/asg.rs b/tamer/benches/asg.rs deleted file mode 100644 index a87f3b82..00000000 --- a/tamer/benches/asg.rs +++ /dev/null @@ -1,491 +0,0 @@ -// Abstract semantic graph benchmarks -// -// Copyright (C) 2014-2023 Ryan Specialty, LLC. -// -// This file is part of TAME. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// Note that the baseline tests have a _suffix_ rather than a prefix so that -// they are still grouped with the associated test in the output, since it's -// sorted lexically by function name. - -#![feature(test)] - -extern crate tamer; -extern crate test; - -use test::Bencher; - -mod base { - use super::*; - use tamer::{ - asg::{ - AsgError, DefaultAsg, Ident, IdentKind, ObjectIndex, - ObjectIndexRelTo, Source, - }, - parse::util::SPair, - span::UNKNOWN_SPAN, - sym::GlobalSymbolIntern, - }; - - type Sut = DefaultAsg; - - fn declare( - asg: &mut Sut, - name: SPair, - kind: IdentKind, - src: Source, - ) -> Result, AsgError> { - let oi_root = asg.root(name); - oi_root.declare(asg, name, kind, src) - } - - fn declare_extern( - asg: &mut Sut, - name: SPair, - kind: IdentKind, - src: Source, - ) -> Result, AsgError> { - lookup_or_missing(asg, name).declare_extern(asg, name, kind, src) - } - - fn lookup(asg: &mut Sut, name: SPair) -> Option> { - let oi_root = asg.root(name); - asg.lookup(oi_root, name) - } - - fn lookup_or_missing(asg: &mut Sut, name: SPair) -> ObjectIndex { - let oi_root = asg.root(name); - oi_root.lookup_or_missing(asg, name) - } - - fn interned_n(n: u16) -> Vec { - (0..n) - .map(|i| SPair(i.to_string().intern(), UNKNOWN_SPAN)) - .collect() - } - - #[bench] - fn declare_1_000(bench: &mut Bencher) { - let mut sut = Sut::new(); - let xs = interned_n(1_000); - - bench.iter(|| { - xs.iter() - .map(|i| { - declare(&mut sut, *i, IdentKind::Meta, Source::default()) - }) - .for_each(drop); - }); - } - - #[bench] - fn declare_1_000_full_inital_capacity(bench: &mut Bencher) { - let mut sut = Sut::with_capacity(1024, 1024); - let xs = interned_n(1_000); - - bench.iter(|| { - xs.iter() - .map(|i| { - declare(&mut sut, *i, IdentKind::Meta, Source::default()) - }) - .for_each(drop); - }); - } - - #[bench] - fn declare_1_000_prog_ident_size(bench: &mut Bencher) { - let mut sut = Sut::new(); - let xs = interned_n(1_000); - - bench.iter(|| { - xs.iter() - .map(|i| { - declare(&mut sut, *i, IdentKind::Meta, Source::default()) - }) - .for_each(drop); - }); - } - - #[bench] - fn declare_extern_1_000(bench: &mut Bencher) { - let mut sut = Sut::new(); - let xs = interned_n(1_000); - - bench.iter(|| { - xs.iter() - .map(|i| { - declare_extern( - &mut sut, - *i, - IdentKind::Meta, - Source::default(), - ) - }) - .for_each(drop); - }); - } - - #[bench] - fn resolve_extern_1_000(bench: &mut Bencher) { - let mut sut = Sut::new(); - let xs = interned_n(1_000); - - xs.iter().for_each(|sym| { - let _ = declare_extern( - &mut sut, - *sym, - IdentKind::Meta, - Source::default(), - ); - }); - - // Bench only the resolution, not initial declare. - bench.iter(|| { - xs.iter() - .map(|sym| { - declare(&mut sut, *sym, IdentKind::Meta, Source::default()) - }) - .for_each(drop); - }); - } - - // N.B.: This benchmark isn't easily comparable to the others because - // `set_fragment` takes ownership over a string, and so we have to clone - // strings for each call. - #[bench] - fn set_fragment_1_000_with_new_str(bench: &mut Bencher) { - let mut sut = Sut::new(); - let xs = interned_n(1_000); - - xs.iter().for_each(|sym| { - declare(&mut sut, *sym, IdentKind::Meta, Source::default()) - .unwrap(); - }); - - // Bench only the resolution, not initial declare. - bench.iter(|| { - xs.iter() - .map(|sym| { - lookup_or_missing(&mut sut, *sym) - .set_fragment(&mut sut, "".into()) - }) // see N.B. - .for_each(drop); - }); - } - - #[bench] - fn lookup_1_000(bench: &mut Bencher) { - let mut sut = Sut::new(); - let xs = interned_n(1_000); - - xs.iter().for_each(|sym| { - let _ = declare(&mut sut, *sym, IdentKind::Meta, Source::default()); - }); - - bench.iter(|| { - xs.iter() - .map(|sym| lookup(&mut sut, *sym).unwrap()) - .for_each(drop); - }); - } - - #[bench] - fn get_1_000(bench: &mut Bencher) { - let mut sut = Sut::new(); - let xs = interned_n(1_000); - - let orefs = xs - .iter() - .map(|sym| { - declare(&mut sut, *sym, IdentKind::Meta, Source::default()) - .unwrap() - }) - .collect::>(); - - bench.iter(|| { - orefs - .iter() - .map(|oref| sut.get(*oref).unwrap()) - .for_each(drop); - }); - } - - // All dependencies on a single node. Petgraph does poorly with - // supernodes at the time of writing, relatively speaking. - #[bench] - fn add_dep_1_000_to_single_node(bench: &mut Bencher) { - let mut sut = Sut::new(); - let xs = interned_n(1_000); - - let orefs = xs - .iter() - .map(|sym| { - declare(&mut sut, *sym, IdentKind::Meta, Source::default()) - .unwrap() - }) - .collect::>(); - - let root = orefs[0]; - - // Note that this adds all edges to one node - bench.iter(|| { - orefs - .iter() - .map(|oref| root.add_opaque_dep(&mut sut, *oref)) - .for_each(drop); - }); - } - - // Same as above but only one edge per node. - #[bench] - fn add_dep_1_000_one_edge_per_node(bench: &mut Bencher) { - let mut sut = Sut::new(); - let xs = interned_n(1_000); - - let orefs = xs - .iter() - .map(|sym| { - declare(&mut sut, *sym, IdentKind::Meta, Source::default()) - .unwrap() - }) - .collect::>(); - - bench.iter(|| { - orefs - .iter() - .zip(orefs.iter().cycle().skip(1)) - .map(|(from, to)| from.add_opaque_dep(&mut sut, *to)) - .for_each(drop); - }); - } - - #[bench] - fn has_dep_1_000_single_node(bench: &mut Bencher) { - let mut sut = Sut::new(); - let xs = interned_n(1_000); - - let orefs = xs - .iter() - .map(|sym| { - declare(&mut sut, *sym, IdentKind::Meta, Source::default()) - .unwrap() - }) - .collect::>(); - - let root = orefs[0]; - - orefs.iter().for_each(|oref| { - root.add_opaque_dep(&mut sut, *oref); - }); - - bench.iter(|| { - orefs - .iter() - .map(|oref| root.has_edge_to(&sut, *oref)) - .for_each(drop); - }); - } - - // Same as above but only one edge per node. - #[bench] - fn has_dep_1_000_one_edge_per_node(bench: &mut Bencher) { - let mut sut = Sut::new(); - let xs = interned_n(1_000); - - let orefs = xs - .iter() - .map(|sym| { - declare(&mut sut, *sym, IdentKind::Meta, Source::default()) - .unwrap() - }) - .collect::>(); - - orefs.iter().zip(orefs.iter().cycle().skip(1)).for_each( - |(from, to)| { - from.add_opaque_dep(&mut sut, *to); - }, - ); - - bench.iter(|| { - orefs - .iter() - .zip(orefs.iter().cycle().skip(1)) - .map(|(from, to)| from.has_edge_to(&sut, *to)) - .for_each(drop); - }); - } -} - -mod object { - use super::*; - - mod ident { - use super::*; - use tamer::{ - asg::{Ident, IdentKind, Source}, - parse::util::SPair, - span::UNKNOWN_SPAN as S0, - }; - - type Sut = Ident; - - #[bench] - fn declare_1_000(bench: &mut Bencher) { - let sym = SPair("sym".into(), S0); - - bench.iter(|| { - (0..1000).map(|_| Sut::declare(sym)).for_each(drop); - }); - } - - #[bench] - fn resolve_1_000_missing(bench: &mut Bencher) { - let sym = SPair("sym".into(), S0); - - bench.iter(|| { - (0..1000) - .map(|_| { - Sut::declare(sym).resolve( - S0, - IdentKind::Meta, - Source::default(), - ) - }) - .for_each(drop); - }); - } - - #[bench] - fn extern_1_000_missing(bench: &mut Bencher) { - let sym = SPair("sym".into(), S0); - - bench.iter(|| { - (0..1000) - .map(|_| { - Sut::declare(sym).extern_( - S0, - IdentKind::Meta, - Source::default(), - ) - }) - .for_each(drop); - }); - } - - #[bench] - fn resolve_1_000_extern(bench: &mut Bencher) { - let sym = SPair("sym".into(), S0); - - bench.iter(|| { - (0..1000) - .map(|_| { - Sut::declare(sym) - .extern_(S0, IdentKind::Meta, Source::default()) - .unwrap() - .resolve(S0, IdentKind::Meta, Source::default()) - }) - .for_each(drop); - }); - } - - #[bench] - fn resolve_1_000_override(bench: &mut Bencher) { - let sym = SPair("sym".into(), S0); - - bench.iter(|| { - (0..1000) - .map(|_| { - Sut::declare(sym) - .resolve( - S0, - IdentKind::Meta, - Source { - virtual_: true, - ..Default::default() - }, - ) - .unwrap() - .resolve( - S0, - IdentKind::Meta, - Source { - override_: true, - ..Default::default() - }, - ) - }) - .for_each(drop); - }); - } - - // Override encountered before virtual - #[bench] - fn resolve_1_000_override_virt_after_override(bench: &mut Bencher) { - let sym = SPair("sym".into(), S0); - - bench.iter(|| { - (0..1000) - .map(|_| { - Sut::declare(sym) - .resolve( - S0, - IdentKind::Meta, - Source { - override_: true, - ..Default::default() - }, - ) - .unwrap() - .resolve( - S0, - IdentKind::Meta, - Source { - virtual_: true, - ..Default::default() - }, - ) - }) - .for_each(drop); - }); - } - - #[bench] - fn set_fragment_1_000_resolved_with_new_str(bench: &mut Bencher) { - let sym = SPair("sym".into(), S0); - - bench.iter(|| { - (0..1000) - .map(|_| { - Sut::declare(sym) - .resolve(S0, IdentKind::Meta, Source::default()) - .unwrap() - .set_fragment("".into()) - }) - .for_each(drop); - }); - } - - // No need to do all of the others, since they're all the same thing. - #[bench] - fn declared_name_1_000(bench: &mut Bencher) { - let sym = SPair("sym".into(), S0); - - bench.iter(|| { - (0..1000).map(|_| Sut::declare(sym).name()).for_each(drop); - }); - } - } -} diff --git a/tamer/benches/asg_lower_xmle.rs b/tamer/benches/asg_lower_xmle.rs deleted file mode 100644 index 6c1ab3f4..00000000 --- a/tamer/benches/asg_lower_xmle.rs +++ /dev/null @@ -1,121 +0,0 @@ -// Benchmarking of sorting of ASG into xmle sections -// -// Copyright (C) 2014-2023 Ryan Specialty, LLC. -// -// This file is part of TAME. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -#![feature(test)] - -extern crate tamer; -extern crate test; - -use test::Bencher; - -pub(crate) use tamer::{ - asg::{AsgError, DefaultAsg, Ident, IdentKind, ObjectIndex, Source}, - ld::xmle::{lower::sort, Sections}, - num::Dtype, - parse::util::SPair, - span::UNKNOWN_SPAN, - sym::GlobalSymbolIntern, -}; - -type TestAsg = DefaultAsg; - -fn interned_n(n: u16) -> Vec { - (0..n) - .map(|i| SPair(i.to_string().intern(), UNKNOWN_SPAN)) - .collect() -} - -fn declare( - asg: &mut TestAsg, - name: SPair, - kind: IdentKind, - src: Source, -) -> Result, AsgError> { - let oi_root = asg.root(name); - oi_root.declare(asg, name, kind, src) -} - -#[bench] -fn sort_1_with_1_000_existing_supernode(bench: &mut Bencher) { - let mut sut = TestAsg::new(); - let xs = interned_n(1_000); - - let orefs = xs - .iter() - .map(|sym| { - declare( - &mut sut, - *sym, - IdentKind::Rate(Dtype::Integer), - Source::default(), - ) - .unwrap() - }) - .collect::>(); - - let root = orefs[0]; - - // All edges from a single node. - orefs.iter().skip(1).for_each(|to| { - root.add_opaque_dep(&mut sut, *to); - }); - - root.root(&mut sut); - - bench.iter(|| { - drop(sort(&sut, Sections::new())); - }); -} - -#[bench] -fn sort_1_with_1_000_existing_one_edge_per_node_one_path(bench: &mut Bencher) { - let mut sut = TestAsg::new(); - let xs = interned_n(1_000); - - let orefs = xs - .iter() - .map(|sym| { - declare( - &mut sut, - *sym, - IdentKind::Rate(Dtype::Integer), - Source::default(), - ) - .unwrap() - }) - .collect::>(); - - // Note that there's no `cycle` call on the iterator, like the - // above tests, to make sure we don't create a cycle on the - // graph. - orefs - .iter() - .zip(orefs.iter().skip(1)) - .for_each(|(from, to)| { - from.add_opaque_dep(&mut sut, *to); - }); - - let root = orefs[0]; - - root.root(&mut sut); - - bench.iter(|| { - drop(sort(&sut, Sections::new())); - }); -}