tamer: tests: System test support

This provides a test harness for running shell-based system tests.  The
first of such tests will be introduced in the following commit.

This is done in place of integration tests written in Rust because it will
invoke the final binary exactly as the user or build system (using TAMER)
will, providing greater confidence.  Besides, a lot of things are simply
more convenient to do in shell.  ...though some of you may debate that.

DEV-13708

tamer/Makefile.am

@@ -47,12 +47,21 @@ html-am:
 
 # note that 'cargo check' is something else; see 'cargo --help'
 test: check
-check-am: check-lint check-fmt
+check-am: check-lint check-fmt check-cargo check-system
+
+.PHONY: check-cargo
+check-cargo:
 	RUSTFLAGS="$(RUSTFLAGS)" @CARGO@ +@RUST_TC@ @CARGO_FLAGS@ test --quiet @FEATURES@
 
+.PHONY: check-system
+check-system: bin
+	tests/run-tests
+
+.PHONY: check-lint
 check-lint:
 	@CARGO@ +@RUST_TC@ @CARGO_FLAGS@ clippy
 
+.PHONY: check-fmt
 check-fmt:
 	@CARGO@ +@RUST_TC@ @CARGO_FLAGS@ fmt -- --check
 

tamer/tests/README.md

@@ -0,0 +1,13 @@
+# System and Integration Tests
+Rust files in this directory will be recognized by Cargo and will be
+automatically compiled and run by `make check`.
+
+Shell scripts prefixed with `test-` will be recognized by our test harness
+and run on `make check`.  These scripts should be preferred when confidence
+in the system end-to-end is required, since they invoke the binaries just
+the same as the user or build process would.
+
+Unit and integration tests written in Rust are located alongside the modules
+they test in [`../src/`](../src/).  Benchmarks are in
+[`../benches`](../benches).
+

tamer/tests/run-tests

@@ -0,0 +1,66 @@
+#!/bin/bash
+# Run all executable `test-*` tests.
+
+set -euo pipefail
+
+mypath=$(dirname "$0")
+. "$mypath/../conf.sh"
+
+warn-verbose-tracing() {
+  if tamer-flag parser-trace-stderr; then
+    cat <<EOM
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!!!  parser-trace-stderr !!!!
+!!!!  is enabled;  output !!!!
+!!!! will be very verbose !!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+EOM
+  fi
+}
+
+find-tests() {
+  find . -name 'test-*' -executable
+}
+
+main() {
+  warn-verbose-tracing
+
+  local -i count=0
+  local -i failn=0
+  local fail_names=
+
+  while read -r test; do
+    echo "$test"
+    sed 's/./-/g' <<< "$test"
+
+    ((++count))
+
+    (time -p "$test") || {
+      echo "ERR (exit code $?)"
+      echo
+
+      ((++failn))
+      fail_names="${fail_names:+$fail_names$'\n'}  - $test failed"
+
+      continue
+    }
+
+    echo OK
+    echo
+  done < <(find-tests)
+
+  warn-verbose-tracing
+
+  if ((failn > 0)); then
+    printf 'There were %d test failure(s):\n%s\n' "$failn" "$fail_names"
+    echo -e '\e[31m'
+  fi
+
+  printf '%d tests, %d failed\n' "$count" "$failn"
+  echo -en '\e[0m'
+
+  [ "$failn" -eq 0 ]
+}
+
+main "$@"
+