progtest: Exit with non-zero status on test failure

Not a very useful test runner if it doesn't ever fail, now is it?

* Makefile.am (check): Invoke new test/runner-test.  Depend on modindex.
* bin/runner.js: Exit with non-zero status on assertion failure.
* test/_stub: Add stub program with good and bad test cases to test
    exit code.
* test/runner-test: Add system test.

progtest/Makefile.am

@@ -33,9 +33,10 @@ modindex: $(nsindex)
 	$(CURDIR)/build-aux/gen-index "$*" > "$@"
 
 test: check
-check:
+check: modindex
 	PATH="$(PATH):$(CURDIR)/node_modules/mocha/bin" \
 		mocha @NODE_DESTRUCTURE@ --recursive test/
+	test/runner-test
 
 browserify: tame-progtest.js
 tame-progtest.js: check modindex

progtest/bin/runner.js

@@ -33,4 +33,17 @@ const runner = require( '../src/env' ).console(
 );
 
 runner( case_yaml )
-    .catch( e => console.error( e ) );
+    .then( results =>
+    {
+        const failed = results.some(
+            ( { failures } ) => failures.length
+        );
+
+        process.exit( +failed );
+    } )
+    .catch( e =>
+    {
+        console.error( e );
+        process.exit( 1 );
+    } );
+

progtest/test/_stub/bad.yml

@@ -0,0 +1,7 @@
+# Simple test case that should fail
+
+- description: Failure
+  data:
+    in: 0
+  expect:
+    out: 1

progtest/test/_stub/good.yml

@@ -0,0 +1,7 @@
+# Simple test case that should succeed
+
+- description: Success
+  data:
+    in: 1
+  expect:
+    out: 1

progtest/test/_stub/program.js

@@ -0,0 +1,25 @@
+/**
+ * Stub program ("rater")
+ *
+ *  Copyright (C) 2018 R-T Specialty, LLC.
+ *
+ *  This file is part of TAME.
+ *
+ *  TAME is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+exports.rater = data =>
+{
+    return { vars: { out: data.in } };
+};

progtest/test/runner-test

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Runner script system test
+#
+#  Copyright (C) 2018 R-T Specialty, LLC.
+#
+#  This file is part of TAME.
+#
+#  TAME is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -euo pipefail
+
+cd "$( dirname "$0" )"
+
+declare -r bin=../bin
+
+
+main()
+{
+  set -x
+
+  # should succeed
+  "$bin"/runner _stub/program.js _stub/good.yml >/dev/null
+
+  # should fail
+  "$bin"/runner _stub/program.js _stub/bad.yml >/dev/null && exit 1 || true
+}
+
+main "$@"