diff --git a/.gitignore b/.gitignore index 5fab2621..db0b0567 100644 --- a/.gitignore +++ b/.gitignore @@ -7,8 +7,8 @@ *.info # autotools- and configure-generated -Makefile.in -Makefile +/Makefile.in +/Makefile /aclocal.m4 /*.cache/ /configure diff --git a/doc/.gitignore b/doc/.gitignore index 3f9b6966..14d345e2 100644 --- a/doc/.gitignore +++ b/doc/.gitignore @@ -17,3 +17,6 @@ version.texi *.txt *.html +/Makefile.in +/Makefile + diff --git a/progtest/.gitignore b/progtest/.gitignore new file mode 100644 index 00000000..4d64059f --- /dev/null +++ b/progtest/.gitignore @@ -0,0 +1,2 @@ +/node_modules + diff --git a/progtest/Makefile b/progtest/Makefile new file mode 100644 index 00000000..8c92c08e --- /dev/null +++ b/progtest/Makefile @@ -0,0 +1,25 @@ +# tame-progtest Makefile +# +# Copyright (C) 2018 R-T Specialty, LLC. +# +# This file is part of TAME. +# +# TAME is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +.PHONY: check test + +test: check +check: + PATH="$(PATH):$(CURDIR)/node_modules/mocha/bin" \ + mocha --harmony_destructuring --recursive test/ diff --git a/progtest/README.md b/progtest/README.md new file mode 100644 index 00000000..a18ba1b3 --- /dev/null +++ b/progtest/README.md @@ -0,0 +1,5 @@ +# Program Testing +A /program/ is a top-level package (either marked as with `@program="true"`, +or with a root `rater` node). This system provides a means of writing and +running test cases. + diff --git a/progtest/bin/runner.js b/progtest/bin/runner.js new file mode 100644 index 00000000..98a134e9 --- /dev/null +++ b/progtest/bin/runner.js @@ -0,0 +1,51 @@ +/** + * Test case runner script + * + * Copyright (C) 2018 R-T Specialty, LLC. + * + * This file is part of TAME. + * + * TAME is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +"use strict"; + +const program = require( process.argv[ 2 ] ); +const filename = process.argv[ 3 ]; + +const fs = require( 'fs' ); +const yaml_reader = require( 'js-yaml' ); + +const TestCase = require( '../src/TestCase' ); +const YamlTestReader = require( '../src/reader/YamlTestReader' ); +const ConstResolver = require( '../src/reader/ConstResolver' ); +const DateResolver = require( '../src/reader/DateResolver' ); +const TestRunner = require( '../src/TestRunner' ); +const ConsoleTestReporter = require( '../src/reporter/ConsoleTestReporter' ); + +const runner = TestRunner( + ConsoleTestReporter( process.stdout ), + program +); + +const reader = YamlTestReader + .use( DateResolver ) + .use( ConstResolver( program ) ) + ( yaml_reader, TestCase ); + +const cases = reader.loadCases( + fs.readFileSync( filename, 'utf8' ) +); + +const results = runner.runTests( cases ); diff --git a/progtest/package.json b/progtest/package.json new file mode 100644 index 00000000..71667968 --- /dev/null +++ b/progtest/package.json @@ -0,0 +1,15 @@ +{ + "name": "tame-progtest", + "description": "TAME Program testing", + "version": "0.0.0", + "author": "R-T Specialty, LLC", + + "dependencies": { + "easejs": "0.2.9", + "js-yaml": "3.10.0" + }, + "devDependencies": { + }, + + "license": "GPL-3.0+" +} diff --git a/progtest/src/TestCase.js b/progtest/src/TestCase.js new file mode 100644 index 00000000..5ea39a39 --- /dev/null +++ b/progtest/src/TestCase.js @@ -0,0 +1,103 @@ +/** + * Test case + * + * Copyright (C) 2018 R-T Specialty, LLC. + * + * This file is part of TAME. + * + * TAME is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +"use strict"; + +const { Class } = require( 'easejs' ); + + +module.exports = Class( 'TestCase', +{ + /** + * Test case data + * + * @type {Object} test case data + */ + 'private _caseData': {}, + + get description() + { + return this._caseData.description || ""; + }, + + get data() + { + return this._caseData.data || {}; + }, + + get expect() + { + return this._caseData.expect || {}; + }, + + + constructor( case_data ) + { + this._caseData = case_data; + }, + + + 'public mapEachValue'( callback ) + { + const [ new_data, new_expect ] = [ this.data, this.expect ].map( src => + { + const new_src = {}; + + Object.keys( src ).forEach( + key => new_src[ key ] = this._visitDeep( + src[ key ], + callback + ) + ); + + return new_src; + } ); + + return module.exports( { + description: this.description, + data: new_data, + expect: new_expect, + } ); + }, + + + /** + * Recursively resolve constants + * + * Only scalars and arrays are supported + * + * @param {number|Array} input scalar or array of inputs + * @param {Object} consts constant mapping + * + * @return {number|Array} resolved value(s) + */ + 'private _visitDeep'( val, callback ) + { + if ( Array.isArray( val ) ) + { + return val.map( + x => this._visitDeep( x, callback ) + ); + } + + return callback( val ); + } +} ); diff --git a/progtest/src/TestRunner.js b/progtest/src/TestRunner.js new file mode 100644 index 00000000..92690187 --- /dev/null +++ b/progtest/src/TestRunner.js @@ -0,0 +1,153 @@ +/** + * Test case runner + * + * Copyright (C) 2018 R-T Specialty, LLC. + * + * This file is part of TAME. + * + * TAME is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +"use strict"; + +const { Class } = require( 'easejs' ); + + +/** + * Run test cases and report results + */ +module.exports = Class( 'TestRunner', +{ + /** + * SUT + * + * @type {Program} + */ + 'private _program': null, + + /** + * Test reporter + * + * @type {TestReporter} + */ + 'private _reporter': null, + + + /** + * Initialize runner for program PROGRAM + * + * @param {TestReporter} reporter test reporter + * @param {Program} program SUT + */ + constructor( reporter, program ) + { + // primitive check to guess whether this might be a program + if ( typeof program.rater !== 'function' ) + { + throw TypeError( "program#rater is not a function" ); + } + + this._reporter = reporter; + this._program = program; + }, + + + /** + * Run set of test cases + * + * @param {Array} dfns array of TestCases + * + * @return {Array>} results + */ + 'public runTests'( dfns ) + { + const total = dfns.length; + + this._reporter.preRun( total ); + + const results = dfns.map( + ( test, i ) => this._runTest( test, i, total ) + ); + + this._reporter.done( results ); + + return results; + }, + + + /** + * Run individual test case + * + * @param {Object} _ source test case + * @param {number} test_i test index + * @param {number} total total number of tests + * + * @return {Object} test results + */ + 'private _runTest'( { description: desc, data, expect }, test_i, total ) + { + // no input map---#rate uses params directly + const result = this._program.rater( data ).vars; + + const cmp = Object.keys( expect ).map( + field => [ + field, + this._deepCompare( expect[ field ], result[ field ] ) + ] + ); + + const failures = cmp.filter( ( [ , ok ] ) => !ok ) + .map( ( [ field ] ) => ( { + field: field, + expect: expect[ field ], + result: result[ field ], + } ) ); + + const succeeded = cmp.length - failures.length; + + const result_data = { + desc: desc, + i: test_i, + total: cmp.length, + failures: failures, + }; + + this._reporter.testCaseResult( result_data, total ); + + return result_data; + }, + + + /** + * Recursively compare values (scalar, array) + * + * @param {number|Array} x first value + * @param {number|Array} y second value + * + * @return {boolean} whether X deeply equals Y + */ + 'private _deepCompare'( x, y ) + { + // vector/matrix/etc + if ( Array.isArray( x ) ) + { + return Array.isArray( y ) + && ( x.length === y.length ) + && x.every( ( xval, i ) => xval === y[ i ] ); + } + + // scalar + return x === y; + }, +} ); diff --git a/progtest/src/reader/ConstResolver.js b/progtest/src/reader/ConstResolver.js new file mode 100644 index 00000000..b8861f62 --- /dev/null +++ b/progtest/src/reader/ConstResolver.js @@ -0,0 +1,110 @@ +/** + * Constant resolver for test case reader + * + * Copyright (C) 2018 R-T Specialty, LLC. + * + * This file is part of TAME. + * + * TAME is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +"use strict"; + + +const { Trait } = require( 'easejs' ); +const TestReader = require( './TestReader' ); + + +/** + * Resolve program constants by replacing them with their numeric value + * + * The result is a loaded YAML file that contains only numeric input. All + * non-numeric input is interpreted as a constant. + * + * Any non-numeric value is considered to be a constant, so it is important + * to perform all other data transformations before applying constant + * resolution. + */ +module.exports = Trait( 'ConstResolver' ) + .implement( TestReader ) + .extend( +{ + /** + * Program from which to load constants + * + * @type {Program} + */ + 'private _program': null, + + + /** + * Initialize with program from which to load constants + * + * @param {Program} program source program + */ + __mixin( program ) + { + this._program = program; + }, + + + /** + * Load test cases and resolve constants + * + * @param {*} src data source + * + * @return {Array} array of test cases + */ + 'abstract override public loadCases'( yaml ) + { + const data = this.__super( yaml ); + const { consts } = this._program.rater; + + return data.map( + testcase => testcase.mapEachValue( + value => this._resolve( value, consts ) + ) + ); + }, + + + /** + * Resolve constant + * + * If the constant is not known (via CONSTS), an error is thrown. + * + * @param {number|Array} input scalar or array of inputs + * @param {Object} consts constant mapping + * + * @throws {Error} if constant is unknown in CONSTS + * + * @return {number|Array} resolved value(s) + */ + 'private _resolve'( input, consts ) + { + // already a number, return as-is + if ( !isNaN( +input ) ) + { + return input; + } + + const result = consts[ input ]; + if ( result === undefined ) + { + throw Error( `unknown constant: ${input}` ); + } + + return result; + } +} ); diff --git a/progtest/src/reader/DateResolver.js b/progtest/src/reader/DateResolver.js new file mode 100644 index 00000000..319254e4 --- /dev/null +++ b/progtest/src/reader/DateResolver.js @@ -0,0 +1,61 @@ +/** + * Date resolver for test case reader + * + * Copyright (C) 2018 R-T Specialty, LLC. + * + * This file is part of TAME. + * + * TAME is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +"use strict"; + + +const { Trait } = require( 'easejs' ); +const TestReader = require( './TestReader' ); + + +/** + * Resolve dates of the format MM/DD/YYYY to Unix timestamps + * + * This allows easily readable dates to be included in test cases without + * having to worry about Unix timestamps. For higher precision, Unix + * timestamps must be used. + */ +module.exports = Trait( 'DateResolver' ) + .implement( TestReader ) + .extend( +{ + /** + * Load test cases and resolve dates + * + * Dates will be replaced with Unix timestamps. + * + * @param {*} src data source + * + * @return {Array} array of test cases + */ + 'abstract override public loadCases'( src ) + { + const data = this.__super( src ); + + return data.map( + testcase => testcase.mapEachValue( + value => ( /^\d{2}\/\d{2}\/\d{4}$/.test( value ) ) + ? ( ( new Date( value ) ).getTime() / 1000 ) + : value + ) + ); + } +} ); diff --git a/progtest/src/reader/TestReader.js b/progtest/src/reader/TestReader.js new file mode 100644 index 00000000..717b12ca --- /dev/null +++ b/progtest/src/reader/TestReader.js @@ -0,0 +1,40 @@ +/** + * Test case reader + * + * Copyright (C) 2018 R-T Specialty, LLC. + * + * This file is part of TAME. + * + * TAME is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +"use strict"; + +const { Interface } = require( 'easejs' ); + + +module.exports = Interface( 'TestReader', +{ + /** + * Load test cases from an implementation-defined data source SRC + * + * The produced object will be an array of cases, each containing a + * `description`, `data`, and `expect`. + * + * @param {*} src data source + * + * @return {Array} array of test cases + */ + 'public loadCases': [ 'src' ], +} ); diff --git a/progtest/src/reader/YamlTestReader.js b/progtest/src/reader/YamlTestReader.js new file mode 100644 index 00000000..a28c1f33 --- /dev/null +++ b/progtest/src/reader/YamlTestReader.js @@ -0,0 +1,79 @@ +/** + * YAML test case reader + * + * Copyright (C) 2018 R-T Specialty, LLC. + * + * This file is part of TAME. + * + * TAME is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +"use strict"; + +const { Class } = require( 'easejs' ); +const TestReader = require( './TestReader' ); + + +module.exports = Class( 'YamlTestReader' ) + .implement( TestReader ) + .extend( +{ + /** + * YAML parser + * + * @type {Object} + */ + 'private _yamlParser': null, + + /** + * TestCase constructor + * + * @type {function(Object)} + */ + 'private _createTestCase': null, + + + /** + * Initialize with YAML parser + * + * The parser must conform to the API of `js-yaml`. + * + * @param {Object} yaml_parser YAML parser + * @param {function(Object)} test_case_ctor TestCase constructor + */ + constructor( yaml_parser, test_case_ctor ) + { + this._yamlParser = yaml_parser; + this._createTestCase = test_case_ctor; + }, + + + /** + * Load test cases from a YAML string + * + * The produced object will be an array of cases, each containing a + * `description`, `data`, and `expect`. + * + * @param {string} src source YAML + * + * @return {Array} array of test cases + */ + 'virtual public loadCases'( yaml ) + { + const data = this._yamlParser.safeLoad( yaml ) + .map( this._createTestCase ); + + return data; + }, +} ); diff --git a/progtest/src/reporter/ConsoleTestReporter.js b/progtest/src/reporter/ConsoleTestReporter.js new file mode 100644 index 00000000..2a16e038 --- /dev/null +++ b/progtest/src/reporter/ConsoleTestReporter.js @@ -0,0 +1,183 @@ +/** + * Console test reporter + * + * Copyright (C) 2018 R-T Specialty, LLC. + * + * This file is part of TAME. + * + * TAME is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +"use strict"; + +const { Class } = require( 'easejs' ); + + +/** + * Real-time reporting of test cases to the console + * + * Test cases will be output in a block of dots (success) or 'F's (failure), + * in a style similar to PHPUnit. If failures occur, they will be output to + * in more detail after all tests have run. + */ +module.exports = Class( 'ConsoleTestReporter', +{ + /** + * Standard out + * + * @type {Object} standard out + */ + 'private _stdout': null, + + + /** + * Initialize reporter with target console + * + * STDOUT must follow Node.js' API. + * + * @param {Object} stdout standard out + */ + constructor( stdout ) + { + this._stdout = stdout; + }, + + + /** + * Invoked before tests are run + * + * The only information provided here is the number of test cases to be + * run, which can be used to produce a progress indicator. + * + * @param {number} total number of test cases + * + * @return {undefined} + */ + 'public preRun'( total ) + { + // this reporter does nothing with this method + }, + + + /** + * Invoked for each test case immediately after it has been run + * + * For the format of RESULT, see TestRunner. + * + * @param {Object} result test case result + * + * @return {undefined} + */ + 'public testCaseResult'( result, total ) + { + const { i, failures } = result; + + const ind = ( failures.length === 0 ) + ? '.' + : 'F'; + + const sep = ( i % 50 === 49 ) + ? ` ${i+1}/${total}\n` + : ''; + + this._stdout.write( ind + sep ); + }, + + + /** + * Invoked after all test cases have been run + * + * RESULTS is an array containing each result that was previously + * reported to `#testCaseResult`. + * + * A final line will be output, preceded by an empty line, summarizing + * the number of tests, assertions, and failures for each. + * + * @param {Array} results all test results + * + * @return {undefined} + */ + 'public done'( results ) + { + this._outputFailureReport( results ); + this._outputSummary( results ); + }, + + + /** + * For each failure, output each expected and resulting value + * + * Failures are prefixed with a 1-indexed number. + * + * @param {Object} result test case result} + * + * @return {undefined} + */ + 'private _outputFailureReport'( results ) + { + const report = results + .filter( ( { failures } ) => failures.length > 0 ) + .map( this._reportTestFailure.bind( this ) ) + .join( '\n' ) + + this._stdout.write( "\n\n" + report ); + }, + + + /** + * Generate report for test case failure + * + * @param {Object} _ test case result data + * + * @return {string} report + */ + 'private _reportTestFailure'( { i, desc, failures } ) + { + return `[#${i+1}] ${desc}\n` + + failures.map( ( { field, expect, result } ) => + ` ${field}:\n` + + ` expected: ` + JSON.stringify( expect ) + `\n` + + ` result: ` + JSON.stringify( result ) + `\n` + ).join( '' ); + }, + + + /** + * Output a line, preceded by an empty line, summarizing the number of + * tests, assertions, and failures for each + * + * @param {Array} results all test results + * + * @return {undefined} + */ + 'private _outputSummary'( results ) + { + const [ failed, afailed, acount ] = results.reduce( + ( [ failed, afailed, acount ], { failures, total } ) => + [ + ( failed + +( failures.length > 0 ) ), + ( afailed + failures.length ), + ( acount + total ) + ], + [ 0, 0, 0 ] + ); + + const test_total = results.length; + + this._stdout.write( + `\n${test_total} tests, ${failed} failed (` + + `${acount} assertions, ${afailed} failures)` + ); + }, +} ); diff --git a/progtest/src/reporter/NullTestReporter.js b/progtest/src/reporter/NullTestReporter.js new file mode 100644 index 00000000..d29fb051 --- /dev/null +++ b/progtest/src/reporter/NullTestReporter.js @@ -0,0 +1,79 @@ +/** + * Reporter that does nothing + * + * Copyright (C) 2018 R-T Specialty, LLC. + * + * This file is part of TAME. + * + * TAME is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +"use strict"; + +const { Class } = require( 'easejs' ); + + +/** + * Reporter that does nothing + * + * This is useful if you want a fully background process. + */ +module.exports = Class( 'NullTestReporter', +{ + /** + * Invoked before tests are run + * + * The only information provided here is the number of test cases to be + * run, which can be used to produce a progress indicator. + * + * @param {number} total number of test cases + * + * @return {undefined} + */ + 'public preRun'( total ) + { + // this reporter does nothing with this method + }, + + + /** + * Invoked for each test case immediately after it has been run + * + * For the format of RESULT, see TestRunner. + * + * @param {Object} result test case result + * + * @return {undefined} + */ + 'public testCaseResult'( result, total ) + { + // this reporter does nothing with this method + }, + + + /** + * Invoked after all test cases have been run + * + * RESULTS is an array containing each result that was previously + * reported to `#testCaseResult`. + * + * @param {Array} results all test results + * + * @return {undefined} + */ + 'public done'( results ) + { + // this reporter does nothing with this method + }, +} ); diff --git a/progtest/src/reporter/TestReporter.js b/progtest/src/reporter/TestReporter.js new file mode 100644 index 00000000..ee7ce8a0 --- /dev/null +++ b/progtest/src/reporter/TestReporter.js @@ -0,0 +1,68 @@ +/** + * Test reporter + * + * Copyright (C) 2018 R-T Specialty, LLC. + * + * This file is part of TAME. + * + * TAME is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +"use strict"; + +const { Interface } = require( 'easejs' ); + + +/** + * Real-time reporting of test cases + */ +module.exports = Interface( 'TestReporter', +{ + /** + * Invoked before tests are run + * + * The only information provided here is the number of test cases to be + * run, which can be used to produce a progress indicator. + * + * @param {number} total number of test cases + * + * @return {undefined} + */ + 'public preRun': [ 'total' ], + + + /** + * Invoked for each test case immediately after it has been run + * + * For the format of RESULT, see TestRunner. + * + * @param {Object} result test case result + * + * @return {undefined} + */ + 'public testCaseResult': [ 'result' ], + + + /** + * Invoked after all test cases have been run + * + * RESULTS is an array containing each result that was previously + * reported to `#testCaseResult`. + * + * @param {Array} results all test results + * + * @return {undefined} + */ + 'public done': [ 'results' ], +} ); diff --git a/progtest/test/TestCaseTest.js b/progtest/test/TestCaseTest.js new file mode 100644 index 00000000..95b46d69 --- /dev/null +++ b/progtest/test/TestCaseTest.js @@ -0,0 +1,99 @@ +/** + * Tests TestCase + * + * Copyright (C) 2018 R-T Specialty, LLC. + * + * This file is part of TAME. + * + * TAME is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +"use strict"; + +const { expect } = require( 'chai' ); +const Sut = require( '../src/TestCase' ); + + +describe( "TestCase", () => +{ + it( "allows retrieving raw data", () => + { + const data = { + description: "Foo bar", + data: { foo: [ 5 ] }, + expect: { bar: [ 1 ] }, + }; + + const sut = Sut( data ); + + expect( sut.description ).to.equal( data.description ); + expect( sut.data ).to.deep.equal( data.data ); + expect( sut.expect ).to.deep.equal( data.expect ); + } ); + + + it( "provides sane defaults for missing data", () => + { + const sut = Sut( {} ); + + expect( sut.description ).to.equal( "" ); + expect( sut.data ).to.deep.equal( {} ); + expect( sut.expect ).to.deep.equal( {} ); + } ); + + + describe( "#mapEachValue", () => + { + it( "visits each 'data' and 'expect' value", () => + { + // tests scalar, vector, matrix; mixed with non-constants + const testcase = { + description: 'test desc', + + data: { + foo: 'bar', + bar: [ 'baz', 'quux' ], + baz: [ [ 'quuux', 'foox' ], [ 'moo', 'cow' ] ], + }, + expect: { + quux: 'out', + quuux: [ 'of', 'names' ], + }, + }; + + const expected = { + data: { + foo: 'OKbar', + bar: [ 'OKbaz', 'OKquux' ], + baz: [ [ 'OKquuux', 'OKfoox' ], [ 'OKmoo', 'OKcow' ] ], + }, + expect: { + quux: 'OKout', + quuux: [ 'OKof', 'OKnames' ], + }, + }; + + const result = Sut( testcase ).mapEachValue( val => `OK${val}` ); + + // derived from the original + expect( result.description ).to.equal( testcase.description ); + expect( result.data ).to.deep.equal( expected.data ); + expect( result.expect ).to.deep.equal( expected.expect ); + + // but not the original (should return a new object) + expect( result.data ).to.not.equal( testcase.data ); + expect( result.expect ).to.not.equal( testcase.expect ); + } ); + } ); +} ); diff --git a/progtest/test/TestRunnerTest.js b/progtest/test/TestRunnerTest.js new file mode 100644 index 00000000..fc06e04b --- /dev/null +++ b/progtest/test/TestRunnerTest.js @@ -0,0 +1,144 @@ +/** + * Tests TestReader + * + * Copyright (C) 2018 R-T Specialty, LLC. + * + * This file is part of TAME. + * + * TAME is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +"use strict"; + +const { expect } = require( 'chai' ); +const { Class } = require( 'easejs' ); +const Sut = require( '../src/TestRunner' ); +const TestReporter = require( '../src/reporter/TestReporter' ); +const NullTestReporter = require( '../src/reporter/NullTestReporter' ); + + +describe( "TestRunner", () => +{ + it( "runs each test against given program", () => + { + const given = []; + + const program = { + rater( data ) + { + return rate_results[ given.push( data ) - 1 ]; + } + }; + + const test_cases = [ + { + description: "first", + data: { a: 1 }, + expect: { foo: 1 }, + }, + { + description: "second", + data: { a: 2 }, + expect: { + foo: [ 1, 2 ], + bar: [ 3, 1 ], + baz: [ 4, 2 ], + }, + }, + ]; + + const rate_results = [ + // no failures + { vars: { foo: 1 } }, + + // bar, baz failures + { vars: { + foo: [ 1, 2 ], + bar: [ 3, 4 ], + baz: [ 4, 5 ], + } }, + ]; + + const expect_failures = [ + [], + [ + { + field: 'bar', + expect: test_cases[ 1 ].expect.bar, + result: rate_results[ 1 ].vars.bar, + }, + { + field: 'baz', + expect: test_cases[ 1 ].expect.baz, + result: rate_results[ 1 ].vars.baz, + }, + ] + ]; + + const results = Sut( NullTestReporter(), program ) + .runTests( test_cases ); + + test_cases.forEach( ( test_case, i ) => + { + const result = results[ i ]; + + expect( result.desc ).to.equal( test_case.description ); + expect( result.i ).to.equal( i ); + expect( result.total ).to.equal( + Object.keys( test_case.expect ).length + ); + expect( result.failures ).to.deep.equal( expect_failures[ i ] ); + } ); + } ); + + + it( "invokes reporter before, during, and after test cases", done => + { + let pre = false; + let results = []; + + const program = { rater: () => ( { vars: {} } ) }; + + const mock_reporter = Class.implement( TestReporter ).extend( + { + preRun( total ) + { + expect( total ).to.equal( 2 ); + pre = true; + }, + + testCaseResult( result, total ) + { + expect( pre ).to.equal( true ); + expect( total ).to.equal( 2 ); + + results.push( result ); + }, + + done( given_results ) + { + expect( pre ).to.equal( true ); + expect( results ).to.deep.equal( given_results ); + + done(); + }, + } )(); + + // see done() above + Sut( mock_reporter, program ).runTests( [ + { description: '', data: {}, expect: {} }, + { description: '', data: {}, expect: {} }, + ] ); + } ); +} ); diff --git a/progtest/test/reader/ConstResolverTest.js b/progtest/test/reader/ConstResolverTest.js new file mode 100644 index 00000000..2060f2a7 --- /dev/null +++ b/progtest/test/reader/ConstResolverTest.js @@ -0,0 +1,104 @@ +/** + * Tests ConstResolver + * + * Copyright (C) 2018 R-T Specialty, LLC. + * + * This file is part of TAME. + * + * TAME is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +"use strict"; + +const { expect } = require( 'chai' ); +const { Class } = require( 'easejs' ); +const TestCase = require( '../../src/TestCase' ); +const TestReader = require( '../../src/reader/TestReader' ); +const Sut = require( '../../src/reader/ConstResolver' ); + +const StubTestReader = Class.implement( TestReader ).extend( +{ + constructor( parsed_data ) + { + this.parsedData = parsed_data; + }, + + 'virtual public loadCases'( _ ) + { + return this.parsedData; + } +} ); + + +describe( "ConstResolver", () => +{ + [ 'data', 'expect' ].forEach( field => + { + it( `replaces known ${field} constants from program`, () => + { + const program = { + rater: { + consts: { FOO: 1, BAR: 2 }, + }, + }; + + // tests scalar, vector, matrix; mixed with non-constants + const parsed_data = [ + TestCase( { [field]: { foo: 'FOO', bar: 4 } } ), + TestCase( + { [field]: { + foo: [ 'FOO', 'BAR', 5 ], + bar: [ [ 'FOO', 3 ], [ 'FOO', 'BAR' ] ], + } } + ), + ]; + + const { FOO, BAR } = program.rater.consts; + + const expected = [ + TestCase( { [field]: { foo: FOO, bar: 4 } } ), + TestCase( + { [field]: { + foo: [ FOO, BAR, 5 ], + bar: [ [ FOO, 3 ], [ FOO, BAR ] ], + } } + ), + ]; + + // anything just to proxy + const given_yaml = 'fooml'; + + const result = StubTestReader + .use( Sut( program ) )( parsed_data ) + .loadCases( given_yaml ); + + result.forEach( + ( tcase, i ) => expect( tcase[ field ] ) + .to.deep.equal( expected[ i ][ field ] ) + ); + } ); + + + it( `throws error on unknown $field constant`, () => + { + const program = { rater: { consts: {} } }; + const parsed_data = [ TestCase( { [field]: { foo: 'UNKNOWN' } } ) ]; + + expect( + () => StubTestReader.use( Sut( program ) )( parsed_data ) + .loadCases( '' ) + ).to.throw( Error, 'UNKNOWN' ); + } ); + } ); +} ); diff --git a/progtest/test/reader/DateResolverTest.js b/progtest/test/reader/DateResolverTest.js new file mode 100644 index 00000000..4b2225d9 --- /dev/null +++ b/progtest/test/reader/DateResolverTest.js @@ -0,0 +1,85 @@ +/** + * Tests DateResolver + * + * Copyright (C) 2018 R-T Specialty, LLC. + * + * This file is part of TAME. + * + * TAME is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +"use strict"; + +const { expect } = require( 'chai' ); +const { Class } = require( 'easejs' ); +const TestCase = require( '../../src/TestCase' ); +const TestReader = require( '../../src/reader/TestReader' ); +const Sut = require( '../../src/reader/DateResolver' ); + +const MockTestReader = Class.implement( TestReader ).extend( +{ + constructor( parsed_data, expected_load ) + { + this.parsedData = parsed_data; + this.expectedLoad = expected_load; + }, + + 'virtual public loadCases'( given ) + { + expect( given ).to.equal( this.expectedLoad ); + return this.parsedData; + } +} ); + + +describe( "DateResolver", () => +{ + [ 'data', 'expect' ].forEach( field => + { + it( `converts ${field} dates into Unix timestamps`, () => + { + const date = '10/25/1989'; + const time = ( new Date( date ) ).getTime() / 1000; + + // tests scalar, vector, matrix; mixed with non-constants + const parsed_data = [ + TestCase( + { [field]: { + foo: [ 5, 'NOTADATE', date ], + } } + ), + ]; + + const expected = [ + TestCase( + { [field]: { + foo: [ 5, 'NOTADATE', time ], + } } + ), + ]; + + // anything just to proxy + const given_yaml = 'fooml'; + + const result = MockTestReader + .use( Sut )( parsed_data, given_yaml ) + .loadCases( given_yaml ); + + result.forEach( + ( tcase, i ) => expect( tcase[ field ] ) + .to.deep.equal( expected[ i ][ field ] ) + ); + } ); + } ); +} ); diff --git a/progtest/test/reader/YamlTestReaderTest.js b/progtest/test/reader/YamlTestReaderTest.js new file mode 100644 index 00000000..9fb26fd0 --- /dev/null +++ b/progtest/test/reader/YamlTestReaderTest.js @@ -0,0 +1,55 @@ +/** + * Tests TestReader + * + * Copyright (C) 2018 R-T Specialty, LLC. + * + * This file is part of TAME. + * + * TAME is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +"use strict"; + +const { expect } = require( 'chai' ); +const Sut = require( '../../src/reader/YamlTestReader' ); + + +describe( "YamlTestReader", () => +{ + it( "parses given yaml", () => + { + const yaml = "foo: bar"; + + const parsed = [ + { + description: "first desc", + data: { "foo": "bar" }, + expect: { "bar": "baz" }, + }, + ]; + + const case_ctor = ( data ) => ( { ok: data } ); + + const mock_parser = { + safeLoad( given ) + { + expect( given ).to.equal( yaml ); + return parsed; + } + }; + + expect( Sut( mock_parser, case_ctor ).loadCases( yaml ) ) + .to.deep.equal( [ { ok: parsed[0] } ] ); + } ); +} ); diff --git a/progtest/test/reporter/ConsoleTestReporterTest.js b/progtest/test/reporter/ConsoleTestReporterTest.js new file mode 100644 index 00000000..07e3e52c --- /dev/null +++ b/progtest/test/reporter/ConsoleTestReporterTest.js @@ -0,0 +1,177 @@ +/** + * Tests ConsoleTestReporter + * + * Copyright (C) 2018 R-T Specialty, LLC. + * + * This file is part of TAME. + * + * TAME is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +"use strict"; + +const { expect } = require( 'chai' ); +const Sut = require( '../../src/reporter/ConsoleTestReporter' ); + + +describe( "ConsoleTestReporter", () => +{ + describe( "#testCaseResult", () => + { + it( "outputs indicator for each test case", () => + { + let output = ''; + + const stdout = { write: str => output += str }; + const sut = Sut( stdout ); + + [ + { i: 0, failures: [] }, + { i: 1, failures: [] }, + { i: 2, failures: [ {} ] }, + { i: 3, failures: [ {}, {} ] }, + { i: 4, failures: [] }, + ].forEach( + result => sut.testCaseResult( result, 5 ) + ); + + expect( output ).to.equal( '..FF.' ); + } ); + + + it( "outputs line break with count after 40 cases", () => + { + let output = ''; + + const stdout = { write: str => output += str }; + const sut = Sut( stdout ); + + const results = ( new Array( 130 ) ).join( '.' ).split( '.' ) + .map( ( _, i ) => ( { i: i, failures: [] } ) ); + + results.forEach( + result => sut.testCaseResult( result, 130 ) + ); + + expect( output ).to.equal( + ( new Array( 51 ) ).join( '.' ) + ' 50/130\n' + + ( new Array( 51 ) ).join( '.' ) + ' 100/130\n' + + ( new Array( 31 ) ).join( '.' ) + ); + } ); + } ); + + + describe( "done", () => + { + it( "outputs report of failures to stdout", () => + { + let output = ''; + + const stdout = { write: str => output += str }; + + const results = [ + { i: 0, total: 1, desc: "test 0", failures: [] }, + { i: 1, total: 2, desc: "test 1", failures: [] }, + + { + i: 2, + total: 3, + desc: "test 2", + failures: [ + { + field: "foo", + expect: [ 1 ], + result: [ 2, 3 ] + }, + ], + }, + { + i: 3, + total: 4, + desc: "test 3", + failures: [ + { + field: "bar", + expect: 2, + result: 3, + }, + { + field: "baz", + expect: [ [ 4 ] ], + result: [ 5 ], + } + ], + }, + ]; + + const stringified = results.map( + result => result.failures.map( + failure => ( { + expect: JSON.stringify( failure.expect ), + result: JSON.stringify( failure.result ), + } ) + ) + ); + + Sut( stdout ).done( results ); + + const fail_output = output.match( /\n\n\[#3\](.|\n)*\n\n/ )[0]; + + // 1-indexed output + expect( fail_output ).to.equal( + `\n\n` + + `[#3] test 2\n` + + ` foo:\n` + + ` expected: ` + stringified[ 2 ][ 0 ].expect + `\n` + + ` result: ` + stringified[ 2 ][ 0 ].result + `\n` + + `\n` + + `[#4] test 3\n` + + ` bar:\n` + + ` expected: ` + stringified[ 3 ][ 0 ].expect + `\n` + + ` result: ` + stringified[ 3 ][ 0 ].result + `\n` + + ` baz:\n` + + ` expected: ` + stringified[ 3 ][ 1 ].expect + `\n` + + ` result: ` + stringified[ 3 ][ 1 ].result + `\n\n` + ); + } ); + + + it( "outputs summary on last line of stdout", () => + { + let output = ''; + + const stdout = { write: str => output += str }; + const sut = Sut( stdout, {} ); + + Sut( stdout ).done( [ + { i: 0, total: 1, failures: [] }, + { i: 1, total: 2, failures: [] }, + { i: 2, total: 3, failures: [ {} ] }, + { i: 3, total: 4, failures: [ {}, {} ] }, + { i: 4, total: 5, failures: [] }, + ] ); + + const lines = output.split( '\n' ); + + // preceded by empty line + expect( lines[ lines.length - 2 ] ).to.equal( "" ); + + // last line + expect( lines[ lines.length - 1 ] ).to.equal( + `5 tests, 2 failed (15 assertions, 3 failures)` + ); + } ); + } ); +} );