employer
/
tame
1
0
Fork 0
Code Packages Releases Activity

progtest: Initial working console runner

master
Mike Gerwitz 2018-02-16 12:19:37 -05:00
parent 0c020b736d
commit 47f0f4039b
22 changed files with 1643 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -7,8 +7,8 @@
*.info
# autotools- and configure-generated
Makefile.in
Makefile
/Makefile.in
/Makefile
/aclocal.m4
/*.cache/
/configure

3
doc/.gitignore vendored
View File

@ -17,3 +17,6 @@ version.texi
*.txt
*.html
/Makefile.in
/Makefile

2
progtest/.gitignore vendored 100644
View File

@ -0,0 +1,2 @@
/node_modules

25
progtest/Makefile 100644
View File

@ -0,0 +1,25 @@
# tame-progtest Makefile
#
# Copyright (C) 2018 R-T Specialty, LLC.
#
# This file is part of TAME.
#
# TAME is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
.PHONY: check test
test: check
check:
PATH="$(PATH):$(CURDIR)/node_modules/mocha/bin" \
mocha --harmony_destructuring --recursive test/

5
progtest/README.md 100644
View File

@ -0,0 +1,5 @@
# Program Testing
A /program/ is a top-level package (either marked as with `@program="true"`,
or with a root `rater` node). This system provides a means of writing and
running test cases.

51
progtest/bin/runner.js 100644
View File

@ -0,0 +1,51 @@
/**
* Test case runner script
*
* Copyright (C) 2018 R-T Specialty, LLC.
*
* This file is part of TAME.
*
* TAME is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
const program = require( process.argv[ 2 ] );
const filename = process.argv[ 3 ];
const fs = require( 'fs' );
const yaml_reader = require( 'js-yaml' );
const TestCase = require( '../src/TestCase' );
const YamlTestReader = require( '../src/reader/YamlTestReader' );
const ConstResolver = require( '../src/reader/ConstResolver' );
const DateResolver = require( '../src/reader/DateResolver' );
const TestRunner = require( '../src/TestRunner' );
const ConsoleTestReporter = require( '../src/reporter/ConsoleTestReporter' );
const runner = TestRunner(
ConsoleTestReporter( process.stdout ),
program
);
const reader = YamlTestReader
.use( DateResolver )
.use( ConstResolver( program ) )
( yaml_reader, TestCase );
const cases = reader.loadCases(
fs.readFileSync( filename, 'utf8' )
);
const results = runner.runTests( cases );

15
progtest/package.json 100644
View File

@ -0,0 +1,15 @@
{
"name": "tame-progtest",
"description": "TAME Program testing",
"version": "0.0.0",
"author": "R-T Specialty, LLC",
"dependencies": {
"easejs": "0.2.9",
"js-yaml": "3.10.0"
},
"devDependencies": {
},
"license": "GPL-3.0+"
}

103
progtest/src/TestCase.js 100644
View File

@ -0,0 +1,103 @@
/**
* Test case
*
* Copyright (C) 2018 R-T Specialty, LLC.
*
* This file is part of TAME.
*
* TAME is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
const { Class } = require( 'easejs' );
module.exports = Class( 'TestCase',
{
/**
* Test case data
*
* @type {Object} test case data
*/
'private _caseData': {},
get description()
{
return this._caseData.description || "";
},
get data()
{
return this._caseData.data || {};
},
get expect()
{
return this._caseData.expect || {};
},
constructor( case_data )
{
this._caseData = case_data;
},
'public mapEachValue'( callback )
{
const [ new_data, new_expect ] = [ this.data, this.expect ].map( src =>
{
const new_src = {};
Object.keys( src ).forEach(
key => new_src[ key ] = this._visitDeep(
src[ key ],
callback
)
);
return new_src;
} );
return module.exports( {
description: this.description,
data: new_data,
expect: new_expect,
} );
},
/**
* Recursively resolve constants
*
* Only scalars and arrays are supported
*
* @param {number|Array} input scalar or array of inputs
* @param {Object} consts constant mapping
*
* @return {number|Array} resolved value(s)
*/
'private _visitDeep'( val, callback )
{
if ( Array.isArray( val ) )
{
return val.map(
x => this._visitDeep( x, callback )
);
}
return callback( val );
}
} );

153
progtest/src/TestRunner.js 100644
View File

@ -0,0 +1,153 @@
/**
* Test case runner
*
* Copyright (C) 2018 R-T Specialty, LLC.
*
* This file is part of TAME.
*
* TAME is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
const { Class } = require( 'easejs' );
/**
* Run test cases and report results
*/
module.exports = Class( 'TestRunner',
{
/**
* SUT
*
* @type {Program}
*/
'private _program': null,
/**
* Test reporter
*
* @type {TestReporter}
*/
'private _reporter': null,
/**
* Initialize runner for program PROGRAM
*
* @param {TestReporter} reporter test reporter
* @param {Program} program SUT
*/
constructor( reporter, program )
{
// primitive check to guess whether this might be a program
if ( typeof program.rater !== 'function' )
{
throw TypeError( "program#rater is not a function" );
}
this._reporter = reporter;
this._program = program;
},
/**
* Run set of test cases
*
* @param {Array<TestCase>} dfns array of TestCases
*
* @return {Array<Object<desc,i,total,failures>>} results
*/
'public runTests'( dfns )
{
const total = dfns.length;
this._reporter.preRun( total );
const results = dfns.map(
( test, i ) => this._runTest( test, i, total )
);
this._reporter.done( results );
return results;
},
/**
* Run individual test case
*
* @param {Object<TestCase>} _ source test case
* @param {number} test_i test index
* @param {number} total total number of tests
*
* @return {Object<desc,i,total,failures>} test results
*/
'private _runTest'( { description: desc, data, expect }, test_i, total )
{
// no input map---#rate uses params directly
const result = this._program.rater( data ).vars;
const cmp = Object.keys( expect ).map(
field => [
field,
this._deepCompare( expect[ field ], result[ field ] )
]
);
const failures = cmp.filter( ( [ , ok ] ) => !ok )
.map( ( [ field ] ) => ( {
field: field,
expect: expect[ field ],
result: result[ field ],
} ) );
const succeeded = cmp.length - failures.length;
const result_data = {
desc: desc,
i: test_i,
total: cmp.length,
failures: failures,
};
this._reporter.testCaseResult( result_data, total );
return result_data;
},
/**
* Recursively compare values (scalar, array)
*
* @param {number|Array<number>} x first value
* @param {number|Array<number>} y second value
*
* @return {boolean} whether X deeply equals Y
*/
'private _deepCompare'( x, y )
{
// vector/matrix/etc
if ( Array.isArray( x ) )
{
return Array.isArray( y )
&& ( x.length === y.length )
&& x.every( ( xval, i ) => xval === y[ i ] );
}
// scalar
return x === y;
},
} );

110
progtest/src/reader/ConstResolver.js 100644
View File

@ -0,0 +1,110 @@
/**
* Constant resolver for test case reader
*
* Copyright (C) 2018 R-T Specialty, LLC.
*
* This file is part of TAME.
*
* TAME is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
const { Trait } = require( 'easejs' );
const TestReader = require( './TestReader' );
/**
* Resolve program constants by replacing them with their numeric value
*
* The result is a loaded YAML file that contains only numeric input. All
* non-numeric input is interpreted as a constant.
*
* Any non-numeric value is considered to be a constant, so it is important
* to perform all other data transformations before applying constant
* resolution.
*/
module.exports = Trait( 'ConstResolver' )
.implement( TestReader )
.extend(
{
/**
* Program from which to load constants
*
* @type {Program}
*/
'private _program': null,
/**
* Initialize with program from which to load constants
*
* @param {Program} program source program
*/
__mixin( program )
{
this._program = program;
},
/**
* Load test cases and resolve constants
*
* @param {*} src data source
*
* @return {Array<TestCase>} array of test cases
*/
'abstract override public loadCases'( yaml )
{
const data = this.__super( yaml );
const { consts } = this._program.rater;
return data.map(
testcase => testcase.mapEachValue(
value => this._resolve( value, consts )
)
);
},
/**
* Resolve constant
*
* If the constant is not known (via CONSTS), an error is thrown.
*
* @param {number|Array} input scalar or array of inputs
* @param {Object} consts constant mapping
*
* @throws {Error} if constant is unknown in CONSTS
*
* @return {number|Array} resolved value(s)
*/
'private _resolve'( input, consts )
{
// already a number, return as-is
if ( !isNaN( +input ) )
{
return input;
}
const result = consts[ input ];
if ( result === undefined )
{
throw Error( `unknown constant: ${input}` );
}
return result;
}
} );

61
progtest/src/reader/DateResolver.js 100644
View File

@ -0,0 +1,61 @@
/**
* Date resolver for test case reader
*
* Copyright (C) 2018 R-T Specialty, LLC.
*
* This file is part of TAME.
*
* TAME is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
const { Trait } = require( 'easejs' );
const TestReader = require( './TestReader' );
/**
* Resolve dates of the format MM/DD/YYYY to Unix timestamps
*
* This allows easily readable dates to be included in test cases without
* having to worry about Unix timestamps. For higher precision, Unix
* timestamps must be used.
*/
module.exports = Trait( 'DateResolver' )
.implement( TestReader )
.extend(
{
/**
* Load test cases and resolve dates
*
* Dates will be replaced with Unix timestamps.
*
* @param {*} src data source
*
* @return {Array<TestCase>} array of test cases
*/
'abstract override public loadCases'( src )
{
const data = this.__super( src );
return data.map(
testcase => testcase.mapEachValue(
value => ( /^\d{2}\/\d{2}\/\d{4}$/.test( value ) )
? ( ( new Date( value ) ).getTime() / 1000 )
: value
)
);
}
} );

40
progtest/src/reader/TestReader.js 100644
View File

@ -0,0 +1,40 @@
/**
* Test case reader
*
* Copyright (C) 2018 R-T Specialty, LLC.
*
* This file is part of TAME.
*
* TAME is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
const { Interface } = require( 'easejs' );
module.exports = Interface( 'TestReader',
{
/**
* Load test cases from an implementation-defined data source SRC
*
* The produced object will be an array of cases, each containing a
* `description`, `data`, and `expect`.
*
* @param {*} src data source
*
* @return {Array<TestCase>} array of test cases
*/
'public loadCases': [ 'src' ],
} );

79
progtest/src/reader/YamlTestReader.js 100644
View File

@ -0,0 +1,79 @@
/**
* YAML test case reader
*
* Copyright (C) 2018 R-T Specialty, LLC.
*
* This file is part of TAME.
*
* TAME is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
const { Class } = require( 'easejs' );
const TestReader = require( './TestReader' );
module.exports = Class( 'YamlTestReader' )
.implement( TestReader )
.extend(
{
/**
* YAML parser
*
* @type {Object}
*/
'private _yamlParser': null,
/**
* TestCase constructor
*
* @type {function(Object)}
*/
'private _createTestCase': null,
/**
* Initialize with YAML parser
*
* The parser must conform to the API of `js-yaml`.
*
* @param {Object} yaml_parser YAML parser
* @param {function(Object)} test_case_ctor TestCase constructor
*/
constructor( yaml_parser, test_case_ctor )
{
this._yamlParser = yaml_parser;
this._createTestCase = test_case_ctor;
},
/**
* Load test cases from a YAML string
*
* The produced object will be an array of cases, each containing a
* `description`, `data`, and `expect`.
*
* @param {string} src source YAML
*
* @return {Array<TestCase>} array of test cases
*/
'virtual public loadCases'( yaml )
{
const data = this._yamlParser.safeLoad( yaml )
.map( this._createTestCase );
return data;
},
} );

183
progtest/src/reporter/ConsoleTestReporter.js 100644
View File

@ -0,0 +1,183 @@
/**
* Console test reporter
*
* Copyright (C) 2018 R-T Specialty, LLC.
*
* This file is part of TAME.
*
* TAME is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
const { Class } = require( 'easejs' );
/**
* Real-time reporting of test cases to the console
*
* Test cases will be output in a block of dots (success) or 'F's (failure),
* in a style similar to PHPUnit. If failures occur, they will be output to
* in more detail after all tests have run.
*/
module.exports = Class( 'ConsoleTestReporter',
{
/**
* Standard out
*
* @type {Object} standard out
*/
'private _stdout': null,
/**
* Initialize reporter with target console
*
* STDOUT must follow Node.js' API.
*
* @param {Object} stdout standard out
*/
constructor( stdout )
{
this._stdout = stdout;
},
/**
* Invoked before tests are run
*
* The only information provided here is the number of test cases to be
* run, which can be used to produce a progress indicator.
*
* @param {number} total number of test cases
*
* @return {undefined}
*/
'public preRun'( total )
{
// this reporter does nothing with this method
},
/**
* Invoked for each test case immediately after it has been run
*
* For the format of RESULT, see TestRunner.
*
* @param {Object} result test case result
*
* @return {undefined}
*/
'public testCaseResult'( result, total )
{
const { i, failures } = result;
const ind = ( failures.length === 0 )
? '.'
: 'F';
const sep = ( i % 50 === 49 )
? ` ${i+1}/${total}\n`
: '';
this._stdout.write( ind + sep );
},
/**
* Invoked after all test cases have been run
*
* RESULTS is an array containing each result that was previously
* reported to `#testCaseResult`.
*
* A final line will be output, preceded by an empty line, summarizing
* the number of tests, assertions, and failures for each.
*
* @param {Array<Object>} results all test results
*
* @return {undefined}
*/
'public done'( results )
{
this._outputFailureReport( results );
this._outputSummary( results );
},
/**
* For each failure, output each expected and resulting value
*
* Failures are prefixed with a 1-indexed number.
*
* @param {Object} result test case result}
*
* @return {undefined}
*/
'private _outputFailureReport'( results )
{
const report = results
.filter( ( { failures } ) => failures.length > 0 )
.map( this._reportTestFailure.bind( this ) )
.join( '\n' )
this._stdout.write( "\n\n" + report );
},
/**
* Generate report for test case failure
*
* @param {Object<i,desc,failures>} _ test case result data
*
* @return {string} report
*/
'private _reportTestFailure'( { i, desc, failures } )
{
return `[#${i+1}] ${desc}\n` +
failures.map( ( { field, expect, result } ) =>
` ${field}:\n` +
` expected: ` + JSON.stringify( expect ) + `\n` +
` result: ` + JSON.stringify( result ) + `\n`
).join( '' );
},
/**
* Output a line, preceded by an empty line, summarizing the number of
* tests, assertions, and failures for each
*
* @param {Array<Object>} results all test results
*
* @return {undefined}
*/
'private _outputSummary'( results )
{
const [ failed, afailed, acount ] = results.reduce(
( [ failed, afailed, acount ], { failures, total } ) =>
[
( failed + +( failures.length > 0 ) ),
( afailed + failures.length ),
( acount + total )
],
[ 0, 0, 0 ]
);
const test_total = results.length;
this._stdout.write(
`\n${test_total} tests, ${failed} failed (` +
`${acount} assertions, ${afailed} failures)`
);
},
} );

79
progtest/src/reporter/NullTestReporter.js 100644
View File

@ -0,0 +1,79 @@
/**
* Reporter that does nothing
*
* Copyright (C) 2018 R-T Specialty, LLC.
*
* This file is part of TAME.
*
* TAME is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
const { Class } = require( 'easejs' );
/**
* Reporter that does nothing
*
* This is useful if you want a fully background process.
*/
module.exports = Class( 'NullTestReporter',
{
/**
* Invoked before tests are run
*
* The only information provided here is the number of test cases to be
* run, which can be used to produce a progress indicator.
*
* @param {number} total number of test cases
*
* @return {undefined}
*/
'public preRun'( total )
{
// this reporter does nothing with this method
},
/**
* Invoked for each test case immediately after it has been run
*
* For the format of RESULT, see TestRunner.
*
* @param {Object} result test case result
*
* @return {undefined}
*/
'public testCaseResult'( result, total )
{
// this reporter does nothing with this method
},
/**
* Invoked after all test cases have been run
*
* RESULTS is an array containing each result that was previously
* reported to `#testCaseResult`.
*
* @param {Array<Object>} results all test results
*
* @return {undefined}
*/
'public done'( results )
{
// this reporter does nothing with this method
},
} );

68
progtest/src/reporter/TestReporter.js 100644
View File

@ -0,0 +1,68 @@
/**
* Test reporter
*
* Copyright (C) 2018 R-T Specialty, LLC.
*
* This file is part of TAME.
*
* TAME is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
const { Interface } = require( 'easejs' );
/**
* Real-time reporting of test cases
*/
module.exports = Interface( 'TestReporter',
{
/**
* Invoked before tests are run
*
* The only information provided here is the number of test cases to be
* run, which can be used to produce a progress indicator.
*
* @param {number} total number of test cases
*
* @return {undefined}
*/
'public preRun': [ 'total' ],
/**
* Invoked for each test case immediately after it has been run
*
* For the format of RESULT, see TestRunner.
*
* @param {Object} result test case result
*
* @return {undefined}
*/
'public testCaseResult': [ 'result' ],
/**
* Invoked after all test cases have been run
*
* RESULTS is an array containing each result that was previously
* reported to `#testCaseResult`.
*
* @param {Array<Object>} results all test results
*
* @return {undefined}
*/
'public done': [ 'results' ],
} );

99
progtest/test/TestCaseTest.js 100644
View File

@ -0,0 +1,99 @@
/**
* Tests TestCase
*
* Copyright (C) 2018 R-T Specialty, LLC.
*
* This file is part of TAME.
*
* TAME is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
const { expect } = require( 'chai' );
const Sut = require( '../src/TestCase' );
describe( "TestCase", () =>
{
it( "allows retrieving raw data", () =>
{
const data = {
description: "Foo bar",
data: { foo: [ 5 ] },
expect: { bar: [ 1 ] },
};
const sut = Sut( data );
expect( sut.description ).to.equal( data.description );
expect( sut.data ).to.deep.equal( data.data );
expect( sut.expect ).to.deep.equal( data.expect );
} );
it( "provides sane defaults for missing data", () =>
{
const sut = Sut( {} );
expect( sut.description ).to.equal( "" );
expect( sut.data ).to.deep.equal( {} );
expect( sut.expect ).to.deep.equal( {} );
} );
describe( "#mapEachValue", () =>
{
it( "visits each 'data' and 'expect' value", () =>
{
// tests scalar, vector, matrix; mixed with non-constants
const testcase = {
description: 'test desc',
data: {
foo: 'bar',
bar: [ 'baz', 'quux' ],
baz: [ [ 'quuux', 'foox' ], [ 'moo', 'cow' ] ],
},
expect: {
quux: 'out',
quuux: [ 'of', 'names' ],
},
};
const expected = {
data: {
foo: 'OKbar',
bar: [ 'OKbaz', 'OKquux' ],
baz: [ [ 'OKquuux', 'OKfoox' ], [ 'OKmoo', 'OKcow' ] ],
},
expect: {
quux: 'OKout',
quuux: [ 'OKof', 'OKnames' ],
},
};
const result = Sut( testcase ).mapEachValue( val => `OK${val}` );
// derived from the original
expect( result.description ).to.equal( testcase.description );
expect( result.data ).to.deep.equal( expected.data );
expect( result.expect ).to.deep.equal( expected.expect );
// but not the original (should return a new object)
expect( result.data ).to.not.equal( testcase.data );
expect( result.expect ).to.not.equal( testcase.expect );
} );
} );
} );

144
progtest/test/TestRunnerTest.js 100644
View File

@ -0,0 +1,144 @@
/**
* Tests TestReader
*
* Copyright (C) 2018 R-T Specialty, LLC.
*
* This file is part of TAME.
*
* TAME is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
const { expect } = require( 'chai' );
const { Class } = require( 'easejs' );
const Sut = require( '../src/TestRunner' );
const TestReporter = require( '../src/reporter/TestReporter' );
const NullTestReporter = require( '../src/reporter/NullTestReporter' );
describe( "TestRunner", () =>
{
it( "runs each test against given program", () =>
{
const given = [];
const program = {
rater( data )
{
return rate_results[ given.push( data ) - 1 ];
}
};
const test_cases = [
{
description: "first",
data: { a: 1 },
expect: { foo: 1 },
},
{
description: "second",
data: { a: 2 },
expect: {
foo: [ 1, 2 ],
bar: [ 3, 1 ],
baz: [ 4, 2 ],
},
},
];
const rate_results = [
// no failures
{ vars: { foo: 1 } },
// bar, baz failures
{ vars: {
foo: [ 1, 2 ],
bar: [ 3, 4 ],
baz: [ 4, 5 ],
} },
];
const expect_failures = [
[],
[
{
field: 'bar',
expect: test_cases[ 1 ].expect.bar,
result: rate_results[ 1 ].vars.bar,
},
{
field: 'baz',
expect: test_cases[ 1 ].expect.baz,
result: rate_results[ 1 ].vars.baz,
},
]
];
const results = Sut( NullTestReporter(), program )
.runTests( test_cases );
test_cases.forEach( ( test_case, i ) =>
{
const result = results[ i ];
expect( result.desc ).to.equal( test_case.description );
expect( result.i ).to.equal( i );
expect( result.total ).to.equal(
Object.keys( test_case.expect ).length
);
expect( result.failures ).to.deep.equal( expect_failures[ i ] );
} );
} );
it( "invokes reporter before, during, and after test cases", done =>
{
let pre = false;
let results = [];
const program = { rater: () => ( { vars: {} } ) };
const mock_reporter = Class.implement( TestReporter ).extend(
{
preRun( total )
{
expect( total ).to.equal( 2 );
pre = true;
},
testCaseResult( result, total )
{
expect( pre ).to.equal( true );
expect( total ).to.equal( 2 );
results.push( result );
},
done( given_results )
{
expect( pre ).to.equal( true );
expect( results ).to.deep.equal( given_results );
done();
},
} )();
// see done() above
Sut( mock_reporter, program ).runTests( [
{ description: '', data: {}, expect: {} },
{ description: '', data: {}, expect: {} },
] );
} );
} );

104
progtest/test/reader/ConstResolverTest.js 100644
View File

@ -0,0 +1,104 @@
/**
* Tests ConstResolver
*
* Copyright (C) 2018 R-T Specialty, LLC.
*
* This file is part of TAME.
*
* TAME is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
const { expect } = require( 'chai' );
const { Class } = require( 'easejs' );
const TestCase = require( '../../src/TestCase' );
const TestReader = require( '../../src/reader/TestReader' );
const Sut = require( '../../src/reader/ConstResolver' );
const StubTestReader = Class.implement( TestReader ).extend(
{
constructor( parsed_data )
{
this.parsedData = parsed_data;
},
'virtual public loadCases'( _ )
{
return this.parsedData;
}
} );
describe( "ConstResolver", () =>
{
[ 'data', 'expect' ].forEach( field =>
{
it( `replaces known ${field} constants from program`, () =>
{
const program = {
rater: {
consts: { FOO: 1, BAR: 2 },
},
};
// tests scalar, vector, matrix; mixed with non-constants
const parsed_data = [
TestCase( { [field]: { foo: 'FOO', bar: 4 } } ),
TestCase(
{ [field]: {
foo: [ 'FOO', 'BAR', 5 ],
bar: [ [ 'FOO', 3 ], [ 'FOO', 'BAR' ] ],
} }
),
];
const { FOO, BAR } = program.rater.consts;
const expected = [
TestCase( { [field]: { foo: FOO, bar: 4 } } ),
TestCase(
{ [field]: {
foo: [ FOO, BAR, 5 ],
bar: [ [ FOO, 3 ], [ FOO, BAR ] ],
} }
),
];
// anything just to proxy
const given_yaml = 'fooml';
const result = StubTestReader
.use( Sut( program ) )( parsed_data )
.loadCases( given_yaml );
result.forEach(
( tcase, i ) => expect( tcase[ field ] )
.to.deep.equal( expected[ i ][ field ] )
);
} );
it( `throws error on unknown $field constant`, () =>
{
const program = { rater: { consts: {} } };
const parsed_data = [ TestCase( { [field]: { foo: 'UNKNOWN' } } ) ];
expect(
() => StubTestReader.use( Sut( program ) )( parsed_data )
.loadCases( '' )
).to.throw( Error, 'UNKNOWN' );
} );
} );
} );

85
progtest/test/reader/DateResolverTest.js 100644
View File

@ -0,0 +1,85 @@
/**
* Tests DateResolver
*
* Copyright (C) 2018 R-T Specialty, LLC.
*
* This file is part of TAME.
*
* TAME is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
const { expect } = require( 'chai' );
const { Class } = require( 'easejs' );
const TestCase = require( '../../src/TestCase' );
const TestReader = require( '../../src/reader/TestReader' );
const Sut = require( '../../src/reader/DateResolver' );
const MockTestReader = Class.implement( TestReader ).extend(
{
constructor( parsed_data, expected_load )
{
this.parsedData = parsed_data;
this.expectedLoad = expected_load;
},
'virtual public loadCases'( given )
{
expect( given ).to.equal( this.expectedLoad );
return this.parsedData;
}
} );
describe( "DateResolver", () =>
{
[ 'data', 'expect' ].forEach( field =>
{
it( `converts ${field} dates into Unix timestamps`, () =>
{
const date = '10/25/1989';
const time = ( new Date( date ) ).getTime() / 1000;
// tests scalar, vector, matrix; mixed with non-constants
const parsed_data = [
TestCase(
{ [field]: {
foo: [ 5, 'NOTADATE', date ],
} }
),
];
const expected = [
TestCase(
{ [field]: {
foo: [ 5, 'NOTADATE', time ],
} }
),
];
// anything just to proxy
const given_yaml = 'fooml';
const result = MockTestReader
.use( Sut )( parsed_data, given_yaml )
.loadCases( given_yaml );
result.forEach(
( tcase, i ) => expect( tcase[ field ] )
.to.deep.equal( expected[ i ][ field ] )
);
} );
} );
} );

55
progtest/test/reader/YamlTestReaderTest.js 100644
View File

@ -0,0 +1,55 @@
/**
* Tests TestReader
*
* Copyright (C) 2018 R-T Specialty, LLC.
*
* This file is part of TAME.
*
* TAME is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
const { expect } = require( 'chai' );
const Sut = require( '../../src/reader/YamlTestReader' );
describe( "YamlTestReader", () =>
{
it( "parses given yaml", () =>
{
const yaml = "foo: bar";
const parsed = [
{
description: "first desc",
data: { "foo": "bar" },
expect: { "bar": "baz" },
},
];
const case_ctor = ( data ) => ( { ok: data } );
const mock_parser = {
safeLoad( given )
{
expect( given ).to.equal( yaml );
return parsed;
}
};
expect( Sut( mock_parser, case_ctor ).loadCases( yaml ) )
.to.deep.equal( [ { ok: parsed[0] } ] );
} );
} );

177
progtest/test/reporter/ConsoleTestReporterTest.js 100644
View File

@ -0,0 +1,177 @@
/**
* Tests ConsoleTestReporter
*
* Copyright (C) 2018 R-T Specialty, LLC.
*
* This file is part of TAME.
*
* TAME is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
const { expect } = require( 'chai' );
const Sut = require( '../../src/reporter/ConsoleTestReporter' );
describe( "ConsoleTestReporter", () =>
{
describe( "#testCaseResult", () =>
{
it( "outputs indicator for each test case", () =>
{
let output = '';
const stdout = { write: str => output += str };
const sut = Sut( stdout );
[
{ i: 0, failures: [] },
{ i: 1, failures: [] },
{ i: 2, failures: [ {} ] },
{ i: 3, failures: [ {}, {} ] },
{ i: 4, failures: [] },
].forEach(
result => sut.testCaseResult( result, 5 )
);
expect( output ).to.equal( '..FF.' );
} );
it( "outputs line break with count after 40 cases", () =>
{
let output = '';
const stdout = { write: str => output += str };
const sut = Sut( stdout );
const results = ( new Array( 130 ) ).join( '.' ).split( '.' )
.map( ( _, i ) => ( { i: i, failures: [] } ) );
results.forEach(
result => sut.testCaseResult( result, 130 )
);
expect( output ).to.equal(
( new Array( 51 ) ).join( '.' ) + ' 50/130\n' +
( new Array( 51 ) ).join( '.' ) + ' 100/130\n' +
( new Array( 31 ) ).join( '.' )
);
} );
} );
describe( "done", () =>
{
it( "outputs report of failures to stdout", () =>
{
let output = '';
const stdout = { write: str => output += str };
const results = [
{ i: 0, total: 1, desc: "test 0", failures: [] },
{ i: 1, total: 2, desc: "test 1", failures: [] },
{
i: 2,
total: 3,
desc: "test 2",
failures: [
{
field: "foo",
expect: [ 1 ],
result: [ 2, 3 ]
},
],
},
{
i: 3,
total: 4,
desc: "test 3",
failures: [
{
field: "bar",
expect: 2,
result: 3,
},
{
field: "baz",
expect: [ [ 4 ] ],
result: [ 5 ],
}
],
},
];
const stringified = results.map(
result => result.failures.map(
failure => ( {
expect: JSON.stringify( failure.expect ),
result: JSON.stringify( failure.result ),
} )
)
);
Sut( stdout ).done( results );
const fail_output = output.match( /\n\n\[#3\](.|\n)*\n\n/ )[0];
// 1-indexed output
expect( fail_output ).to.equal(
`\n\n` +
`[#3] test 2\n` +
` foo:\n` +
` expected: ` + stringified[ 2 ][ 0 ].expect + `\n` +
` result: ` + stringified[ 2 ][ 0 ].result + `\n` +
`\n` +
`[#4] test 3\n` +
` bar:\n` +
` expected: ` + stringified[ 3 ][ 0 ].expect + `\n` +
` result: ` + stringified[ 3 ][ 0 ].result + `\n` +
` baz:\n` +
` expected: ` + stringified[ 3 ][ 1 ].expect + `\n` +
` result: ` + stringified[ 3 ][ 1 ].result + `\n\n`
);
} );
it( "outputs summary on last line of stdout", () =>
{
let output = '';
const stdout = { write: str => output += str };
const sut = Sut( stdout, {} );
Sut( stdout ).done( [
{ i: 0, total: 1, failures: [] },
{ i: 1, total: 2, failures: [] },
{ i: 2, total: 3, failures: [ {} ] },
{ i: 3, total: 4, failures: [ {}, {} ] },
{ i: 4, total: 5, failures: [] },
] );
const lines = output.split( '\n' );
// preceded by empty line
expect( lines[ lines.length - 2 ] ).to.equal( "" );
// last line
expect( lines[ lines.length - 1 ] ).to.equal(
`5 tests, 2 failed (15 assertions, 3 failures)`
);
} );
} );
} );