From a3e359a050027092961ba6e8603da25e07bf7c84 Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Tue, 13 Jun 2017 10:21:28 -0400 Subject: [PATCH 01/17] Add node dapi HTTP implementation * src/dapi/http/HttpError.js: Add error subtype. * src/dapi/http/NodeHttpImpl.js: Add node-based HTTP impl. * test/dapi/http/HttpErrorTest.js: Add test. * test/dapi/http/NodeHttpImplTest.js: Add test. --- src/dapi/http/HttpError.js | 56 ++++ src/dapi/http/NodeHttpImpl.js | 217 +++++++++++++++ test/dapi/http/HttpErrorTest.js | 47 ++++ test/dapi/http/NodeHttpImplTest.js | 413 +++++++++++++++++++++++++++++ 4 files changed, 733 insertions(+) create mode 100644 src/dapi/http/HttpError.js create mode 100644 src/dapi/http/NodeHttpImpl.js create mode 100644 test/dapi/http/HttpErrorTest.js create mode 100644 test/dapi/http/NodeHttpImplTest.js diff --git a/src/dapi/http/HttpError.js b/src/dapi/http/HttpError.js new file mode 100644 index 0000000..9326cbb --- /dev/null +++ b/src/dapi/http/HttpError.js @@ -0,0 +1,56 @@ +/** + * Error representing non-200 HTTP status code + * + * Copyright (C) 2017 R-T Specialty, LLC. + * + * This file is part of the Liza Data Collection Framework. + * + * liza is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +'use strict'; + +const { Class } = require( 'easejs' ); + + +/** + * Represents error in performing HTTP request + */ +module.exports = Class( 'HttpError' ) + .extend( Error, +{ + /** + * HTTP status code + * @type {number} + */ + 'public statuscode': 500, + + + /** + * Set error message and HTTP status code + * + * The HTTP status code defaults to 500 if not set. No check is + * performed to determine whether the given status code is a valid error + * code. + * + * The mesage is _not_ automatically set from the status code. + * + * @param {string} message error message + * @param {number=} statuscode HTTP status code + */ + __construct( message, statuscode ) + { + this.statuscode = statuscode || 500; + }, +} ); diff --git a/src/dapi/http/NodeHttpImpl.js b/src/dapi/http/NodeHttpImpl.js new file mode 100644 index 0000000..495e2e1 --- /dev/null +++ b/src/dapi/http/NodeHttpImpl.js @@ -0,0 +1,217 @@ +/** + * HTTP over Node.js-compatible API + * + * Copyright (C) 2017 R-T Specialty, LLC. + * + * This file is part of the Liza Data Collection Framework. + * + * liza is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +'use strict'; + +const { Class } = require( 'easejs' ); +const HttpImpl = require( './HttpImpl' ); +const HttpError = require( './HttpError' ); + + +/** + * HTTP adapter using Node.js-compatible objects (e.g. its `http` modules) + */ +module.exports = Class( 'NodeHttpImpl' ) + .implement( HttpImpl ) + .extend( +{ + /** + * Clients for desired protocols (e.g. HTTP(s)) + * @type {Object} + */ + 'private _protoHandlers': {}, + + /** + * URL parser + * @type {url} + */ + 'private _urlParser': '', + + + /** + * Initialize with protocol handlers and URL parser + * + * `proto_handlers` must be a key-value mapping of the protocol string + * to a handler object conforming to Node's http(s) APIs---that is, it + * should provide a `#request` method. + * + * @param {Object} proto_handlers protocol handler key-value map + * @param {Object} url_parser URL parser + */ + constructor( proto_handlers, url_parser ) + { + this._protoHandlers = proto_handlers; + this._urlParser = url_parser; + }, + + + /** + * Perform HTTP request + * + * If the request is synchronous, it must still return the data via the + * provided callback. The provided data is expected to be key-value if an + * object is given, otherwise a string of binary data. + * + * @param {string} url destination URL + * @param {string} method RFC-2616-compliant HTTP method + * @param {Object|string} data request params + * @param {function(?Error, ?string)} callback server response callback + * + * @return {HttpImpl} self + */ + 'public requestData'( url, method, data, callback ) + { + const options = this._urlParser.parse( url ); + const protocol = options.protocol.replace( /:$/, '' ); + const handler = this._protoHandlers[ protocol ]; + + if ( !handler ) + { + throw Error( `No handler for ${protocol}` ); + } + + this._setOptions( options, method, data ); + + let forbid_end = false; + + const req = handler.request( options, res => + { + let data = ''; + + res.on( 'data', chunk => data += chunk ); + res.on( 'end', () => + !forbid_end && this.requestEnd( res, data, callback ) + ); + } ); + + req.on( 'error', e => + { + this.serveError( e, null, null, callback ); + + // guarantee that the callback will not be invoked a second time + // if something tries to end the request + forbid_end = true; + } ); + + if ( method === 'POST' ) + { + req.write( data ); + } + + req.end(); + }, + + + /** + * Set request options + * + * @param {Object} options request options + * @param {string} method HTTP method + * @param {string} data request data + * + * @return {Object} request headers + */ + 'private _setOptions'( options, method, data ) + { + const { headers = {} } = options; + + options.method = method; + + if ( method === 'POST' ) + { + headers[ 'Content-Type' ] = 'application/x-www-form-urlencoded'; + + options.headers = headers; + } + else + { + if ( data ) + { + options.path += '?' + data; + } + } + }, + + + + /** + * Invoked when a request is completed + * + * Subtypes may override this method to handle their own request + * processing before the continuation `callback` is invoked with the + * final data. + * + * To override only error situations, see `#serveError`. + * + * @param {Object} res Node http.ServerResponse + * @param {string} data raw response data + * @param {function(?Error,?string)} callback completion continuation + * + * @return {undefined} + */ + 'virtual protected requestEnd'( res, data, callback ) + { + if ( !this.isSuccessful( res ) ) + { + this.serveError( + HttpError( res.statusMessage, res.statusCode ), + res, + data, + callback + ); + + return; + } + + callback( null, data ); + }, + + + /** + * Predicate to determine whether HTTP request was successful + * + * Non-2xx status codes represent failures. + * + * @param {Object} res Node http.ServerResponse + * + * @return {boolean} whether HTTP status code represents a success + */ + 'virtual protected isSuccessful'( res ) + { + return ( +res.statusCode >= 200 ) && ( +res.statusCode < 300 ); + }, + + + /** + * Invoke continuation `callback` with an error `e` + * + * @param {Error} e error + * @param {Object} res Node http.ServerResponse + * @param {string} data raw response data + * @param {function(?Error,?data)} callback continuation + * + * @return {undefined} + */ + 'virtual protected serveError'( e, res, data, callback ) + { + callback( e, data ); + }, +} ); diff --git a/test/dapi/http/HttpErrorTest.js b/test/dapi/http/HttpErrorTest.js new file mode 100644 index 0000000..4c3e321 --- /dev/null +++ b/test/dapi/http/HttpErrorTest.js @@ -0,0 +1,47 @@ +/** + * Tests error representing non-200 HTTP status code + * + * Copyright (C) 2017 R-T Specialty, LLC. + * + * This file is part of the Liza Data Collection Framework. + * + * liza is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +const { expect } = require( 'chai' ); +const Sut = require( '../../../' ).dapi.http.HttpError; + +'use strict'; + + +describe( "HttpError", () => +{ + it( "provides HTTP status code", () => + { + const code = 418; + + expect( Sut( 'message', code ).statuscode ) + .to.equal( code ); + } ); + + + // just make sure overriding ctor calls parent + it( "sets message", () => + { + const message = 'foobar'; + + expect( Sut( message ).message ) + .to.equal( message ); + } ); +} ); diff --git a/test/dapi/http/NodeHttpImplTest.js b/test/dapi/http/NodeHttpImplTest.js new file mode 100644 index 0000000..96a4195 --- /dev/null +++ b/test/dapi/http/NodeHttpImplTest.js @@ -0,0 +1,413 @@ +/** + * Test HTTP using Node.js-compatible API + * + * Copyright (C) 2017 R-T Specialty, LLC. + * + * This file is part of the Liza Data Collection Framework. + * + * liza is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +'use strict'; + +const { expect } = require( 'chai' ); +const { Class } = require( 'easejs' ); + +const { + HttpImpl, + NodeHttpImpl: Sut, + HttpError, +} = require( '../../../' ).dapi.http; + + +describe( "NodeHttpImpl", () => +{ + it( 'is an HttpImpl', function() + { + var sut = Sut( function() {} ); + expect( Class.isA( HttpImpl, sut ) ).to.be.ok; + } ); + + + [ + { + label: "uses http for plain HTTP requests", + protocol: 'http:', + method: 'GET', + }, + { + label: "uses http for plain HTTP requests", + protocol: 'https:', + method: 'GET', + } + ].forEach( ( { label, protocol, method } ) => + { + it( label, done => + { + const url_result = { + protocol: protocol, + hostname: 'host', + port: 8888, + path: 'foo', + }; + + const url = _createMockUrl( given_url => url_result ); + + const data = {}; + const callback_expected = {}; + const callback = () => callback_expected; + + const check = proto => ( opts, given_callback ) => + { + expect( opts.protocol ).to.equal( proto ); + expect( opts.hostname ).to.equal( url_result.hostname ); + expect( opts.port ).to.equal( url_result.port ); + expect( opts.path ).to.equal( url_result.path ); + expect( opts.method ).to.equal( method ); + + given_callback( _createMockResp() ); + + done(); + }; + + const http = _createMockHttp( check( 'http:' ) ); + const https = _createMockHttp( check( 'https:' ) ); + + Sut( { http: http, https: https }, url ) + .requestData( '', method, data, callback ); + } ); + } ); + + + it( "returns response when no error", done => + { + const res = _createMockResp(); + const chunks = [ 'a', 'b', 'c', 'd' ]; + + const http = _createMockHttp( ( _, callback ) => + { + callback( res ); + + chunks.forEach( chunk => res.trigger( 'data', chunk ) ); + res.trigger( 'end' ); + } ); + + Sut( { http: http }, _createMockUrl() ) + .requestData( "", 'GET', '', ( e, data ) => + { + expect( e ).to.equal( null ); + expect( data ).to.equal( chunks.join( '' ) ); + + done(); + } ); + } ); + + + it( "adds data to query string on GET", done => + { + const given_path = '/path'; + const expected_query = 'write data'; + + const res = _createMockResp(); + const url = _createMockUrl( given_url => ( { + protocol: 'http:', + path: given_path, + } ) ); + + const http = _createMockHttp( ( options, callback ) => + { + expect( options.path ) + .to.equal( given_path + '?' + expected_query ); + + callback( res ); + res.trigger( 'end' ); + } ); + + Sut( { http: http }, url ) + .requestData( "", 'GET', expected_query, done ); + } ); + + + it( "writes form data on POST", done => + { + const expected_data = 'expected'; + const expected_write = 'write data'; + + const res = _createMockResp(); + + const http = _createMockHttp( ( options, callback ) => + { + expect( http.req.written ).to.equal( expected_write ); + + expect( options.headers[ 'Content-Type' ] ) + .to.equal( 'application/x-www-form-urlencoded' ); + + callback( res ); + + // make sure we're still handling the response as well + res.trigger( 'data', expected_data ); + res.trigger( 'end' ); + } ); + + Sut( { http: http }, _createMockUrl() ) + .requestData( "", 'POST', expected_write, ( e, data ) => + { + expect( e ).to.equal( null ); + expect( data ).to.equal( expected_data ); + + done(); + } ); + } ); + + + it( "returns error and response given non-200 status code", done => + { + const res = _createMockResp(); + const http = _createMockHttp( ( _, callback ) => + { + callback( res ) + + res.statusCode = 418; + res.statusMessage = "I'm a teapot"; + + res.trigger( 'end' ); + } ); + + Sut( { http: http }, _createMockUrl() ) + .requestData( "", 'GET', '', ( e, data ) => + { + expect( e ).to.be.instanceOf( HttpError ); + expect( e.message ).to.equal( res.statusMessage ); + expect( e.statuscode ).to.equal( res.statusCode ); + + done(); + } ); + } ); + + + describe( "given a request error", () => + { + it( "returns error with no response on request error", done => + { + const error = Error( 'test error' ); + const http = _createMockHttp( () => {} ); + + Sut( { http: http }, _createMockUrl() ) + .requestData( "", 'GET', '', ( e, data ) => + { + expect( data ).to.equal( null ); + expect( e ).to.equal( error ); + + done(); + } ); + + // request will be hanging at this point since we didn't call + // the callback, so we can fail the request + http.req.trigger( 'error', error ); + } ); + + // this should never happen in practice, but we want to defend + // against it to make sure the callback is not invoked twice + it( "will not complete request on end", () => + { + let res = _createMockResp(); + + const http = _createMockHttp( ( _, callback ) => + { + // allow hooking `end' + callback( res ); + } ); + + Sut( { http: http }, _createMockUrl() ) + .requestData( "", 'GET', '', ( e, data ) => + { + // will fail on successful callback + expect( data ).to.equal( null ); + } ); + + http.req.trigger( 'error', Error() ); + + // do not invoke a second time (should do nothing) + res.trigger( 'end' ); + } ); + } ); + + + describe( "protected API", () => + { + it( "allows overriding request end behavior", done => + { + const expected_data = "expected"; + const e = Error( "test e" ); + const value = "resp data"; + const res = _createMockResp(); + + const http = _createMockHttp( ( _, callback ) => + { + callback( res ); + + res.trigger( 'data', expected_data ); + res.trigger( 'end' ); + } ); + + Sut.extend( + { + 'override requestEnd'( given_res, data, callback ) + { + expect( given_res ).to.equal( res ); + expect( data ).to.equal( expected_data ); + + callback( e, value ); + }, + } )( { http: http }, _createMockUrl() ) + .requestData( "", 'GET', '', ( given_e, given_data ) => + { + expect( given_e ).to.equal( e ); + expect( given_data ).to.equal( value ); + + done(); + } ); + } ); + + + it( "allows overriding concept of success", done => + { + const res = _createMockResp(); + + const http = _createMockHttp( ( _, callback ) => + { + callback( res ); + res.trigger( 'end' ); + } ); + + // would normally be a failure + res.statusCode = 500; + + Sut.extend( + { + 'override isSuccessful': ( given_res ) => true, + } )( { http: http }, _createMockUrl() ) + .requestData( "", 'GET', '', ( e ) => + { + expect( e ).to.equal( null ); + done(); + } ); + } ); + + + it( "allows overriding error handling", done => + { + const expected_e = Error( 'expected' ); + const error = {}; + const value = 'error data'; + + const http = _createMockHttp( ( _, callback ) => + { + callback( _createMockResp() ); + } ); + + Sut.extend( + { + 'override serveError'( + given_e, given_res, given_data, callback + ) + { + expect( given_e ).to.equal( expected_e ); + expect( given_res ).to.equal( null ); + expect( given_data ).to.equal( null ); + + error.e = given_e; + callback( error, value ); + }, + } )( { http: http }, _createMockUrl() ) + .requestData( "", 'GET', '', ( e, given_value ) => + { + expect( e ).to.equal( error ); + expect( e.e ).to.equal( expected_e ); + expect( given_value ).to.equal( value ); + + done(); + } ); + + // we're still hanging the request since we haven't called the + // callback in http + http.req.trigger( 'error', expected_e ); + } ); + } ); +} ); + + +const _createMockHttp = req_callback => +{ + const events = {}; + + return Object.create( { + req: Object.create( { + written: '', + + on( event, hook ) + { + events[ event ] = hook; + }, + + trigger( event, data ) + { + events[ event ]( data ); + }, + + end() + { + // thunk defined by #request below + events.onend(); + }, + + write( data ) + { + this.written = data; + }, + } ), + + request( options, callback ) + { + // not a real event; just for convenience + events.onend = () => req_callback( options, callback ); + + return this.req; + }, + } ); +}; + + +const _createMockUrl = callback => ( { + parse: callback || ( () => ( { protocol: 'http:' } ) ), +} ); + +const _createMockResp = () => Object.create( { + event: { + data() {}, + end() {}, + }, + + statusCode: 200, + + on( ev, hook ) + { + this.event[ ev ] = hook; + }, + + trigger( ev, data ) + { + this.event[ ev ]( data ); + } +} ); From 624f35a489a56f686db1d329e5985ce3ba2525bf Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Tue, 13 Jun 2017 10:26:26 -0400 Subject: [PATCH 02/17] XhrHttpImpl: throw {=>Http}Error * src/dapi/http/XhrHttpImpl.js (serveError): Throw HttpError instead of Error. * test/dapi/http/XhrHttpImplTest.js: Test updated accordingly. --- src/dapi/http/XhrHttpImpl.js | 9 ++++++--- test/dapi/http/XhrHttpImplTest.js | 27 ++++++++++++++++----------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/dapi/http/XhrHttpImpl.js b/src/dapi/http/XhrHttpImpl.js index c4a0820..20e0f19 100644 --- a/src/dapi/http/XhrHttpImpl.js +++ b/src/dapi/http/XhrHttpImpl.js @@ -19,8 +19,11 @@ * along with this program. If not, see . */ -var Class = require( 'easejs' ).Class, - HttpImpl = require( './HttpImpl' ); +'use strict'; + +const Class = require( 'easejs' ).Class; +const HttpImpl = require( './HttpImpl' ); +const HttpError = require( './HttpError' ); /** @@ -272,7 +275,7 @@ module.exports = Class( 'XhrHttpImpl' ) */ 'virtual protected serveError': function( req, callback ) { - var e = Error( req.status + " error from server" ); + var e = HttpError( req.status + " error from server" ); e.status = req.status; callback( e, req.responseText ); diff --git a/test/dapi/http/XhrHttpImplTest.js b/test/dapi/http/XhrHttpImplTest.js index 17e6eac..588f416 100644 --- a/test/dapi/http/XhrHttpImplTest.js +++ b/test/dapi/http/XhrHttpImplTest.js @@ -19,19 +19,24 @@ * along with this program. If not, see . */ -var dapi = require( '../../../' ).dapi, - expect = require( 'chai' ).expect, - Class = require( 'easejs' ).Class, - HttpImpl = dapi.http.HttpImpl, - Sut = dapi.http.XhrHttpImpl, +'use strict'; - DummyXhr = function() +const { expect } = require( 'chai' ); +const { Class } = require( 'easejs' ); + +const { + HttpImpl, + XhrHttpImpl: Sut, + HttpError, +} = require( '../../../' ).dapi.http; + +const DummyXhr = function() +{ + this.open = function() { - this.open = function() - { - DummyXhr.args = arguments; - }; + DummyXhr.args = arguments; }; +}; describe( 'XhrHttpImpl', function() @@ -260,7 +265,7 @@ describe( 'XhrHttpImpl', function() Sut( StubXhr ) .requestData( 'http://foo', 'GET', '', function( err, _ ) { - expect( err ).to.be.instanceOf( Error ); + expect( err ).to.be.instanceOf( HttpError ); expect( err.message ).to.contain( StubXhr.prototype.status From a1d5e7d3c5a7fb51acd61be98deccbd574e5f300 Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Tue, 13 Jun 2017 11:53:37 -0400 Subject: [PATCH 03/17] Instantiate appropriate HttpImpl * src/dapi/DataApiFactory.js (fromType): Instantiate NodeHttpImpl when XMLHttpRequest is undefined. --- src/dapi/DataApiFactory.js | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/dapi/DataApiFactory.js b/src/dapi/DataApiFactory.js index af39742..650e52d 100644 --- a/src/dapi/DataApiFactory.js +++ b/src/dapi/DataApiFactory.js @@ -19,14 +19,14 @@ * along with this program. If not, see . */ -var Class = require( 'easejs' ).Class, - - HttpDataApi = require( './http/HttpDataApi' ), - XhrHttpImpl = require( './http/XhrHttpImpl' ), - JsonResponse = require( './format/JsonResponse' ), - RestrictedDataApi = require( './RestrictedDataApi' ), - StaticAdditionDataApi = require( './StaticAdditionDataApi' ), - BucketDataApi = require( './BucketDataApi' ); +const Class = require( 'easejs' ).Class; +const HttpDataApi = require( './http/HttpDataApi' ); +const XhrHttpImpl = require( './http/XhrHttpImpl' ); +const NodeHttpImpl = require( './http/NodeHttpImpl' ); +const JsonResponse = require( './format/JsonResponse' ); +const RestrictedDataApi = require( './RestrictedDataApi' ); +const StaticAdditionDataApi = require( './StaticAdditionDataApi' ); +const BucketDataApi = require( './BucketDataApi' ); /** @@ -58,10 +58,20 @@ module.exports = Class( 'DataApiFactory', switch ( type ) { case 'rest': + const impl = ( typeof XMLHttpRequest !== 'undefined' ) + ? XhrHttpImpl( XMLHttpRequest ) + : NodeHttpImpl( + { + http: require( 'http' ), + https: require( 'https' ), + }, + require( 'url' ) + ); + api = HttpDataApi.use( JsonResponse )( source, method.toUpperCase(), - XhrHttpImpl( XMLHttpRequest ) + impl ); break; From d2dd9636146e98b3813ca1bc84488ca951fdea22 Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Tue, 20 Jun 2017 10:29:52 -0400 Subject: [PATCH 04/17] doc: Extract design sections into own chapters * assert.texi, bucket.texi, client.texi, pred.texi, program.texi, validation.texi: New files. * design.texi: Extract text into above. * liza.texi: Add @menu references and @include each new file. --- doc/Makefile.am | 4 +- doc/assert.texi | 29 +++ doc/bucket.texi | 23 +++ doc/client.texi | 98 ++++++++++ doc/design.texi | 439 +------------------------------------------- doc/liza.texi | 12 ++ doc/pred.texi | 74 ++++++++ doc/program.texi | 222 ++++++++++++++++++++++ doc/validation.texi | 25 +++ 9 files changed, 487 insertions(+), 439 deletions(-) create mode 100644 doc/assert.texi create mode 100644 doc/bucket.texi create mode 100644 doc/client.texi create mode 100644 doc/pred.texi create mode 100644 doc/program.texi create mode 100644 doc/validation.texi diff --git a/doc/Makefile.am b/doc/Makefile.am index ccbd36a..2e84fc2 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -21,7 +21,9 @@ MAKEINFOHTML = $(MAKEINFO) --html --css-include liza.css info_TEXINFOS = liza.texi -liza_TEXINFOS = design.texi macros.texi config.texi liza.css \ +liza_TEXINFOS = macros.texi config.texi liza.css \ + design.texi assert.texi bucket.texi client.texi \ + pred.texi program.texi validation.texi \ diagram/event-graph.svg diagram/%.svg: diagram/%.dot diff --git a/doc/assert.texi b/doc/assert.texi new file mode 100644 index 0000000..ce16ce9 --- /dev/null +++ b/doc/assert.texi @@ -0,0 +1,29 @@ +@c This document is part of the Liza Data Collection Framework manual. +@c Copyright (C) 2017 R-T Specialty, LLC. +@c +@c Permission is granted to copy, distribute and/or modify this document +@c under the terms of the GNU Free Documentation License, Version 1.3 +@c or any later version published by the Free Software Foundation; +@c with no Invariant Sections, no Front-Cover Texts, and no Back-Cover +@c Texts. A copy of the license is included in the section entitled ``GNU +@c Free Documentation License''. + +@node Assertions +@chapter Assertions +@maintstart +@footnote{ + Assertions are compiled from the Program XML (@pxref{Program XML}). + Rather than using a library, + it compiles a mess of largely duplicate code inline. + This system needs to be @emph{replaced}, + not modified. + + A replacement can either be in the form of a library (removing most + if not all code generation from the Program XML compiler), + or possibly compile into classifications and use the classification + system. + @emph{The latter option is preferred, + and would be more powerful with less maintenance.}} +@maintend + +@helpwanted diff --git a/doc/bucket.texi b/doc/bucket.texi new file mode 100644 index 0000000..2e844c2 --- /dev/null +++ b/doc/bucket.texi @@ -0,0 +1,23 @@ +@c This document is part of the Liza Data Collection Framework manual. +@c Copyright (C) 2017 R-T Specialty, LLC. +@c +@c Permission is granted to copy, distribute and/or modify this document +@c under the terms of the GNU Free Documentation License, Version 1.3 +@c or any later version published by the Free Software Foundation; +@c with no Invariant Sections, no Front-Cover Texts, and no Back-Cover +@c Texts. A copy of the license is included in the section entitled ``GNU +@c Free Documentation License''. + +@node Bucket +@chapter Bucket +@helpwanted + +@menu +* Value Assignment:Bucket Assignment. Writing data to the Bucket. +@end menu + + +@c TODO +@node Bucket Assignment +@section Bucket Value Assignment +@helpwanted diff --git a/doc/client.texi b/doc/client.texi new file mode 100644 index 0000000..8955264 --- /dev/null +++ b/doc/client.texi @@ -0,0 +1,98 @@ +@c This document is part of the Liza Data Collection Framework manual. +@c Copyright (C) 2017 R-T Specialty, LLC. +@c +@c Permission is granted to copy, distribute and/or modify this document +@c under the terms of the GNU Free Documentation License, Version 1.3 +@c or any later version published by the Free Software Foundation; +@c with no Invariant Sections, no Front-Cover Texts, and no Back-Cover +@c Texts. A copy of the license is included in the section entitled ``GNU +@c Free Documentation License''. + +@node Client +@chapter Client +@maintstart +@footnote{ + The client is largely managed by a single class, + @srcref{src/client/Client.js, Client}, + which has grown out of control. + @code{Client} mediates essentially the entire system. + Code is to be extracted out of this class as it is touched. + + The other system mammoth is @code{Ui} (@pxref{Program UI}).} +@maintend + +@helpwanted + +@menu +* Error Handling:: +@end menu + + +@node Error Handling +@section Error Handling + +@maintenance{ + The complexity of this system and integration into legacy + layers has caused maintenance trouble in the past. + Each of the error checking layers need to be integrated + to reduce complexity.} + +@cindex Error +@cindex Failure +There are three layers of error checking:@footnote{ + Primarily for legacy reasons. + They are being consolodated as the system is touched.} + +@enumerate + @item Required field checking@mdash{ + }whether all required questions have been answered. + @item Type Validation@mdash{ + }verify that questions contain valid data according to their + declared type. @ref{Validation}. + @item Assertions@mdash{ + }arbitrary checks on data. + @ref{Assertions}. +@end enumerate + +@cindex Required Field +@cindex Field, Required +@cindex Field, Fixed +@cindex Error, Fixed +@cindex Error, Required +@cindex Fixed, Error +Required fields fail serially@mdash{ + }the system will notify the user of the required field, + and direct him/her to it (usually through scrolling). +A field is marked as @dfn{fixed} according to the rules + in @ref{Managing Error State}. + + +@menu +* Managing Error State:: Determining when failures should be marked as ``fixed'' +@end menu + + +@node Managing Error State +@subsection Managing Error State + +@cindex Failure Stack +@cindex Error Stack +Each failure caused by assertions is associated with a + @dfn{failure stack}. +The stack represents the trail of assertions that have run, + containing the ids of all values asserted against. +When any field or classification changes that is represented on the + failure stack, + the failure for the failed field associated with that failure stack + is cleared. + +@exnotice{ + If an assertion for some question @var{foo} first checked the value + of bucket field @var{bar}, + and within its failure checked the value @var{c:predicate}, + the failure stack would contain both of those ids. + If either @var{bar} or the @var{predicate} classification changed, + the question @var{foo} would have its error cleared.} + +Error state is managed by + @srcref{src/validate/ValidStateMonitor.js, ValidStateMonitor}. diff --git a/doc/design.texi b/doc/design.texi index 7bde80d..c054464 100644 --- a/doc/design.texi +++ b/doc/design.texi @@ -102,441 +102,4 @@ The main components of the system are: @xref{Validation}. @end table -More information about each can be found in their respective section. - - -@menu -* Assertions:: -* Bucket:: -* Client:: -* Predicate System:: -* Program:: -* Program UI:: -* Program XML:: -* Validation:: -@end menu - - - -@node Assertions -@section Assertions -@maintstart -@footnote{ - Assertions are compiled from the Program XML (@pxref{Program XML}). - Rather than using a library, - it compiles a mess of largely duplicate code inline. - This system needs to be @emph{replaced}, - not modified. - - A replacement can either be in the form of a library (removing most - if not all code generation from the Program XML compiler), - or possibly compile into classifications and use the classification - system. - @emph{The latter option is preferred, - and would be more powerful with less maintenance.}} -@maintend - -@helpwanted - - - -@node Bucket -@section Bucket -@helpwanted - -@menu -* Value Assignment:Bucket Assignment. Writing data to the Bucket. -@end menu - - -@c TODO -@node Bucket Assignment -@subsection Bucket Value Assignment -@helpwanted - - - -@node Client -@section Client -@maintstart -@footnote{ - The client is largely managed by a single class, - @srcref{src/client/Client.js, Client}, - which has grown out of control. - @code{Client} mediates essentially the entire system. - Code is to be extracted out of this class as it is touched. - - The other system mammoth is @code{Ui} (@pxref{Program UI}).} -@maintend - -@helpwanted - -@menu -* Error Handling:: -@end menu - - -@node Error Handling -@subsection Error Handling - -@maintenance{ - The complexity of this system and integration into legacy - layers has caused maintenance trouble in the past. - Each of the error checking layers need to be integrated - to reduce complexity.} - -@cindex Error -@cindex Failure -There are three layers of error checking:@footnote{ - Primarily for legacy reasons. - They are being consolodated as the system is touched.} - -@enumerate - @item Required field checking@mdash{ - }whether all required questions have been answered. - @item Type Validation@mdash{ - }verify that questions contain valid data according to their - declared type. @ref{Validation}. - @item Assertions@mdash{ - }arbitrary checks on data. - @ref{Assertions}. -@end enumerate - -@cindex Required Field -@cindex Field, Required -@cindex Field, Fixed -@cindex Error, Fixed -@cindex Error, Required -@cindex Fixed, Error -Required fields fail serially@mdash{ - }the system will notify the user of the required field, - and direct him/her to it (usually through scrolling). -A field is marked as @dfn{fixed} according to the rules - in @ref{Managing Error State}. - - -@menu -* Managing Error State:: Determining when failures should be marked as ``fixed'' -@end menu - - -@node Managing Error State -@subsubsection Managing Error State - -@cindex Failure Stack -@cindex Error Stack -Each failure caused by assertions is associated with a - @dfn{failure stack}. -The stack represents the trail of assertions that have run, - containing the ids of all values asserted against. -When any field or classification changes that is represented on the - failure stack, - the failure for the failed field associated with that failure stack - is cleared. - -@exnotice{ - If an assertion for some question @var{foo} first checked the value - of bucket field @var{bar}, - and within its failure checked the value @var{c:predicate}, - the failure stack would contain both of those ids. - If either @var{bar} or the @var{predicate} classification changed, - the question @var{foo} would have its error cleared.} - -Error state is managed by - @srcref{src/validate/ValidStateMonitor.js, ValidStateMonitor}. - - - -@node Predicate System -@section Predicate System -@maintstart -@footnote{ - New programs (using the old incarnation of TAME) use the classifier - embedded into the rater by TAME. - Old ones, however, still use the @dfn{Global Classifier}. - This system isn't as well tested as TAME's@mdash{ - }which needs to work properly for the sake of calculating premium@mdash{ - }and has suffered from a number of bugs in the past. - - The solution is to migrate all programs to TAME and remove that old - code.} -@maintend - -@tip{ - For a practical application of these concepts, see its use in the - Program@tie{}XML (@pxref{Specifying Predicates}).} - -@cindex Predicate -@cindex Classifier -@cindex Applicability -@cindex Domain of discourse, Predicate -The @dfn{predicate system} determines the @dfn{applicability} of - certain objects (like questions and assertions) by associating them - with predicates. -The domain of discourse (variables which may be quantified) is listed - in @ref{t:predicate-dod}. - -What it means for some object to be applicable depends on the context. - -@float Table, t:predicate-dod -@multitable @columnfractions 0.25 0.10 0.65 - @headitem Type @tab Prefix @tab Description - - @item Classifications - @tab @emph{None} - @tab Results of applying an external @dfn{classifier} to the bucket - (@pxref{Bucket}). - - @item Bucket Truth Predicate - @tab @code{q:} - @tab - Whether the given name in the bucket (@pxref{Bucket}) is - non-empty and non-zero. - The prefix @samp{q:} refers to its most common use - case---questions (@pxref{Program UI,,Program@tie{}UI}). -@end multitable -@caption{Predicate system domain of discourse} -@end float - -This system is limited to universal quantification over the domain of - discourse. -For other quantifiers and higher-order logic, - defer to one of the systems that contributes to the domain of - discourse, - like the classifier.@footnote{ - This is usually TAME. - The Program XML also supports inline classifications - with TAME's syntax (@pxref{Specifying Predicates}).} - -Predicates are usually specified in the Program XML - (@pxref{Specifying Predicates}) and compiled into the program - (@pxref{Program}). - - - -@node Program -@section Program -@maintstart -@footnote{ - The @code{Program} class was one of the first prototypes created, - and has evolved poorly with the rest of the system. - It is the base class for all compiled programs, - and it glues together too many other systems with a terrible - API and little to no encapsulation. - - With that said, it is one of the least touched classes (thus its - state); developers rarely have the need to touch @code{Program}.} -@maintend - - -@cindex Program -The @dfn{Program} is a declarative representation of an entire system. -It is the highest level of abstraction from a data perspective. -The user observes and interacts with a Program using the - @ref{Program UI,,Program@tie{}UI}. - -@cindex Program, XML -Programs contain a lot of metadata that is not in a convenience - human-readable (or modifiable) format, - some of which are redundant. -Programs are ideally compiled from a @ref{Program XML,,Program@tie{}XML} - document. - - - -@node Program UI -@section Program UI -@maintenance{ - The @code{Ui} class, - in addition to @srcref{src/client/Client,Client} (@pxref{Client}), - represent the two monoliths of the system. - This mediates all UI-related tasks, - and still has far too many concerns with far too many - dependencies. - Code is to be extracted out of this class as it is touched. -} - - -@cindex Program, User Interface -@cindex User Interface, Program -The @dfn{Program UI} renders a @ref{Program} as a form. - -@cindex Step -@cindex Group -At the highest level, - steps are rendered in a tab-like manner, - above the main form content. -A step contains groups, - which in turn contain elements such as questions. -Groups are delimited in some manner defined by their style - (@pxref{Group Styles}). - -@cindex Question -@cindex Question, Value Formatting -@cindex Bucket, Updating -Questions are rendered as form fields. -Any time the respective @ref{Bucket} field is changed, - the form field is updated to reflect those changes, - after having first been formatted with the appropriate validator - (@pxref{Formatting Values}). -When a question is changed by the user, - the value is expected to be propagated to the Bucket - (@pxref{Bucket Assignment}). - -@cindex Navigation Bar -@cindex User Interface, Navigation Bar -@cindex User Interface, Button Navigation -Navigation between steps can be done via the - @dfn{Navigation Bar} above the step@tie{}content, - or using ``Go@tie{}Back'' and ``Continue'' buttons at the foot of the - step@tie{}content. - -@cindex Sidebar -A @dfn{Sidebar} is rendered adjacent to the step content. -It displays the name of the Program, - as well as configurable metadata (usually through the @samp{sidebar} - node of the @ref{Program XML,,Program@tie{}XML}). -It also displays question help text (also configured through the XML) - and any error messages (@pxref{Error Handling}). - -@menu -* Group Styles:: Different ways of displaying groups of questions to - the user. -@end menu - - -@node Group Styles -@subsection Group Styles -@refactor{ - Some group styles still use jQuery; - they should be modified to use modern formatters and Liza DOM - abstractions (see @srcrefraw{src/ui/field} - and @srcrefraw{src/ui/styler}).} - -@cindex Group, Styling -Groups support a number of @dfn{group styles} that determine how - they are delimited from other groups; - how the elements they contain are rendered and laid out; - and how multiple indexes are displayed, added, and removed. -A list of available styles is detailed in @ref{t:group-styles}. - -@float Table, t:group-styles -@multitable @columnfractions 0.15 0.65 0.10 0.10 - @headitem Name @tab Description @tab Multi-Index? @tab Add/Remove Index? - - @item @samp{default} - @tab - Groups are unstyled by default@mdash{ - }they render elements as flat fields like a traditional form. - Only the first index of elements is rendered. - @tab@center N - @tab@center N - - @item @samp{collapsetable} - @tab - Renders element label in the leftmost column like @samp{sidetable}. - Indexes are groups of rows delimited by headings, - which collapse the respective group of rows when clicked. - @tab@center Y - @tab@center Add - - @item @samp{sidetable} - @tab - Renders elements as rows with label in the leftmost column rather - than the top row. - Each index is rendered as a column. - @tab@center Y - @tab@center Add - - @item @samp{tabbedblock} - @tab - Each group is rendered as a block, - with each index rendered as a tab to the right of it. - Clicking a tab toggles the body content to the associated index. - Elements are rendered within the box. - @tab@center Y - @tab@center N - - @item @samp{tabbed} - @tab - Like @samp{default}, - but each index has a tab at the top of the group. - Clicking a tab toggles the body content to the associated index. - @tab@center Y - @tab@center Y - - @item @samp{table} - @tab - A vanilla table with elements as columns, - their labels across the top row. - Each index is rendered in its own row. - @tab@center Y - @tab@center Y -@end multitable -@caption{Group styles and index support} -@end float - - - -@node Program XML -@section Program XML -@helpwanted - -@menu -* Specifying Predicates:: -@end menu - - -@node Specifying Predicates -@subsection Specifying Predicates - -Object predicates (@pxref{Predicate System}) are specified using the - @xmlattr{when} attribute of certain nodes. -It must contain a string of references understood by the system - (see domain of discourse, @ref{Predicate System}), - all of which much match for the predicate to be true. - -@float Figure, f:pred-when -@example - - - -@end example -@caption{Using the @xmlattr{when} attribute} -@end float - -In @ref{f:pred-when} above, - question @samp{vacant_desc} will be applicable when @emph{all} of - the values of @samp{vacant}, @samp{property}, - and@tie{}@samp{q:describe} are true.@footnote{ - @xref{Predicate System} for what ``true'' means for a particular - variable in the domain of discourse.} -Within the context of the @progxml, - this concretely means that the classifications - @samp{vacant} and@tie{}@samp{property} are true, - and that the question @samp{describe} is answered ``yes''. -It reads as a sentence: - ``@samp{vacant_desc}'' is applicable when we should @tie{}``describe - a vacant property''. - - - -@node Validation -@section Validation -@helpwanted - -@menu -* Formatting Values:: -@end menu - - -@node Formatting Values -@subsection Formatting Values - -@cindex Question -@cindex Question, Value Formatting -@helpwanted +More information about each can be found in their respective chapter/section. diff --git a/doc/liza.texi b/doc/liza.texi index d091cd9..e8591a8 100644 --- a/doc/liza.texi +++ b/doc/liza.texi @@ -76,6 +76,12 @@ Free Documentation License''. @menu * Design and Architecture:Design. Design and structure of framework +* Assertions:: +* Bucket:: +* Client:: +* Predicate System:: +* Program:: +* Validation:: * License:: Document License * Concept Index:: @ifset DEVNOTES @@ -89,6 +95,12 @@ Free Documentation License''. @end ifnottex @include design.texi +@include assert.texi +@include bucket.texi +@include client.texi +@include pred.texi +@include program.texi +@include validation.texi @include license.texi @node Concept Index diff --git a/doc/pred.texi b/doc/pred.texi new file mode 100644 index 0000000..8ee01eb --- /dev/null +++ b/doc/pred.texi @@ -0,0 +1,74 @@ +@c This document is part of the Liza Data Collection Framework manual. +@c Copyright (C) 2017 R-T Specialty, LLC. +@c +@c Permission is granted to copy, distribute and/or modify this document +@c under the terms of the GNU Free Documentation License, Version 1.3 +@c or any later version published by the Free Software Foundation; +@c with no Invariant Sections, no Front-Cover Texts, and no Back-Cover +@c Texts. A copy of the license is included in the section entitled ``GNU +@c Free Documentation License''. + +@node Predicate System +@chapter Predicate System +@maintstart +@footnote{ + New programs (using the old incarnation of TAME) use the classifier + embedded into the rater by TAME. + Old ones, however, still use the @dfn{Global Classifier}. + This system isn't as well tested as TAME's@mdash{ + }which needs to work properly for the sake of calculating premium@mdash{ + }and has suffered from a number of bugs in the past. + + The solution is to migrate all programs to TAME and remove that old + code.} +@maintend + +@tip{ + For a practical application of these concepts, see its use in the + Program@tie{}XML (@pxref{Specifying Predicates}).} + +@cindex Predicate +@cindex Classifier +@cindex Applicability +@cindex Domain of discourse, Predicate +The @dfn{predicate system} determines the @dfn{applicability} of + certain objects (like questions and assertions) by associating them + with predicates. +The domain of discourse (variables which may be quantified) is listed + in @ref{t:predicate-dod}. + +What it means for some object to be applicable depends on the context. + +@float Table, t:predicate-dod +@multitable @columnfractions 0.25 0.10 0.65 + @headitem Type @tab Prefix @tab Description + + @item Classifications + @tab @emph{None} + @tab Results of applying an external @dfn{classifier} to the bucket + (@pxref{Bucket}). + + @item Bucket Truth Predicate + @tab @code{q:} + @tab + Whether the given name in the bucket (@pxref{Bucket}) is + non-empty and non-zero. + The prefix @samp{q:} refers to its most common use + case---questions (@pxref{Program UI,,Program@tie{}UI}). +@end multitable +@caption{Predicate system domain of discourse} +@end float + +This system is limited to universal quantification over the domain of + discourse. +For other quantifiers and higher-order logic, + defer to one of the systems that contributes to the domain of + discourse, + like the classifier.@footnote{ + This is usually TAME. + The Program XML also supports inline classifications + with TAME's syntax (@pxref{Specifying Predicates}).} + +Predicates are usually specified in the Program XML + (@pxref{Specifying Predicates}) and compiled into the program + (@pxref{Program}). diff --git a/doc/program.texi b/doc/program.texi new file mode 100644 index 0000000..76d6482 --- /dev/null +++ b/doc/program.texi @@ -0,0 +1,222 @@ +@c This document is part of the Liza Data Collection Framework manual. +@c Copyright (C) 2017 R-T Specialty, LLC. +@c +@c Permission is granted to copy, distribute and/or modify this document +@c under the terms of the GNU Free Documentation License, Version 1.3 +@c or any later version published by the Free Software Foundation; +@c with no Invariant Sections, no Front-Cover Texts, and no Back-Cover +@c Texts. A copy of the license is included in the section entitled ``GNU +@c Free Documentation License''. + + +@node Program +@chapter Program +@maintstart +@footnote{ + The @code{Program} class was one of the first prototypes created, + and has evolved poorly with the rest of the system. + It is the base class for all compiled programs, + and it glues together too many other systems with a terrible + API and little to no encapsulation. + + With that said, it is one of the least touched classes (thus its + state); developers rarely have the need to touch @code{Program}.} +@maintend + + +@cindex Program +The @dfn{Program} is a declarative representation of an entire system. +It is the highest level of abstraction from a data perspective. +The user observes and interacts with a Program using the + @ref{Program UI,,Program@tie{}UI}. + +@cindex Program, XML +Programs contain a lot of metadata that is not in a convenience + human-readable (or modifiable) format, + some of which are redundant. +Programs are ideally compiled from a @ref{Program XML,,Program@tie{}XML} + document. + +@menu +* Program UI:: +* Program XML:: +@end menu + +@node Program UI +@section Program UI +@maintenance{ + The @code{Ui} class, + in addition to @srcref{src/client/Client,Client} (@pxref{Client}), + represent the two monoliths of the system. + This mediates all UI-related tasks, + and still has far too many concerns with far too many + dependencies. + Code is to be extracted out of this class as it is touched. +} + + +@cindex Program, User Interface +@cindex User Interface, Program +The @dfn{Program UI} renders a @ref{Program} as a form. + +@cindex Step +@cindex Group +At the highest level, + steps are rendered in a tab-like manner, + above the main form content. +A step contains groups, + which in turn contain elements such as questions. +Groups are delimited in some manner defined by their style + (@pxref{Group Styles}). + +@cindex Question +@cindex Question, Value Formatting +@cindex Bucket, Updating +Questions are rendered as form fields. +Any time the respective @ref{Bucket} field is changed, + the form field is updated to reflect those changes, + after having first been formatted with the appropriate validator + (@pxref{Formatting Values}). +When a question is changed by the user, + the value is expected to be propagated to the Bucket + (@pxref{Bucket Assignment}). + +@cindex Navigation Bar +@cindex User Interface, Navigation Bar +@cindex User Interface, Button Navigation +Navigation between steps can be done via the + @dfn{Navigation Bar} above the step@tie{}content, + or using ``Go@tie{}Back'' and ``Continue'' buttons at the foot of the + step@tie{}content. + +@cindex Sidebar +A @dfn{Sidebar} is rendered adjacent to the step content. +It displays the name of the Program, + as well as configurable metadata (usually through the @samp{sidebar} + node of the @ref{Program XML,,Program@tie{}XML}). +It also displays question help text (also configured through the XML) + and any error messages (@pxref{Error Handling}). + +@menu +* Group Styles:: Different ways of displaying groups of questions to + the user. +@end menu + + +@node Group Styles +@subsection Group Styles +@refactor{ + Some group styles still use jQuery; + they should be modified to use modern formatters and Liza DOM + abstractions (see @srcrefraw{src/ui/field} + and @srcrefraw{src/ui/styler}).} + +@cindex Group, Styling +Groups support a number of @dfn{group styles} that determine how + they are delimited from other groups; + how the elements they contain are rendered and laid out; + and how multiple indexes are displayed, added, and removed. +A list of available styles is detailed in @ref{t:group-styles}. + +@float Table, t:group-styles +@multitable @columnfractions 0.15 0.65 0.10 0.10 + @headitem Name @tab Description @tab Multi-Index? @tab Add/Remove Index? + + @item @samp{default} + @tab + Groups are unstyled by default@mdash{ + }they render elements as flat fields like a traditional form. + Only the first index of elements is rendered. + @tab@center N + @tab@center N + + @item @samp{collapsetable} + @tab + Renders element label in the leftmost column like @samp{sidetable}. + Indexes are groups of rows delimited by headings, + which collapse the respective group of rows when clicked. + @tab@center Y + @tab@center Add + + @item @samp{sidetable} + @tab + Renders elements as rows with label in the leftmost column rather + than the top row. + Each index is rendered as a column. + @tab@center Y + @tab@center Add + + @item @samp{tabbedblock} + @tab + Each group is rendered as a block, + with each index rendered as a tab to the right of it. + Clicking a tab toggles the body content to the associated index. + Elements are rendered within the box. + @tab@center Y + @tab@center N + + @item @samp{tabbed} + @tab + Like @samp{default}, + but each index has a tab at the top of the group. + Clicking a tab toggles the body content to the associated index. + @tab@center Y + @tab@center Y + + @item @samp{table} + @tab + A vanilla table with elements as columns, + their labels across the top row. + Each index is rendered in its own row. + @tab@center Y + @tab@center Y +@end multitable +@caption{Group styles and index support} +@end float + + + +@node Program XML +@section Program XML +@helpwanted + +@menu +* Specifying Predicates:: +@end menu + + +@node Specifying Predicates +@subsection Specifying Predicates + +Object predicates (@pxref{Predicate System}) are specified using the + @xmlattr{when} attribute of certain nodes. +It must contain a string of references understood by the system + (see domain of discourse, @ref{Predicate System}), + all of which much match for the predicate to be true. + +@float Figure, f:pred-when +@example + + + +@end example +@caption{Using the @xmlattr{when} attribute} +@end float + +In @ref{f:pred-when} above, + question @samp{vacant_desc} will be applicable when @emph{all} of + the values of @samp{vacant}, @samp{property}, + and@tie{}@samp{q:describe} are true.@footnote{ + @xref{Predicate System} for what ``true'' means for a particular + variable in the domain of discourse.} +Within the context of the @progxml, + this concretely means that the classifications + @samp{vacant} and@tie{}@samp{property} are true, + and that the question @samp{describe} is answered ``yes''. +It reads as a sentence: + ``@samp{vacant_desc}'' is applicable when we should @tie{}``describe + a vacant property''. diff --git a/doc/validation.texi b/doc/validation.texi new file mode 100644 index 0000000..8ad0e29 --- /dev/null +++ b/doc/validation.texi @@ -0,0 +1,25 @@ +@c This document is part of the Liza Data Collection Framework manual. +@c Copyright (C) 2017 R-T Specialty, LLC. +@c +@c Permission is granted to copy, distribute and/or modify this document +@c under the terms of the GNU Free Documentation License, Version 1.3 +@c or any later version published by the Free Software Foundation; +@c with no Invariant Sections, no Front-Cover Texts, and no Back-Cover +@c Texts. A copy of the license is included in the section entitled ``GNU +@c Free Documentation License''. + +@node Validation +@chapter Validation +@helpwanted + +@menu +* Formatting Values:: +@end menu + + +@node Formatting Values +@section Formatting Values + +@cindex Question +@cindex Question, Value Formatting +@helpwanted From 83b5f1ab74f76b360889f7fae2cbe5a0c6f2f750 Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Tue, 20 Jun 2017 12:52:45 -0400 Subject: [PATCH 05/17] doc: Update @helpwanted text * doc/macros.texi (helpwanted): Adjust phrasing to account for some existing text. --- doc/macros.texi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/macros.texi b/doc/macros.texi index db725ba..46a4c15 100644 --- a/doc/macros.texi +++ b/doc/macros.texi @@ -71,7 +71,7 @@ @macro helpwanted{} @cindex TODO, Missing Docs @dnindex Missing Docs -@notice{There's nothing here yet. Maybe you can help?} +@notice{There isn't much here yet. Maybe you can help?} @end macro From f873da092855ce1cffebba66bf42cd4ec4fa765d Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Tue, 20 Jun 2017 12:53:14 -0400 Subject: [PATCH 06/17] doc: Add @srcrefjs macros * doc/macros.texi (srcrefjs): Add macros when src is both available and not --- doc/macros.texi | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/doc/macros.texi b/doc/macros.texi index 46a4c15..7eb428b 100644 --- a/doc/macros.texi +++ b/doc/macros.texi @@ -140,11 +140,15 @@ This system has maintenance concerns. @c the path to the file @ifset SRCURI @macro srcref{path, display} -@url{@value{SRCURI}/\path\, @code{\display\}} +@url{@value{SRCURI}/\path\, @file{\display\}} @end macro @macro srcrefraw{path} -@url{@value{SRCURI}/\path\, @code{\path\}} +@url{@value{SRCURI}/\path\, @file{\path\}} +@end macro + +@macro srcrefjs{base,module} +@srcref{src/\base\/\module\.js, \module\} @end macro @end ifset @@ -156,6 +160,12 @@ This system has maintenance concerns. @macro srcrefraw{path} @file{\path\} @end macro + +@c intended to display a name without JS, +@c so just do that rather than the actual path +@macro srcrefjs{base,path} +@srcrefraw{\base\/\path\} +@end macro @end ifclear From be0121d8cba9d6ed5b6a014fa3c03bc9524251f3 Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Tue, 20 Jun 2017 12:54:11 -0400 Subject: [PATCH 07/17] doc: Add @progxmlref, @dapi, @procguic{,x}ref, @proguicrefsuffix * doc/macros.texi (@progxmlref, @dapi, @progcuicref, @proguicxref @proguicrefsuffix): Add macros. --- doc/macros.texi | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/doc/macros.texi b/doc/macros.texi index 7eb428b..f8d052c 100644 --- a/doc/macros.texi +++ b/doc/macros.texi @@ -183,3 +183,24 @@ This system has maintenance concerns. @macro progxml Program@tie{}XML @end macro + +@macro progxmlref +@ref{Program XML,,Program@tie{}XML} +@end macro + +@macro dapi +Data@tie{}API +@end macro + +@c todo: link to reference directly +@macro proguicref{ref} +`\ref\' @proguicrefsuffix +@end macro + +@macro proguicxref{ref} +See `\ref\' @proguicrefsuffix +@end macro + +@macro proguicrefsuffix{} +in the Liza Program@tie{}UI Compiler manual +@end macro From 5c99243cdcd05136d5716dea9ef44ad72135a1f2 Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Tue, 20 Jun 2017 12:55:20 -0400 Subject: [PATCH 08/17] doc: Add beginning of dapi documentation * doc/Makefile.am (liza_TEXINFOS) [dapi.texi]: Include file. * doc/dapi.texi: Add file. * doc/design.texi: Add mention in list and reference chapter. * doc/liza.texi: Reference chapter. --- doc/Makefile.am | 2 +- doc/dapi.texi | 149 ++++++++++++++++++++++++++++++++++++++++++++++++ doc/design.texi | 6 ++ doc/liza.texi | 2 + 4 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 doc/dapi.texi diff --git a/doc/Makefile.am b/doc/Makefile.am index 2e84fc2..9221169 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -23,7 +23,7 @@ MAKEINFOHTML = $(MAKEINFO) --html --css-include liza.css info_TEXINFOS = liza.texi liza_TEXINFOS = macros.texi config.texi liza.css \ design.texi assert.texi bucket.texi client.texi \ - pred.texi program.texi validation.texi \ + dapi.texi pred.texi program.texi validation.texi \ diagram/event-graph.svg diagram/%.svg: diagram/%.dot diff --git a/doc/dapi.texi b/doc/dapi.texi new file mode 100644 index 0000000..ca61c92 --- /dev/null +++ b/doc/dapi.texi @@ -0,0 +1,149 @@ +@c This document is part of the Liza Data Collection Framework manual. +@c Copyright (C) 2017 R-T Specialty, LLC. +@c +@c Permission is granted to copy, distribute and/or modify this document +@c under the terms of the GNU Free Documentation License, Version 1.3 +@c or any later version published by the Free Software Foundation; +@c with no Invariant Sections, no Front-Cover Texts, and no Back-Cover +@c Texts. A copy of the license is included in the section entitled ``GNU +@c Free Documentation License''. + +@node Data API +@chapter Data API +@maintenance{This is a complex system with too much logic lying in + @srcrefjs{dapi,DataApiManager} (having been extracted + from its old home in @srcrefjs{program,Program} ).} + +@helpwanted{} + +The @dfn{Data API} is a declarative abstraction for accessing and + processing remote data (e.g. a RESTful service). +The name stems from how it is used@mdash{ + }to declare an remote API's inputs and outputs. + +This system is generally used indirectly through the @progxmlref{}.@footnote{ + @proguicxref{Data API}.} + +@tip{All interaction with this system should be had through the + @srcrefjs{dapi,DataApiManager}.} + +The @srcrefjs{dapi,DataApiManager} manages the entire operation@mdash{ + }from triggering the initial request, + to performing mapping, + to populating bucket data. +It takes only a @srcrefjs{dapi,DataApiFactory} and @dapi{} definitions. + +Definitions have the following schema:@footnote{ + There are poor design decisions that will likely persist + indefinitely because of integration with other systems, + so future extensions may be messy (especially in the case of + @samp{retvals}). +} + +@verbatim +{ + "type": "string", + "source": "string", + "method": "string", + "params": { + ["string(name)"]: { + "name": "string(name)", + "default": { + "type": "string", + "value": "string" + }, + ... + }, + }, + "retvals": [ "string", ... ], + "static": [ + { + ["string(param)"]: "string", + ... + }, + ... + ], + "static_nonempty": boolean, + "static_multiple": boolean +} +@end verbatim + +Each of the above fields are defined by: + +@table @code + @item type + Any type supported by @srcrefjs{dapi,DataApiFactory} (e.g. @samp{rest}). + + @item source + Type-specific source of data. + For e.g. @samp{rest}, this is a URI. + + @item method + Type-specific method for interacting with the API. + For e.g. @samp{rest}, this is an HTTP@tie{}method. + + @item params + Key-value mapping of input parameter names (as received by @samp{source}) + to their default values. + These inputs must be populated by the caller at the time of the request. + + @item retvals + Array of fields returned by the data source. + + @item static + Static values to prepend to the returned data. + This is often used for adding ``please select'' text, for example. + + @item static_nonempty + Whether statics should be added when there is return data; + Otherwise, + they will be added only if the response yields no results. + + @item static_multiple + Whether statics should be added only if multiple data are returned. + For example, + a ``please select'' is only useful if there is more than one + option for the user to select from. + When @samp{true}, + this has the convenient side-effect of auto-selecting the only + result. +@end table + +An example definition appears in @ref{f:dapi-ex} + +@float Figure, f:dapi-ex +@example + @{ + "type": "rest", + "source": "/foo/city", + "method": "post", + "params": @{ + "getVal": @{ + "name": "getVal", + "default": @{ + "type": "string", + "value": "getCityOptions" + @} + @}, + "zipcode": @{ + "name": "zipcode", + "default": @{ + "type": "ref", + "value": "" + @} + @} + @}, + "retvals": [ "city", "id", "state", "county", "country" ], + "static": [ @{ + "city": "(Please Select)", + "id": "", + "state": "", + "county": "", + "country": "" + @} ], + "static_nonempty": false, + "static_multiple": true + @}, +@end example +@caption{Example @dapi{} definition} +@end float diff --git a/doc/design.texi b/doc/design.texi index c054464..20f3ed9 100644 --- a/doc/design.texi +++ b/doc/design.texi @@ -47,6 +47,12 @@ The main components of the system are: etc. @xref{Client}. + @cindex Data API + @item Data API + Declarative abstraction for accessing and processing remote data + (e.g. a RESTful service). + @xref{Data API}. + @cindex Developer Dialog @item Developer Dialog Renders information about the system for debugging the client. diff --git a/doc/liza.texi b/doc/liza.texi index e8591a8..38cb00a 100644 --- a/doc/liza.texi +++ b/doc/liza.texi @@ -79,6 +79,7 @@ Free Documentation License''. * Assertions:: * Bucket:: * Client:: +* Data API:: * Predicate System:: * Program:: * Validation:: @@ -98,6 +99,7 @@ Free Documentation License''. @include assert.texi @include bucket.texi @include client.texi +@include dapi.texi @include pred.texi @include program.texi @include validation.texi From f870ca964c9d30239d2057c3cab6309eeb8b8f23 Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Wed, 21 Jun 2017 13:20:40 -0400 Subject: [PATCH 09/17] doc: Add @todo * doc/macros.texi (@todo): Add macro. --- doc/macros.texi | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/macros.texi b/doc/macros.texi index f8d052c..bda69d9 100644 --- a/doc/macros.texi +++ b/doc/macros.texi @@ -66,6 +66,11 @@ @end ifset @end macro +@c documentation TODO +@macro todo{text} +@devnotice{TODO: \text\} +@end macro + @c indicate that help is needed to produce docs @macro helpwanted{} From c231983f349755bb55b73e575a582008ceccbbc9 Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Wed, 21 Jun 2017 13:20:56 -0400 Subject: [PATCH 10/17] doc: Add @jsmethod macro * doc/macros.texi (@jsmethod): Add macro. --- doc/macros.texi | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/macros.texi b/doc/macros.texi index bda69d9..7340491 100644 --- a/doc/macros.texi +++ b/doc/macros.texi @@ -184,6 +184,12 @@ This system has maintenance concerns. @end macro +@c JS formatting +@macro jsmethod{name} +@code{#\name\} +@end macro + + @c text to avoid repeated e.g. ties and other formatting @macro progxml Program@tie{}XML From 630af0a0627417a782d792883976d77c50f2a96d Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Wed, 21 Jun 2017 13:55:15 -0400 Subject: [PATCH 11/17] Add ability to provide origin to NodeHttpImpl * src/dapi/http/NodeHttpImpl.js (_parseUrl): Add method. (__construct): Add argument. (requestData): Use it. * test/dapi/http/NodeHttpImplTest.js: Add tests. --- src/dapi/http/NodeHttpImpl.js | 33 +++++++++++++++++++-- test/dapi/http/NodeHttpImplTest.js | 47 ++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/src/dapi/http/NodeHttpImpl.js b/src/dapi/http/NodeHttpImpl.js index 495e2e1..541b332 100644 --- a/src/dapi/http/NodeHttpImpl.js +++ b/src/dapi/http/NodeHttpImpl.js @@ -45,6 +45,12 @@ module.exports = Class( 'NodeHttpImpl' ) */ 'private _urlParser': '', + /** + * Request origin + * @type {string} + */ + 'private _origin': '', + /** * Initialize with protocol handlers and URL parser @@ -53,13 +59,17 @@ module.exports = Class( 'NodeHttpImpl' ) * to a handler object conforming to Node's http(s) APIs---that is, it * should provide a `#request` method. * + * `origin` is prepended to all request URLs. + * * @param {Object} proto_handlers protocol handler key-value map * @param {Object} url_parser URL parser + * @param {string} origin request origin */ - constructor( proto_handlers, url_parser ) + constructor( proto_handlers, url_parser, origin ) { this._protoHandlers = proto_handlers; this._urlParser = url_parser; + this._origin = ( origin !== undefined ) ? ''+origin : ''; }, @@ -79,7 +89,7 @@ module.exports = Class( 'NodeHttpImpl' ) */ 'public requestData'( url, method, data, callback ) { - const options = this._urlParser.parse( url ); + const options = this._parseUrl( url ); const protocol = options.protocol.replace( /:$/, '' ); const handler = this._protoHandlers[ protocol ]; @@ -120,6 +130,25 @@ module.exports = Class( 'NodeHttpImpl' ) }, + /** + * Parse given URL + * + * If the URL begins with a slash, the origin is prepended. + * + * @param {string} url URL + * + * @return {Object} parsed URL + */ + 'private _parseUrl'( url ) + { + const origin = ( url[ 0 ] === '/' ) + ? this._origin + : ''; + + return this._urlParser.parse( origin + url ); + }, + + /** * Set request options * diff --git a/test/dapi/http/NodeHttpImplTest.js b/test/dapi/http/NodeHttpImplTest.js index 96a4195..72ba65a 100644 --- a/test/dapi/http/NodeHttpImplTest.js +++ b/test/dapi/http/NodeHttpImplTest.js @@ -90,6 +90,53 @@ describe( "NodeHttpImpl", () => } ); + describe( "given an origin", () => + { + it( "prepends to URL if URL begins with a slash", done => + { + const origin = 'https://foo.com'; + const path = '/quux/quuux'; + + const url = _createMockUrl( given_url => + { + expect( given_url ).to.equal( origin + path ); + done(); + } ); + + const http = _createMockHttp( ( _, callback ) => + { + callback( res ); + res.trigger( 'end' ); + } ); + + Sut( { http: http }, url, origin ) + .requestData( path, 'GET', {}, () => {} ); + } ); + + + it( "does not prepend to URL that does not begin with a slash", done => + { + const origin = 'https://bar.com'; + const path = 'http://foo.com/quux/quuux'; + + const url = _createMockUrl( given_url => + { + expect( given_url ).to.equal( path ); + done(); + } ); + + const http = _createMockHttp( ( _, callback ) => + { + callback( res ); + res.trigger( 'end' ); + } ); + + Sut( { http: http }, url, origin ) + .requestData( path, 'GET', {}, () => {} ); + } ); + } ); + + it( "returns response when no error", done => { const res = _createMockResp(); From 07b5d2f1b6cb7c4cf16863651ec9f876cc77f4c6 Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Thu, 22 Jun 2017 13:07:30 -0400 Subject: [PATCH 12/17] doc: Began server documentation * server.texi: Add file. --- doc/Makefile.am | 3 +- doc/bucket.texi | 13 +++ doc/design.texi | 5 +- doc/liza.texi | 2 + doc/server.texi | 231 ++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 249 insertions(+), 5 deletions(-) create mode 100644 doc/server.texi diff --git a/doc/Makefile.am b/doc/Makefile.am index 9221169..508ed8b 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -23,7 +23,8 @@ MAKEINFOHTML = $(MAKEINFO) --html --css-include liza.css info_TEXINFOS = liza.texi liza_TEXINFOS = macros.texi config.texi liza.css \ design.texi assert.texi bucket.texi client.texi \ - dapi.texi pred.texi program.texi validation.texi \ + dapi.texi pred.texi program.texi server.texi \ + validation.texi \ diagram/event-graph.svg diagram/%.svg: diagram/%.dot diff --git a/doc/bucket.texi b/doc/bucket.texi index 2e844c2..0b4a810 100644 --- a/doc/bucket.texi +++ b/doc/bucket.texi @@ -14,6 +14,8 @@ @menu * Value Assignment:Bucket Assignment. Writing data to the Bucket. +* Bucket Diff:: Representing bucket changes. +* Calculated Values:: Dynamic data derived from other values. @end menu @@ -21,3 +23,14 @@ @node Bucket Assignment @section Bucket Value Assignment @helpwanted + + +@node Bucket Diff +@section Bucket Diff +@cindex Bucket diff +@helpwanted + + +@node Calculated Values +@section Calculated Values +@helpwanted diff --git a/doc/design.texi b/doc/design.texi index 20f3ed9..d605fde 100644 --- a/doc/design.texi +++ b/doc/design.texi @@ -34,10 +34,6 @@ The main components of the system are: and provides hooks that drive the rest of the system. @xref{Bucket}. - @cindex Calculated Value - @item Calculated Values - A small sub-system for calculating bucket values from other values. - @cindex Client @item Client Basic logic for navigating between steps, @@ -97,6 +93,7 @@ The main components of the system are: and other types of processing. Code is shared with the client, ensuring identical behavior for appropriate behaviors. + @xref{Server}. @cindex Type Validation @cindex Validation, Type diff --git a/doc/liza.texi b/doc/liza.texi index 38cb00a..c8bd87e 100644 --- a/doc/liza.texi +++ b/doc/liza.texi @@ -82,6 +82,7 @@ Free Documentation License''. * Data API:: * Predicate System:: * Program:: +* Server:: * Validation:: * License:: Document License * Concept Index:: @@ -102,6 +103,7 @@ Free Documentation License''. @include dapi.texi @include pred.texi @include program.texi +@include server.texi @include validation.texi @include license.texi diff --git a/doc/server.texi b/doc/server.texi new file mode 100644 index 0000000..752f74a --- /dev/null +++ b/doc/server.texi @@ -0,0 +1,231 @@ +@c This document is part of the Liza Data Collection Framework manual. +@c Copyright (C) 2017 R-T Specialty, LLC. +@c +@c Permission is granted to copy, distribute and/or modify this document +@c under the terms of the GNU Free Documentation License, Version 1.3 +@c or any later version published by the Free Software Foundation; +@c with no Invariant Sections, no Front-Cover Texts, and no Back-Cover +@c Texts. A copy of the license is included in the section entitled ``GNU +@c Free Documentation License''. + +@node Server +@chapter Liza Server +@maintenance{The @srcrefjs{server/daemon,Daemon} monolith and + @srcrefjs{server,Server}, + among other things, + need refactoring.} + +@helpwanted{} + +@cindex Server +The @dfn{server}@footnote{ + @cindex Quote Server + Which may also be referenced as ``quote server'' in certain legacy + contexts, + referring to Liza's origin as an insurance rating system.} + is a RESTful service that serves as the HTTP server. +It is designed to run under Node.js, + motivated by the benefits of sharing code with the@tie{}Client + (@pxref{Client}). +The daemon is handled by the abstract @srcrefjs{server/daemon,Daemon} + monolith, + which requires that a concrete @code{#getEncryptionService} + method be defined by a subtype or trait. +An example script to start the server is shown in @ref{f:server-start}. + +@cindex Encryption Service +@tip{For local development, + or to avoid use of any encryption service, + use @srcrefjs{server/daemon,DevDaemon}, + which uses a dummy encryption service.} + +@float Figure, f:server-start +@example + const @{ Daemon @} = require( 'liza' ).server.daemon; + const port = 8080; + const log_priority = 10; + + Daemon( port, log_priority ).start(); +@end example +@caption{Starting the server daemon} +@end float + +@cindex HTTP Server +The HTTP server is managed by + @srcrefjs{server/daemon,http_server}. + + +@menu +* Requests:: Handling HTTP requests +* Posting Data:: Handling step saves and other posts. +* Encryption Service:: Managing sensitive data. +@end menu + + + +@node Requests +@section HTTP Requests +@helpwanted{} + +@cindex Session +@cindex PHPSESSID +@cindex Memcache +Each HTTP request produces a @srcrefjs{server/request,UserRequest} + associated with a @srcrefjs{server/request,UserSession}. +Sessions are tightly coupled with PHP@footnote{ + They don't have to be@mdash{}refactoring is needed.}; + an existing PHP session is expected, + as identified by the @samp{PHPSESSID} cookie. +Sessions are shared via Memcache + (see @srcrefjs{server/cache,ResilientMemcache}).@footnote{ + Via a @url{https://secure.php.net/manual/en/memcached.sessions.php,memcache session handler}.} + If a session is not found (or is invalid), + an HTTP@tie{}@code{500} status code is returned and the + HTTP@tie{}request is aborted. + +@cindex Timeout +@cindex Request timeout +Requests are subject to a 120@tie{}second timeout, + after which the request will be served an HTTP@tie{}@code{408} + status code. +Note that this @emph{does not stop background processing}@mdash{ + }this timeout exists to prevent the user from hanging indefinitely. + +@cindex Long-running requests +@tip{If a process intends to perform background processing for any length + of time (longer than a few seconds), + it should complete the request as quickly as possible and + use some other mechanism to report back progress + (e.g. polling).} + +The @srcrefjs{server/request,UserRequest} exposes raw request data with + minor processing. + +@table @strong + @item Path (@jsmethod{getUri}) + The path component of the URI. The method name is unfortunate. + + @item Query data (@jsmethod{getGetData}) + Query string processed into a key/value object. + Despite the name, + this is also populated if non-GET requests contain query strings. + + @item POST data (@jsmethod{getPostData}) + POST data processed into an object as if it were a query string + (just as @jsmethod{getGetData}). + Since this requires data that is streamed asynchronously, + this method takes a callback that waits for all data to become + available; + if the data are already available, + it is immediately invoked with the processed POST data. + + @item Cookies (@jsmethod{getCookies}) + Cookies parsed into a key/value object. + + @item Remote address (@jsmethod{getRemoteAddr}) + IP address of the origin of the request. + If the server is behind a proxy that sets the + @samp{X-Forwarded-For} header, + it is used instead. + + @item Host address (@jsmethod{getHostAddr}) + Hostname of the server. + If the server is behind a proxy that sets the + @samp{X-Forwarded-Host} header, + it is used instead. + + @item Origin (@jsmethod{getOrigin}) + Origin of request. + Only available if at lease one of the @samp{Origin} or + @samp{Referer} headers are set. + This is useful mainly for determining the protocol and host while + behind a proxy. + + @item User agent (@jsmethod{getUserAgent}) + The user agent string of the request. + + @item Session ID (@jsmethod{getSessionId}) + The user's unique session id (@samp{PHPSESSID}). + + @item Session ID name (@jsmethod{getSessionIdName}) + The name of the cookie from which the session ID originated + (hard-coded to @samp{PHPSESSID}). +@end table + +@todo{Document return format and writing response data.} + + + +@node Posting Data +@section Posting Data +@cindex Post +@cindex Bucket diff +@cindex Step save +A diff of the bucket data (@pxref{Bucket Diff}) is posted to the + server on step@tie{}save. +This operation is performed asynchronously@mdash{ + }the client need not wait for the step to save before the next can + be requested. + +Since validations are shared between the server and the client + (@pxref{Validation}), + saving should only fail in exception situations. +Should a failure occur, + the server will instruct the client to kick the user back to the + previous step (@dfn{kickback}). + +A step cannot be saved if it is locked; + such attempts will result in an error. + +To prevent a user from skipping steps, + the client may post only one step past the last step that has + successfully saved; + otherwise, the user is kicked back to the last step that was saved. + +Once those basic checks have passed, + the document is updated: + +@enumerate + @item + @cindex Data sanitization + The diff is first @dfn{sanitized} to strip out unknown fields, + internal fields posted by non-internal users, + and to filter fields on permitted characters; + + @item + The sanitized diff is then applied to the existing bucket on the + document; + + @item + @cindex Calculated values, server-side + Calculated values marked for storage (@pxref{Calculated Values}) are + re-calculated on the server (the values posted by the client have + already been discarded by the first step in this list); + + @item + @cindex Premium calculation date + The last premium calculation date is cleared (indicating that + premiums are no longer valid);@footnote{ + This concept is tightly coupled with insurance; + it should be factored out at some point.} + + @item + @cindex Encryption + Data marked as sensitive is encrypted and the ciphertext written to + the bucket in place of the plaintext (@pxref{Encryption Service}); + + @item + @cindex Top visited step + The current step is incremented and the @dfn{top visited + step}@tie{} is set to the larger of the incremented step or the + existing top visited step id; and then + + @item + The new document state and bucket data are written to the database. +@end enumerate + + + +@node Encryption Service +@section Encryption Service +@helpwanted From 95c28d4db428aa298e90d1e95b48096630f66f77 Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Tue, 27 Jun 2017 10:49:45 -0400 Subject: [PATCH 13/17] Extract Server construction into DocumentServer * src/server/DocumentServer.js: Add facade. * src/server/daemon/controller.js: Use it. --- src/server/DocumentServer.js | 56 +++++++++++++++++++++++++++++++++ src/server/daemon/controller.js | 15 +++++---- 2 files changed, 63 insertions(+), 8 deletions(-) create mode 100644 src/server/DocumentServer.js diff --git a/src/server/DocumentServer.js b/src/server/DocumentServer.js new file mode 100644 index 0000000..c715606 --- /dev/null +++ b/src/server/DocumentServer.js @@ -0,0 +1,56 @@ +/** + * Facade for vanilla document server + * + * Copyright (C) 2017 R-T Specialty, LLC. + * + * This file is part of the Liza Data Collection Framework. + * + * liza is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +const { Class } = require( 'easejs' ); + +const { + bucket: { + bucket_filter, + }, + + dapi: { + DataApiFactory, + DataApiManager, + }, + + server: { + Server, + + request: { + DataProcessor, + JsonServerResponse, + }, + }, +} = require( '../..' ); + + +/** + * Vanilla document server + */ +module.exports = Class( 'DocumentServer', +{ + 'public create': ( dao, logger, enc_service, origin_url ) => Server( + new JsonServerResponse.create(), + dao, + logger, + enc_service + ), +} ); diff --git a/src/server/daemon/controller.js b/src/server/daemon/controller.js index d9579c7..2260f20 100644 --- a/src/server/daemon/controller.js +++ b/src/server/daemon/controller.js @@ -42,8 +42,13 @@ const { QuoteDataBucket, }, + dapi: { + DataApiFactory, + DataApiManager, + }, + server: { - Server, + DocumentServer, db: { MongoServerDao, @@ -70,7 +75,6 @@ const { request: { CapturedUserResponse, - JsonServerResponse, SessionSpoofHttpClient, UserResponse, }, @@ -106,12 +110,7 @@ exports.init = function( logger, enc_service ) var dao = MongoServerDao( db ); - server = Server( - new JsonServerResponse.create(), - dao, - logger, - enc_service - ); + server = DocumentServer().create( dao, logger, enc_service ); server_cache = _createCache( server ); server.init( server_cache, exports.rater ); From b3f8ed96dd08f330b53cfe0f55de8a027acffd33 Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Tue, 27 Jun 2017 10:52:46 -0400 Subject: [PATCH 14/17] Fix typo in RestrictedDataApi error * src/dapi/RestrictedDataApi.js (_requestParamCheck): Unknown param error typo fix. --- src/dapi/RestrictedDataApi.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dapi/RestrictedDataApi.js b/src/dapi/RestrictedDataApi.js index 73b8440..d4758cf 100644 --- a/src/dapi/RestrictedDataApi.js +++ b/src/dapi/RestrictedDataApi.js @@ -126,7 +126,7 @@ module.exports = Class( 'RestrictedDataApi' ) // fail on unknown params if ( !( this._params[ name ] ) ) { - throw Error( 'Unkown param: ' + name ); + throw Error( 'Unknown param: ' + name ); } } From d47d77bb5e89c6ff675fb2ddbf6da16105226d40 Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Wed, 28 Jun 2017 14:51:34 -0400 Subject: [PATCH 15/17] Add server.meta.DapiMetaSource Encapsulates the nasty. * src/server/meta/DapiMetaSource.js: Add class. * test/server/meta/DapiMetaSourceTest.js: Add test case. --- src/server/meta/DapiMetaSource.js | 109 ++++++++++++++++++ test/server/meta/DapiMetaSourceTest.js | 149 +++++++++++++++++++++++++ 2 files changed, 258 insertions(+) create mode 100644 src/server/meta/DapiMetaSource.js create mode 100644 test/server/meta/DapiMetaSourceTest.js diff --git a/src/server/meta/DapiMetaSource.js b/src/server/meta/DapiMetaSource.js new file mode 100644 index 0000000..fecfe2d --- /dev/null +++ b/src/server/meta/DapiMetaSource.js @@ -0,0 +1,109 @@ +/** + * Data-API-based metadata population + * + * Copyright (C) 2017 R-T Specialty, LLC. + * + * This file is part of the Liza Data Collection Framework. + * + * liza is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +"use strict"; + +const { Class } = require( 'easejs' ); + + +/** + * Retrieve data for meta field using Data API + * + * TODO: The reason this class exists at all is to encapsulate the horrid + * API. Once refactored, perhaps this class will no longer be necessary. + */ +module.exports = Class( 'DapiMetaSource', +{ + /** + * Metabucket constructor + * @type {function()} + */ + 'private _bucketf': null, + + + /** + * Initialize with metabucket constructor + * @type {function()} + */ + constructor( bucketf ) + { + this._bucketf = bucketf; + }, + + + /** + * Retrieve field data + * + * @param {string} field field name + * @param {number} index field index + * @param {DataApiManager} dapi_manager manager for dapi calls + * @param {Object} dapi dapi descriptor + * @param {Object} data dapi input data + * + * @return {Promise} object containing `field`, `index`, and return data + */ + 'public getFieldData'( field, index, dapi_manager, dapi, data ) + { + const metabucket = this._bucketf(); + + return new Promise( ( resolve, reject ) => + { + dapi_manager.getApiData( + dapi.name, + data, + ( err, api_data ) => + { + dapi_manager.setFieldData( + dapi.name, + index, + api_data, + dapi.value, + '', + false + ); + + dapi_manager.expandFieldData( + dapi.name, + index, + metabucket, + dapi.mapdest, + true, + { + [dapi.name]: { + [index]: api_data[ 0 ][ dapi.value ], + }, + } + ); + + resolve( { + field: field, + index: index, + data: metabucket.getData(), + } ); + }, + field, + index, + {}, + reject + ); + } ); + }, +} ); diff --git a/test/server/meta/DapiMetaSourceTest.js b/test/server/meta/DapiMetaSourceTest.js new file mode 100644 index 0000000..87ff860 --- /dev/null +++ b/test/server/meta/DapiMetaSourceTest.js @@ -0,0 +1,149 @@ +/** + * Tests Data-API-based metadata population + * + * Copyright (C) 2017 R-T Specialty, LLC. + * + * This file is part of the Liza Data Collection Framework. + * + * liza is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +"use strict"; + +const expect = require( 'chai' ).expect; +const Sut = require( '../../../' ).server.meta.DapiMetaSource; + +describe( "DapiMetaSource", () => +{ + it( "populates field with dapi response", () => + { + const dapim = createStubDapiManager(); + const field_name = 'field_foo'; + const index = 1; + + const dapi = { + name: 'dapi_name', + value: 'foo', + mapdest: { map: 'dest' }, + }; + + // input data to dapi + const given_data = {}; + + // dapi output data (response) + const ret_data = [ {} ]; + + const bucket_result = { + [dapi.name]: { + [index]: ret_data[ 0 ][ dapi.value ], + }, + }; + + const metabucket = getStubBucket(); + + // g prefix = "given" + // all these show why we want to encapsulate this garbage + dapim.getApiData = ( gapi, gdata, gcallback, gname, gindex ) => + { + expect( gapi ).to.equal( dapi.name ); + expect( gdata ).to.equal( given_data ); + expect( gname ).to.equal( field_name ); + expect( gindex ).to.equal( index ); + + // make sure we handle async + process.nextTick( () => gcallback( null, ret_data ) ); + }; + + dapim.setFieldData = + ( gname, gindex, gdata, gvalue, glabel, gunchanged ) => + { + expect( gname ).to.equal( dapi.name ); + expect( gindex ).to.equal( index ); + expect( gdata ).to.equal( ret_data ); + expect( gvalue ).to.equal( dapi.value ); + expect( glabel ).to.equal( '' ); // unused + expect( gunchanged ).to.equal( false ); + }; + + dapim.expandFieldData = + ( gname, gindex, gbucket, gmap, gpredictive, gdiff ) => + { + expect( gname ).to.equal( dapi.name ); + expect( gindex ).to.equal( index ); + expect( gbucket ).to.equal( metabucket ); + expect( gmap ).to.equal( dapi.mapdest ); + expect( gpredictive ).to.equal( true ); + expect( gdiff ).to.deep.equal( bucket_result ); + + metabucket.getData = () => bucket_result; + }; + + return Sut( () => metabucket ) + .getFieldData( field_name, index, dapim, dapi, given_data ) + .then( result => + { + expect( result.field ).to.equal( field_name ); + expect( result.index ).to.equal( index ); + expect( result.data ).to.equal( bucket_result ); + } ); + } ); + + + it( "rejects promise on error", () => + { + const e = Error( "Test error" ); + const dapim = createStubDapiManager(); + + dapim.getApiData = ( _, __, ___, ____, _____, ______, failc ) => + { + failc( e ); + }; + + return Sut( () => getStubBucket() ) + .getFieldData( 'name', 0, dapim, {}, {} ) + .catch( given_e => + { + expect( given_e ).to.equal( e ); + + return true; + } ); + } ); +} ); + + +function createStubDapiManager() +{ + return { + getApiData() {}, + setFieldData() {}, + expandFieldData() {}, + }; +} + + +function getStubBucket() +{ + return { + setValues() {}, + getData() {}, + }; +} + + +function createStubDb() +{ + return { + saveQuoteMeta() {}, + }; +} From 65ab92f7019bbe469345ceeed9a771350c1088f2 Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Wed, 28 Jun 2017 14:54:34 -0400 Subject: [PATCH 16/17] Add SpoofedNodeHttpImpl Session spoofing is needed for making authenticated requests. --- src/dapi/DataApiManager.js | 3 +- src/dapi/http/HttpImpl.js | 6 +- src/dapi/http/NodeHttpImpl.js | 7 +- src/dapi/http/SpoofedNodeHttpImpl.js | 78 +++++++++++++++++++++ src/dapi/http/XhrHttpImpl.js | 6 ++ test/dapi/http/HttpDataApiTest.js | 10 ++- test/dapi/http/SpoofedNodeHttpImplTest.js | 84 +++++++++++++++++++++++ 7 files changed, 188 insertions(+), 6 deletions(-) create mode 100644 src/dapi/http/SpoofedNodeHttpImpl.js create mode 100644 test/dapi/http/SpoofedNodeHttpImplTest.js diff --git a/src/dapi/DataApiManager.js b/src/dapi/DataApiManager.js index 0ebc862..f67960c 100644 --- a/src/dapi/DataApiManager.js +++ b/src/dapi/DataApiManager.js @@ -98,9 +98,10 @@ module.exports = Class( 'DataApiManager' ) 'private _apis': {}, - __construct: function( api_factory ) + __construct: function( api_factory, apis ) { this._dataApiFactory = api_factory; + this.setApis( apis || {} ); }, diff --git a/src/dapi/http/HttpImpl.js b/src/dapi/http/HttpImpl.js index a9bee06..62d0cf2 100644 --- a/src/dapi/http/HttpImpl.js +++ b/src/dapi/http/HttpImpl.js @@ -47,5 +47,9 @@ module.exports = Interface( 'HttpImpl', * * @return {HttpImpl} self */ - 'public requestData': [ 'url', 'method', 'data', 'callback' ] + 'public requestData': [ 'url', 'method', 'data', 'callback' ], + + // TODO: temporary to work around class extension bug; see + // SpoofedNodeHttpImpl + 'public setOptions': [], } ); diff --git a/src/dapi/http/NodeHttpImpl.js b/src/dapi/http/NodeHttpImpl.js index 541b332..35e687d 100644 --- a/src/dapi/http/NodeHttpImpl.js +++ b/src/dapi/http/NodeHttpImpl.js @@ -98,7 +98,7 @@ module.exports = Class( 'NodeHttpImpl' ) throw Error( `No handler for ${protocol}` ); } - this._setOptions( options, method, data ); + this.setOptions( options, method, data ); let forbid_end = false; @@ -152,13 +152,16 @@ module.exports = Class( 'NodeHttpImpl' ) /** * Set request options * + * TODO: public to work around a class extension trait bug; make + * protected once fixed + * * @param {Object} options request options * @param {string} method HTTP method * @param {string} data request data * * @return {Object} request headers */ - 'private _setOptions'( options, method, data ) + 'virtual public setOptions'( options, method, data ) { const { headers = {} } = options; diff --git a/src/dapi/http/SpoofedNodeHttpImpl.js b/src/dapi/http/SpoofedNodeHttpImpl.js new file mode 100644 index 0000000..ac68812 --- /dev/null +++ b/src/dapi/http/SpoofedNodeHttpImpl.js @@ -0,0 +1,78 @@ +/** + * Node-based HTTP client with session spoofing + * + * Copyright (C) 2017 R-T Specialty, LLC. + * + * This file is part of the Liza Data Collection Framework. + * + * liza is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +'use strict'; + +const { Trait } = require( 'easejs' ); +const HttpImpl = require( './HttpImpl' ); + + +/** + * Spoof user session during request + * + * TODO: Implementing HttpImpl instead of overriding NodeHttpImpl to work + * around a class extension bug; change once fixed. + */ +module.exports = Trait( 'SpoofedNodeHttpImpl' ) + .implement( HttpImpl ) + .extend( +{ + /** + * Session to spoof + * @type {UserSession} + */ + 'private _request': null, + + + /** + * Use session for spoofing requests + * + * @param {UserSession} session session to spoof + */ + __mixin( session ) + { + this._request = session; + }, + + + /** + * Set request options to spoof session + * + * @param {Object} options request options + * @param {string} method HTTP method + * @param {string} data request data + * + * @return {Object} request headers + */ + 'virtual abstract override public setOptions'( options, method, data ) + { + const cookie = this._request.getSessionIdName() + '=' + + this._request.getSessionId(); + + options.headers = { + 'User-Agent': this._request.getUserAgent(), + 'X-Forwarded-For': this._request.getRemoteAddr(), + 'Cookie': cookie, + }; + + return this.__super( options, method, data ); + } +} ); diff --git a/src/dapi/http/XhrHttpImpl.js b/src/dapi/http/XhrHttpImpl.js index 20e0f19..28eccac 100644 --- a/src/dapi/http/XhrHttpImpl.js +++ b/src/dapi/http/XhrHttpImpl.js @@ -279,5 +279,11 @@ module.exports = Class( 'XhrHttpImpl' ) e.status = req.status; callback( e, req.responseText ); + }, + + + 'public setOptions'() + { + // TOOD: remove (see HttpImpl) } } ); diff --git a/test/dapi/http/HttpDataApiTest.js b/test/dapi/http/HttpDataApiTest.js index 12e8d82..e402ae4 100644 --- a/test/dapi/http/HttpDataApiTest.js +++ b/test/dapi/http/HttpDataApiTest.js @@ -27,7 +27,11 @@ var dapi = require( '../../../' ).dapi, dummy_url = 'http://foo', dummy_impl = Class .implement( dapi.http.HttpImpl ) - .extend( { requestData: function( _, __, ___, ____ ) {} } )(), + .extend( + { + requestData: function( _, __, ___, ____ ) {}, + setOptions() {}, + } )(), dummy_sut = Sut( dummy_url, 'GET', dummy_impl ); @@ -86,7 +90,9 @@ describe( 'HttpDataApi', function() { this.provided = arguments; c( this.err, this.data ); - } + }, + + setOptions() {}, } )(); diff --git a/test/dapi/http/SpoofedNodeHttpImplTest.js b/test/dapi/http/SpoofedNodeHttpImplTest.js new file mode 100644 index 0000000..adc8039 --- /dev/null +++ b/test/dapi/http/SpoofedNodeHttpImplTest.js @@ -0,0 +1,84 @@ +/** + * Tests Node-based HTTP client with session spoofing + * + * Copyright (C) 2017 R-T Specialty, LLC. + * + * This file is part of the Liza Data Collection Framework. + * + * liza is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +'use strict'; + +const { expect } = require( 'chai' ); +const { + SpoofedNodeHttpImpl: Sut, + NodeHttpImpl, +} = require( '../../../' ).dapi.http; + + +describe( 'SpoofNodeHttpImpl', () => +{ + it( "adds session headers", done => + { + const user_agent = 'Agent Foo'; + const forward_for = '::1'; + const sessname = 'FOOSESSID'; + const sessid = '12345'; + + const protos = { + http: { + request( given ) + { + expect( given.headers[ 'User-Agent' ] ) + .to.equal( user_agent ); + expect( given.headers[ 'X-Forwarded-For' ] ) + .to.equal( forward_for ); + + expect( given.headers.Cookie ) + .to.contain( sessname + '=' + sessid ); + + done(); + }, + }, + }; + + const url = { + parse: () => ( { + protocol: 'http', + } ) + }; + + const session = getStubSession( { + agent: user_agent, + forward_for: forward_for, + sessname: sessname, + sessid: sessid, + } ); + + const given = NodeHttpImpl.use( Sut( session ) )( protos, url ) + .requestData( '', '', {}, ()=>{} ); + } ); +} ); + + +function getStubSession( { agent, forward_for, sessname, sessid } ) +{ + return { + getUserAgent: () => agent, + getRemoteAddr: () => forward_for, + getSessionIdName: () => sessname, + getSessionId: () => sessid, + }; +} From 0c24e3d2807a3d5aa5f9c768f449c6ba2b7fc6dd Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Wed, 28 Jun 2017 16:12:08 -0400 Subject: [PATCH 17/17] Populate document metadata using Data APIs What a cluster. This was a lot of work to work around existing, bad APIs; there is no time to refactor at the moment; this already took much longer than expected. --- doc/macros.texi | 4 + doc/program.texi | 60 ++++ doc/server.texi | 43 ++- src/dapi/DataApiFactory.js | 21 +- src/server/DocumentServer.js | 21 +- src/server/Server.js | 90 +++--- src/server/daemon/controller.js | 24 +- src/server/db/MongoServerDao.js | 32 +++ src/server/meta/DapiMetaSource.js | 7 + src/server/request/DataProcessor.js | 260 +++++++++++++++++ src/server/request/ServerDataApiFactory.js | 69 +++++ src/server/request/UserRequest.js | 16 ++ test/server/meta/DapiMetaSourceTest.js | 27 +- test/server/request/DataProcessorTest.js | 307 +++++++++++++++++++++ 14 files changed, 918 insertions(+), 63 deletions(-) create mode 100644 src/server/request/DataProcessor.js create mode 100644 src/server/request/ServerDataApiFactory.js create mode 100644 test/server/request/DataProcessorTest.js diff --git a/doc/macros.texi b/doc/macros.texi index 7340491..e5a283c 100644 --- a/doc/macros.texi +++ b/doc/macros.texi @@ -203,6 +203,10 @@ Program@tie{}XML Data@tie{}API @end macro +@macro dapiref +@dapi (@pxref{Data API,,Data@tie{}API}) +@end macro + @c todo: link to reference directly @macro proguicref{ref} `\ref\' @proguicrefsuffix diff --git a/doc/program.texi b/doc/program.texi index 76d6482..94a1e12 100644 --- a/doc/program.texi +++ b/doc/program.texi @@ -40,8 +40,11 @@ Programs are ideally compiled from a @ref{Program XML,,Program@tie{}XML} @menu * Program UI:: * Program XML:: +* Document Metadata:: Document-level data that cannot be modified by + the client. @end menu + @node Program UI @section Program UI @maintenance{ @@ -220,3 +223,60 @@ Within the context of the @progxml, It reads as a sentence: ``@samp{vacant_desc}'' is applicable when we should @tie{}``describe a vacant property''. + + + +@node Document Metadata +@section Document Metadata +@dfn{Document metadata} are metadata that describe certain aspects of the document; + they are stored adjacent to the bucket in @samp{meta}@tie{}on the + document root.@footnote{ + Terminology note: ``document'' and ``quote'' are the same thing; + the latter is transitioning to the former for generality.} +They should be used in place of a bucket field any time + the client has no business knowing about the data. +The @samp{meta} record is called the @dfn{Metabucket}. + +@c don't use a dapi xref here; don't want to confuse the reader by +@c directing them away from this section before they continue reading +@tip{Metadata in the Metabucket should@tie{}@emph{not} be + directly populated by external systems@mdash{ + }@dapi integration should be used instead (see below).} + +Metadata can be populated using any@tie{}@dapiref@mdash{ + }return data populate the Metabucket in the same way that they + populate the Bucket. +Definitions are stored in @code{meta.fields}, + as shown in @ref{f:meta-fields}. + +@float Figure, f:meta-fields +@example +"fields":@{ + ["string(name)": @{ + "desc": "string", + "dapi": @{ + "name": "string", + "map": @{ + "string(dest field)": "string(source field)" + @} + @} + @} +@} +@end example +@caption{Format of @code{meta.fields}.} +@end float + +Further, a key-value mapping of all bucket fields that@mdash{ + }when modified, + need to result in a metadata API@tie{}call@mdash{ + }are stored in the @code{mapis}@tie{}object; + this is shown in @ref{f:mapis}. + +@float Figure, f:mapis +@example +"mapis":@{ + ["string(field name)"]: [ "string(dapi name)", ... ] +@} +@end example +@caption{Format of @code{mapis}.} +@end float diff --git a/doc/server.texi b/doc/server.texi index 752f74a..ca9d777 100644 --- a/doc/server.texi +++ b/doc/server.texi @@ -56,9 +56,10 @@ The HTTP server is managed by @menu -* Requests:: Handling HTTP requests -* Posting Data:: Handling step saves and other posts. -* Encryption Service:: Managing sensitive data. +* Requests:: Handling HTTP requests. +* Posting Data:: Handling step saves and other posts. +* Server-Side Data API Calls:: Accessing external resources on the server. +* Encryption Service:: Managing sensitive data. @end menu @@ -202,6 +203,11 @@ Once those basic checks have passed, re-calculated on the server (the values posted by the client have already been discarded by the first step in this list); + @item + Server-side @dapi{} calls (@pxref{Data API}) are triggered using the + diff as input data and an empty bucket for response storage + (@pxref{Server-Side Data API Calls}); + @item @cindex Premium calculation date The last premium calculation date is cleared (indicating that @@ -226,6 +232,37 @@ Once those basic checks have passed, +@node Server-Side Data API Calls +@section Server-Side Data API Calls +@maintenance{This makes use of @srcrefjs{server/meta,DapiMetaSource} + to encapsulate the horrible API of @srcrefjs{dapi,DataApiManager}; + the latter needs cleanup to remove the former.} + +@cindex Data API +@cindex Document metadata +Server-side @dapi{} calls (@pxref{Data API}) are triggered on + step save (@pxref{Posting Data}) and are handled much like they are + on the client. +Such calls are made automatically only for document metadata. +Results of sever-side calls are @emph{not} written to the bucket + and are therefore useful for data that the client should not be + permitted to modify; + it also allows data to be kept secret from the client.@footnote{ + All bucket data is served to the client, + with the exception of internal fields if the user is non-internal.} + +@dapi{} results on the client can be mapped back to multiple bucket values; + the server, however, has serious concerns with how data are + propagated for data integrity and security reasons. +Further, + document metadata can be structured, + unlike the Bucket which has a rigid matrix format (@pxref{Bucket}). +Therefore, + the entire response is mapped into the parent field; + defined return values are used only for filtering. + + + @node Encryption Service @section Encryption Service @helpwanted diff --git a/src/dapi/DataApiFactory.js b/src/dapi/DataApiFactory.js index 650e52d..446df54 100644 --- a/src/dapi/DataApiFactory.js +++ b/src/dapi/DataApiFactory.js @@ -22,7 +22,6 @@ const Class = require( 'easejs' ).Class; const HttpDataApi = require( './http/HttpDataApi' ); const XhrHttpImpl = require( './http/XhrHttpImpl' ); -const NodeHttpImpl = require( './http/NodeHttpImpl' ); const JsonResponse = require( './format/JsonResponse' ); const RestrictedDataApi = require( './RestrictedDataApi' ); const StaticAdditionDataApi = require( './StaticAdditionDataApi' ); @@ -30,7 +29,7 @@ const BucketDataApi = require( './BucketDataApi' ); /** - * Instantiates the appropriate DataApi object for the givne service type + * Instantiates the appropriate DataApi object for the given service type */ module.exports = Class( 'DataApiFactory', { @@ -58,15 +57,7 @@ module.exports = Class( 'DataApiFactory', switch ( type ) { case 'rest': - const impl = ( typeof XMLHttpRequest !== 'undefined' ) - ? XhrHttpImpl( XMLHttpRequest ) - : NodeHttpImpl( - { - http: require( 'http' ), - https: require( 'https' ), - }, - require( 'url' ) - ); + const impl = this.createHttpImpl(); api = HttpDataApi.use( JsonResponse )( source, @@ -93,6 +84,12 @@ module.exports = Class( 'DataApiFactory', StaticAdditionDataApi( api, nonempty, multiple, static_data ), desc ); - } + }, + + + 'virtual protected createHttpImpl'() + { + return XhrHttpImpl( XMLHttpRequest ); + }, } ); diff --git a/src/server/DocumentServer.js b/src/server/DocumentServer.js index c715606..b07bf8e 100644 --- a/src/server/DocumentServer.js +++ b/src/server/DocumentServer.js @@ -24,19 +24,24 @@ const { Class } = require( 'easejs' ); const { bucket: { bucket_filter, + QuoteDataBucket, }, dapi: { - DataApiFactory, DataApiManager, }, server: { Server, + meta: { + DapiMetaSource, + }, + request: { DataProcessor, JsonServerResponse, + ServerDataApiFactory, }, }, } = require( '../..' ); @@ -51,6 +56,18 @@ module.exports = Class( 'DocumentServer', new JsonServerResponse.create(), dao, logger, - enc_service + enc_service, + + DataProcessor( + bucket_filter, + ( apis, request ) => DataApiManager( + ServerDataApiFactory( + origin_url || request.getOrigin(), + request + ), + apis + ), + DapiMetaSource( QuoteDataBucket ) + ) ), } ); diff --git a/src/server/Server.js b/src/server/Server.js index 7411b01..09b1d10 100644 --- a/src/server/Server.js +++ b/src/server/Server.js @@ -48,6 +48,9 @@ const { }, server: { + request: { + DataProcessor, + }, encsvc: { QuoteDataBucketCipher, }, @@ -109,13 +112,27 @@ module.exports = Class( 'Server' ) */ 'private _cache': null, + /** + * Client-provided data processor + * @type {DataProcessor} + */ + 'private _dataProcessor': null, - 'public __construct': function( response, dao, logger, encsvc ) + + 'public __construct': function( + response, dao, logger, encsvc, data_processor + ) { - this.response = response; - this.dao = dao; - this.logger = logger; - this._encService = encsvc; + if ( !Class.isA( DataProcessor, data_processor ) ) + { + throw TypeError( "Expected DataProcessor" ); + } + + this.response = response; + this.dao = dao; + this.logger = logger; + this._encService = encsvc; + this._dataProcessor = data_processor; }, @@ -1115,14 +1132,19 @@ module.exports = Class( 'Server' ) { try { - var filtered = server._sanitizeBucketData( - post_data.data, request, program + var parsed_data = JSON.parse( post_data.data ); + var bucket = quote.getBucket(); + + const { filtered, dapis } = server._dataProcessor.processDiff( + parsed_data, request, program, bucket ); quote.setData( filtered ); + server._monitorMetadataPromise( quote, dapis ); + // calculated values (store only) - program.initQuote( quote.getBucket(), true ); + program.initQuote( bucket, true ); } catch ( err ) { @@ -1150,33 +1172,27 @@ module.exports = Class( 'Server' ) }, - /** - * Sanitize the given bucket data - * - * Ensures that we are storing only "correct" data within our database. This - * also strips any unknown bucket values, preventing users from using us as - * their own personal database. - */ - 'private _sanitizeBucketData': function( - bucket_data, request, program, permit_null - ) + 'private _monitorMetadataPromise'( quote, dapis ) { - var data = JSON.parse( bucket_data ), - types = program.meta.qtypes, - ignore = {}; - - // if we're not internal, filter out the internal questions - // (so they can't post to them) - if ( request.getSession().isInternal() === false ) - { - for ( id in program.internal ) - { - ignore[ id ] = true; - } - } - - // return the filtered data - return bucket_filter.filter( data, types, ignore, permit_null ); + dapis.map( promise => promise + .then( ( { field, index, data } ) => + this.dao.saveQuoteMeta( + quote, + data, + null, + e => { throw e; } + ) + ) + .catch( e => + server.logger.log( + server.logger.PRIORITY_ERROR, + "Failed to save field %s[%s] metadata: %s", + field, + index, + e.message + ) + ) + ); }, @@ -1619,8 +1635,10 @@ module.exports = Class( 'Server' ) // sanitize, permitting nulls (since the diff will have them) try { - var filtered = _self._sanitizeBucketData( - post_data.data, request, program, true + var data = JSON.parse( post_data.data ); + + var filtered = _self._dataProcessor.sanitizeDiff( + data, request, program, true ); } catch ( e ) diff --git a/src/server/daemon/controller.js b/src/server/daemon/controller.js index 2260f20..0813e2c 100644 --- a/src/server/daemon/controller.js +++ b/src/server/daemon/controller.js @@ -108,9 +108,8 @@ exports.init = function( logger, enc_service ) { native_parser: false, safe: false } ); - var dao = MongoServerDao( db ); - - server = DocumentServer().create( dao, logger, enc_service ); + const dao = MongoServerDao( db ); + server = _createDocumentServer( dao, logger, enc_service ); server_cache = _createCache( server ); server.init( server_cache, exports.rater ); @@ -149,6 +148,25 @@ exports.init = function( logger, enc_service ) } +function _createDocumentServer( dao, logger, enc_service ) +{ + const origin_url = process.env.HTTP_ORIGIN_URL || ''; + + if ( !origin_url ) + { + // this allows the system to work without configuration (e.g. for + // local development), but is really bad + logger.log( logger.PRIORITY_IMPORTANT, + "*** HTTP_ORIGIN_URL environment variable not set; " + + "system will fall back to using the origin of HTTP requests, " + + "meaning an attacker can control where server-side requests go! ***" + ); + } + + return DocumentServer().create( dao, logger, enc_service, origin_url ); +} + + function _initExportService( db, callback ) { db.collection( 'quotes', function( err, collection ) diff --git a/src/server/db/MongoServerDao.js b/src/server/db/MongoServerDao.js index 35d2234..8c3a9be 100644 --- a/src/server/db/MongoServerDao.js +++ b/src/server/db/MongoServerDao.js @@ -521,6 +521,38 @@ module.exports = Class( 'MongoServerDao' ) }, + /** + * Save document metadata (meta field on document) + * + * Only the provided indexes will be modified (that is---data will be + * merged with what is already in the database). + * + * @param {Quote} quote destination quote + * @param {Object} new_meta bucket-formatted data to write + * @param {Function} success callback on success + * @param {Function} failure callback on error + * + * @return {undefined} + */ + 'public saveQuoteMeta'( quote, new_meta, success, failure ) + { + const update = {}; + + for ( var key in new_meta ) + { + var meta = new_meta[ key ]; + + for ( var i in meta ) + { + update[ 'meta.' + key + '.' + i ] = + new_meta[ key ][ i ]; + } + } + + this.mergeData( quote, update, success, failure ); + }, + + /** * Saves the quote lock state to the database * diff --git a/src/server/meta/DapiMetaSource.js b/src/server/meta/DapiMetaSource.js index fecfe2d..1bc8c39 100644 --- a/src/server/meta/DapiMetaSource.js +++ b/src/server/meta/DapiMetaSource.js @@ -71,6 +71,13 @@ module.exports = Class( 'DapiMetaSource', data, ( err, api_data ) => { + if ( api_data.length > 1 ) + { + reject( Error( + "Data API request produced more than one result" + ) ); + } + dapi_manager.setFieldData( dapi.name, index, diff --git a/src/server/request/DataProcessor.js b/src/server/request/DataProcessor.js new file mode 100644 index 0000000..26e55f3 --- /dev/null +++ b/src/server/request/DataProcessor.js @@ -0,0 +1,260 @@ +/** + * Manages DataAPI requests and return data + * + * Copyright (C) 2017 R-T Specialty, LLC. + * + * This file is part of the Liza Data Collection Framework. + * + * liza is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +'use strict'; + +const { Class } = require( 'easejs' ); + +const { QuoteDataBucket } = require( '../../' ).bucket; + + +/** + * Process data provided by the client + * + * TOOD: This contains Data API and bucket merging logic that is better done + * elsewhere. + */ +module.exports = Class( 'DataProcessor', +{ + /** + * Bucket filter + * @type {Object} + */ + 'private _filter': null, + + /** + * Construct Data API manager + * @type {function()} + */ + 'private _dapif': null, + + /** + * Metadata source + * @type {DapiMetaSource} + */ + 'private _metaSource': null, + + + /** + * Initialize processor + * + * @type {Object} filter bucket filter + * @type {function()} dapif data API constructor + * @type {DapiMetaSource} meta_source metadata source + */ + constructor( filter, dapif, meta_source ) + { + this._filter = filter; + this._dapif = dapif; + this._metaSource = meta_source; + }, + + + /** + * Process client-provided data diff + * + * This performs sanitization to ensure that we are storing only + * "correct" data within our database. This also strips any unknown + * bucket values, preventing users from using us as their own personal + * database. + * + * @param {Object} data bucket diff data + * @param {UserRequest} request submitting request + * @param {Program} program active program + * + * @return {Object} processed diff + */ + 'public processDiff'( data, request, program, bucket ) + { + const filtered = this.sanitizeDiff( data, request, program, false ); + const dapi_manager = this._dapif( program.apis, request ); + + // array of promises for any dapi requests + const dapis = this._triggerDapis( + dapi_manager, program, data, bucket + ); + + return { + filtered: filtered, + dapis: dapis, + }; + }, + + + /** + * Sanitize client-provided data + * + * Internal fields will be stripped if the session is not + * internal. Following that, the filter provided via the ctor will be + * applied. + * + * `permit_null` should be used only in the case of bucket diffs, which + * contain nulls as terminators. + * + * @param {Object} data client-provided data + * @param {UserRequest} request client request + * @param {Program} program active program + * @param {boolean} permit_null whether null values should be retained + * + * @return {Object} filtered data + */ + 'public sanitizeDiff'( data, request, program, permit_null ) + { + permit_null = ( permit_null === undefined ) ? false : permit_null; + + if ( !request.getSession().isInternal() ) + { + this._cleanInternals( data, program ); + } + + const types = program.meta.qtypes; + return this._filter.filter( data, types, {}, permit_null ); + }, + + + /** + * Strip internal fields from diff `data` + * + * Internal fields are defined by the program `program`. + * + * @param {Object} data bucket diff data + * @param {Program} program active program + * + * @return {undefined} + */ + 'private _cleanInternals'( data, program ) + { + for ( let id in program.internal ) + { + delete data[ id ]; + } + }, + + + /** + * Trigger metadata Data API requests + * + * @param {DataApiManager} dapi_manager dapi manager + * @param {Program} program active program + * @param {Object} data client-provided data + * @param {Bucket} bucket active bucket + * + * @return {undefined} + */ + 'private _triggerDapis'( dapi_manager, program, data, bucket ) + { + const { + mapis = {}, + meta: { + fields = {}, + }, + } = program; + + const dapi_fields = this._determineDapiFields( mapis, data ); + + return Object.keys( dapi_fields ).map( field => + { + const { dapi } = fields[ field ]; + const indexes = dapi_fields[ field ]; + + return indexes.map( i => + this._metaSource.getFieldData( + field, + i, + dapi_manager, + dapi, + this._mapDapiData( dapi, bucket, i, data ) + ) + ); + } ).reduce( ( result, x ) => result.concat( x ), [] ); + }, + + + /** + * Determine which fields require a Data API to be triggered + * + * @param {Object} mapis metadata dapi descriptors + * @param {Object} data client-provided data + * + * @return {Object} fields with indexes in need of dapi calls + */ + 'private _determineDapiFields'( mapis, data ) + { + return Object.keys( mapis ).reduce( + ( result, src_field ) => + { + if ( data[ src_field ] === undefined ) + { + return result; + } + + const fields = mapis[ src_field ]; + + // get each index that changed + fields.forEach( field => + { + result[ field ] = result[ field ] || []; + + Object.keys( data[ src_field ] ).forEach( i => + { + if ( data[ src_field ][ i ] !== undefined ) + { + result[ field ][ i ] = i; + } + } ); + } ); + + return result; + }, + {} + ); + }, + + + /** + * Map data from bucket to dapi inputs + * + * @param {Object} dapi Data API descriptor + * @param {Bucket} bucket active (source) bucket + * @param {number} index field index + * @param {Object} diff_data client-provided data + * + * @return {Object} key/value dapi input data + */ + 'private _mapDapiData'( dapi, bucket, index, diff_data ) + { + const { mapsrc } = dapi; + + return Object.keys( mapsrc ).reduce( + ( result, srcid ) => + { + const bucketid = mapsrc[ srcid ]; + + const bdata = ( diff_data[ bucketid ] || [] )[ index ] || + ( bucket.getDataByName( bucketid ) || [] )[ index ]; + + result[ srcid ] = bdata || []; + return result; + }, + {} + ); + }, +} ); diff --git a/src/server/request/ServerDataApiFactory.js b/src/server/request/ServerDataApiFactory.js new file mode 100644 index 0000000..830b926 --- /dev/null +++ b/src/server/request/ServerDataApiFactory.js @@ -0,0 +1,69 @@ +/** + * Instantiate appropriate DataApi + * + * Copyright (C) 2017 R-T Specialty, LLC. + * + * This file is part of the Liza Data Collection Framework. + * + * liza is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +const { Class } = require( 'easejs' ); +const { + DataApiFactory, + http: { + NodeHttpImpl, + SpoofedNodeHttpImpl, + }, +} = require( '../..' ).dapi; + + +/** + * Instantiates the appropriate DataApi object for the given service type + */ +module.exports = Class( 'ServerDataApiFactory' ) + .extend( DataApiFactory, +{ + /** + * Origin URL + * @type {string} + */ + 'private _origin': '', + + /** + * Request on behalf of user session + * @type {UserSession} + */ + 'private _session': null, + + + constructor( origin, session ) + { + this._origin = ''+origin; + this._session = session; + }, + + + 'override protected createHttpImpl'() + { + return NodeHttpImpl.use( SpoofedNodeHttpImpl( this._session ) )( + { + http: require( 'http' ), + https: require( 'https' ), + }, + require( 'url' ), + this._origin + ); + }, +} ); diff --git a/src/server/request/UserRequest.js b/src/server/request/UserRequest.js index cd5cdfb..3e681a1 100644 --- a/src/server/request/UserRequest.js +++ b/src/server/request/UserRequest.js @@ -551,6 +551,22 @@ module.exports = Class.extend( require( 'events' ).EventEmitter, }, + 'public getHostAddr': function() + { + return this.request.headers['x-forwarded-host'] + || this.request.headers.host; + }, + + + 'public getOrigin': function() + { + const referrer = this.request.headers.referrer || ""; + + return this.request.headers.origin + || ( referrer.match( '^[a-z]+://[^/]+' ) || [] )[ 0 ]; + }, + + 'public getUserAgent': function() { return this.request.headers['user-agent']; diff --git a/test/server/meta/DapiMetaSourceTest.js b/test/server/meta/DapiMetaSourceTest.js index 87ff860..3791caf 100644 --- a/test/server/meta/DapiMetaSourceTest.js +++ b/test/server/meta/DapiMetaSourceTest.js @@ -110,14 +110,27 @@ describe( "DapiMetaSource", () => failc( e ); }; - return Sut( () => getStubBucket() ) - .getFieldData( 'name', 0, dapim, {}, {} ) - .catch( given_e => - { - expect( given_e ).to.equal( e ); + return expect( + Sut( () => getStubBucket() ) + .getFieldData( 'name', 0, dapim, {}, {} ) + ).to.eventually.be.rejectedWith( e ); + } ); - return true; - } ); + + it( "rejects if more than one result is returned from dapi", () => + { + const dapim = createStubDapiManager(); + + dapim.getApiData = ( _, __, callback ) => + { + // more than one result + callback( null, [ {}, {} ] ); + }; + + return expect( + Sut( () => getStubBucket() ) + .getFieldData( 'name', 0, dapim, {}, {} ) + ).to.eventually.be.rejectedWith( Error ); } ); } ); diff --git a/test/server/request/DataProcessorTest.js b/test/server/request/DataProcessorTest.js new file mode 100644 index 0000000..bd8f81b --- /dev/null +++ b/test/server/request/DataProcessorTest.js @@ -0,0 +1,307 @@ +/** + * Manages DataAPI requests and return data + * + * Copyright (C) 2017 R-T Specialty, LLC. + * + * This file is part of the Liza Data Collection Framework. + * + * liza is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +'use strict'; + +const { Class } = require( 'easejs' ); +const { expect } = require( 'chai' ); +const Sut = require( '../../../' ).server.request.DataProcessor; + + +describe( 'DataProcessor', () => +{ + [ + { + label: "strips internal field data when not internal", + data: { + internal: [ "foo", "bar" ], + foo: [ "bar", "baz" ], + }, + internals: { internal: true }, + internal: false, + expected: { + foo: [ "bar", "baz" ], + }, + }, + { + label: "keeps internal field data when internal", + data: { + internal: [ "foo", "bar" ], + foo: [ "bar", "baz" ], + }, + internals: { internal: true }, + internal: true, + expected: { + internal: [ "foo", "bar" ], + foo: [ "bar", "baz" ], + }, + }, + ].forEach( ( { label, internal, data, internals = {}, expected } ) => + { + const { request, program, sut } = + createSutFromStubs( internal, internals ); + + it( label, () => + { + expect( + sut.processDiff( data, request, program ).filtered + ).to.deep.equal( expected ); + } ); + } ); + + + it( "passes data to bucket filter", () => + { + const { request, program, meta_source } = createStubs(); + const data = {}; + const types = {}; + + program.meta.qtypes = types; + + const filter = { + filter( given_data, given_types, given_ignore, given_null ) + { + expect( given_data ).to.equal( data ); + expect( given_types ).to.equal( types ); + expect( given_null ).to.equal( false ); + + // not used + expect( given_ignore ).to.deep.equal( {} ); + + data.filtered = true; + } + }; + + Sut( filter, () => {}, meta_source ) + .processDiff( data, request, program ); + + expect( data.filtered ).to.equal( true ); + } ); + + + it( "instantiates dapi manager using program and session", done => + { + const { filter, request, program } = createStubs(); + + const dapi_factory = ( given_apis, given_request ) => + { + expect( given_apis ).to.equal( program.apis ); + expect( given_request ).to.equal( request ); + + done(); + } + + Sut( filter, dapi_factory ) + .processDiff( {}, request, program ); + } ); + + + it( "invokes dapi manager when monitored bucket value changes", () => + { + const triggered = {}; + + // g prefix = "given" + const getFieldData = function( gfield, gindex, gdapim, gdapi, gdata) + { + triggered[ gdapi.name ] = triggered[ gdapi.name ] || []; + triggered[ gdapi.name ][ gindex ] = arguments; + + return Promise.resolve( true ); + }; + + const dapi_manager = {}; + + const { + request, + program, + filter, + meta_source, + } = createStubs( false, {}, getFieldData ); + + const sut = Sut( filter, () => dapi_manager, meta_source ); + + program.meta.fields = { + foo: { + dapi: { + name: 'dapi_foo', + mapsrc: { ina: 'src', inb: 'src1' }, + }, + }, + bar: { + dapi: { + name: 'dapi_bar', + mapsrc: { ina: 'src1' }, + }, + }, + baz: { + dapi: { + name: 'dapi_no_call', + mapsrc: {}, + }, + }, + }; + + program.mapis = { + src: [ 'foo', 'bar' ], // change + src1: [ 'foo' ], // change + src2: [ 'baz' ], // do not change + }; + + // data changed + const data = { + src: [ 'src0', 'src1' ], + src1: [ undefined, 'src11' ], + }; + + const bucket = createStubBucket( { + src: [ 'bsrc0', 'bsrc1' ], + src1: [ 'bsrc10', 'bsrc11' ], + } ); + + const { dapis } = sut.processDiff( data, request, program, bucket ); + + const expected = { + dapi_foo: [ + { + name: 'foo', + data: { + ina: data.src[ 0 ], + inb: bucket.data.src1[ 0 ], + }, + }, + { + name: 'foo', + data: { + ina: data.src[ 1 ], + inb: data.src1[ 1 ], + }, + }, + ], + dapi_bar: [ + undefined, + { + name: 'bar', + data: { + ina: data.src1[ 1 ], + }, + }, + ], + }; + + for ( let dapi_name in expected ) + { + let expected_call = expected[ dapi_name ]; + + for ( let i in expected_call ) + { + let chk = expected_call[ i ]; + + if ( chk === undefined ) + { + continue; + } + + let [ gfield, gindex, gdapi_manager, gdapi, gdata ] = + triggered[ dapi_name ][ i ]; + + expect( gfield ).to.equal( chk.name ); + expect( gdapi.name ).to.equal( dapi_name ); + expect( +gindex ).to.equal( +i ); + expect( gdapi_manager ).to.equal( dapi_manager ); + + // see mapsrc + expect( gdata ).to.deep.equal( chk.data ); + } + } + + expect( triggered.dapi_no_call ).to.equal( undefined ); + + return Promise.all( dapis ); + } ); +} ); + + +function createSutFromStubs( /* see createStubs */ ) +{ + const { request, program, filter, meta_source } = + createStubs.apply( null, arguments ); + + return { + request: request, + program: program, + filter: filter, + meta_source: meta_source, + sut: Sut( filter, () => {}, meta_source ), + }; +} + + +function createStubs( internal, internals, getFieldData ) +{ + return { + request: createStubUserRequest( internal || false ), + program: createStubProgram( internals || {} ), + filter: { filter: _ => _ }, + meta_source: createStubDapiMetaSource( getFieldData ), + }; +} + + +function createStubUserRequest( internal ) +{ + return { + getSession: () => ( { + isInternal: () => internal + } ) + }; +} + + +function createStubProgram( internals ) +{ + return { + internal: internals, + meta: { qtypes: {}, fields: {} }, + apis: {}, + }; +} + + +function createStubDapiMetaSource( getFieldData ) +{ + return { + getFieldData: getFieldData || + function( field, index, dapi_manager, dapi, data ){}, + }; +} + + +function createStubBucket( data ) +{ + return { + data: data, + + getDataByName( name ) + { + return data[ name ]; + }, + }; +}