From e003abcd4bb72e9a4d5f93ad599ea81c1f4295fc Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Wed, 16 Oct 2019 14:36:54 -0400 Subject: [PATCH] Integrate TokenedDataApi into system This changes the easejs interface for DataApi, which requires adding the param to everything. The TS interface was created in a previous commit and already contained this parameter. The idea is to remove the easejs interface in the future, but traits are a barrier to that atm. DocumentServer and controller demonstrate the mess that we have with regards to instantiating dependencies. This needs to change---DocumentServer itself was something that was started but never fully realized. It makes this incredibly confusing, difficult to follow, and complicates important error handling that ought to be taking place. It also discourages implementing additional dependencies. I'm not going to go through and provide a ChangeLog-style commit message for this commit. I'm too exhausted by this crap. --- src/dapi/AutoRetry.js | 10 +- src/dapi/BucketDataApi.js | 2 +- src/dapi/DataApi.ts | 5 +- src/dapi/DataApiFactory.js | 4 +- src/dapi/DataApiManager.js | 2 +- src/dapi/QuoteDataApi.js | 2 +- src/dapi/RestDataApi.js | 2 +- src/dapi/RestrictedDataApi.js | 4 +- src/dapi/StaticAdditionDataApi.js | 11 +- src/dapi/format/JsonResponse.js | 4 +- src/dapi/format/ResponseApply.js | 4 +- src/dapi/http/HttpDataApi.js | 2 +- src/server/DocumentServer.js | 72 +++++++-- src/server/Server.js | 2 +- src/server/daemon/controller.js | 144 +++++++++--------- src/server/request/DataProcessor.js | 4 +- src/server/request/ServerDataApiFactory.js | 50 +++++- test/dapi/AutoRetryTest.js | 2 +- test/dapi/DummyDataApi.js | 4 +- test/dapi/format/JsonResponseTest.js | 2 +- .../service/RatingServiceSubmitNotifyTest.js | 2 +- 21 files changed, 211 insertions(+), 123 deletions(-) diff --git a/src/dapi/AutoRetry.js b/src/dapi/AutoRetry.js index e584398..994ed13 100644 --- a/src/dapi/AutoRetry.js +++ b/src/dapi/AutoRetry.js @@ -121,9 +121,9 @@ module.exports = Trait( 'AutoRetry' ) * * @return {DataApi} self */ - 'abstract override public request': function( input, callback ) + 'abstract override public request': function( input, callback, id ) { - this._try( input, callback, this._tries ); + this._try( input, callback, id, this._tries ); return this; }, @@ -141,7 +141,7 @@ module.exports = Trait( 'AutoRetry' ) * * @return {undefined} */ - 'private _try': function( input, callback, n ) + 'private _try': function( input, callback, id, n ) { var _self = this; @@ -178,10 +178,10 @@ module.exports = Trait( 'AutoRetry' ) ( n - 1 ), function() { - _self._try( input, callback, ( n - 1 ) ); + _self._try( input, callback, id, ( n - 1 ) ); }, complete ); - } ); + }, id ); } } ); diff --git a/src/dapi/BucketDataApi.js b/src/dapi/BucketDataApi.js index 1be1167..81dcff0 100644 --- a/src/dapi/BucketDataApi.js +++ b/src/dapi/BucketDataApi.js @@ -66,7 +66,7 @@ module.exports = Class( 'BucketDataApi' ) * * @return {BucketDataApi} self */ - 'public request': function( data, callback ) + 'public request': function( data, callback, id ) { var _self = this.__inst, rows = []; diff --git a/src/dapi/DataApi.ts b/src/dapi/DataApi.ts index 1246239..9bd2c40 100644 --- a/src/dapi/DataApi.ts +++ b/src/dapi/DataApi.ts @@ -63,7 +63,7 @@ export interface DataApi { request( data: DataApiInput, - callback: ( e: Error | null, data: DataApiResult | null ) => void, + callback: NodeCallback, id: string, ): this; } @@ -94,8 +94,9 @@ module.exports = Interface( 'DataApi', * * @param {?Object|string} data params or post data * @param {function(?Error,*):string} callback continuation upon reply + * @param {string} id unique dapi identifier * * @return {DataApi} self */ - 'public request': [ 'data', 'callback' ] + 'public request': [ 'data', 'callback', 'id' ] } ); diff --git a/src/dapi/DataApiFactory.js b/src/dapi/DataApiFactory.js index f84ea32..68ba270 100644 --- a/src/dapi/DataApiFactory.js +++ b/src/dapi/DataApiFactory.js @@ -58,7 +58,7 @@ module.exports = Class( 'DataApiFactory', const nonempty = !!descl.static_nonempty; const multiple = !!descl.static_multiple; - const api = this._createDataApi( type, descl, bucket ); + const api = this.createDataApi( type, descl, bucket ); return RestrictedDataApi( StaticAdditionDataApi( api, nonempty, multiple, static_data ), @@ -93,7 +93,7 @@ module.exports = Class( 'DataApiFactory', * * @return {DataApi} */ - 'private _createDataApi'( type, desc, bucket ) + 'virtual protected createDataApi'( type, desc, bucket ) { const source = ( desc.source || '' ); const method = ( desc.method || '' ); diff --git a/src/dapi/DataApiManager.js b/src/dapi/DataApiManager.js index c0b1fe4..7e5c7fe 100644 --- a/src/dapi/DataApiManager.js +++ b/src/dapi/DataApiManager.js @@ -202,7 +202,7 @@ module.exports = Class( 'DataApiManager' ) _self._pendingApiCall[ id ] = undefined; _self.emit( 'fieldLoaded', name, +index ); } - } ) ) + }, id ) ) .catch( e => fc( e ) ); }; diff --git a/src/dapi/QuoteDataApi.js b/src/dapi/QuoteDataApi.js index c5c8f3c..a8021fa 100644 --- a/src/dapi/QuoteDataApi.js +++ b/src/dapi/QuoteDataApi.js @@ -77,7 +77,7 @@ module.exports = Class( 'QuoteDataApi' ) * * @return {DataApi} self */ - 'public request'( data, callback ) + 'public request'( data, callback, id ) { this._dapi.request( this.mapData( data ), callback ); }, diff --git a/src/dapi/RestDataApi.js b/src/dapi/RestDataApi.js index 7f5e89a..f1e46cb 100644 --- a/src/dapi/RestDataApi.js +++ b/src/dapi/RestDataApi.js @@ -73,7 +73,7 @@ module.exports = Class( 'RestDataApi' ) * * @return {RestDataApi} self */ - 'public request': function( data, callback ) + 'public request': function( data, callback, id ) { var _self = this.__inst; diff --git a/src/dapi/RestrictedDataApi.js b/src/dapi/RestrictedDataApi.js index 452bc36..d2905a9 100644 --- a/src/dapi/RestrictedDataApi.js +++ b/src/dapi/RestrictedDataApi.js @@ -85,7 +85,7 @@ module.exports = Class( 'RestrictedDataApi' ) * * @return {DataApi} self */ - 'virtual public request': function( data, callback ) + 'virtual public request': function( data, callback, id ) { data = data || {}; callback = callback || function() {}; @@ -103,7 +103,7 @@ module.exports = Class( 'RestrictedDataApi' ) err, _self._checkResponse( response, callback ) ); - } ); + }, id ); }, diff --git a/src/dapi/StaticAdditionDataApi.js b/src/dapi/StaticAdditionDataApi.js index cb1fc19..5f6cfa0 100644 --- a/src/dapi/StaticAdditionDataApi.js +++ b/src/dapi/StaticAdditionDataApi.js @@ -66,13 +66,6 @@ module.exports = Class( 'StaticAdditionDataApi' ) */ __construct: function( data_api, nonempty, multiple, static_data ) { - if ( !( Class.isA( DataApi, data_api ) ) ) - { - throw Error( - 'Expected object of type DataApi; given: ' + data_api - ); - } - this._api = data_api; this._static = static_data; this._nonempty = !!nonempty; @@ -88,7 +81,7 @@ module.exports = Class( 'StaticAdditionDataApi' ) * * @return {DataApi} self */ - 'public request': function( data, callback ) + 'public request': function( data, callback, id ) { data = data || {}; callback = callback || function() {}; @@ -110,7 +103,7 @@ module.exports = Class( 'StaticAdditionDataApi' ) err, _self._unshiftData( response ) ); - } ); + }, id ); }, diff --git a/src/dapi/format/JsonResponse.js b/src/dapi/format/JsonResponse.js index e2b75de..f736206 100644 --- a/src/dapi/format/JsonResponse.js +++ b/src/dapi/format/JsonResponse.js @@ -47,14 +47,14 @@ module.exports = Trait( 'JsonResponse' ) * * @return {DataApi} self */ - 'virtual abstract override public request': function( data, callback ) + 'virtual abstract override public request': function( data, callback, id ) { var _self = this; this.__super( data, function( err, resp ) { _self._tryParse( err, resp, callback ); - } ); + }, id ); return this; }, diff --git a/src/dapi/format/ResponseApply.js b/src/dapi/format/ResponseApply.js index 1c39d7e..17d6225 100644 --- a/src/dapi/format/ResponseApply.js +++ b/src/dapi/format/ResponseApply.js @@ -65,12 +65,12 @@ module.exports = Trait( 'ResponseApply' ) * * @return {DataApi} self */ - 'virtual abstract override public request'( data, callback ) + 'virtual abstract override public request'( data, callback, id ) { this.__super( data, ( e, retdata ) => { callback( e, this._dataf( retdata ) ); - } ); + }, id ); return this; }, diff --git a/src/dapi/http/HttpDataApi.js b/src/dapi/http/HttpDataApi.js index 352b519..b99f551 100644 --- a/src/dapi/http/HttpDataApi.js +++ b/src/dapi/http/HttpDataApi.js @@ -128,7 +128,7 @@ module.exports = Class( 'HttpDataApi' ) * * @throws {TypeError} on validation failure */ - 'virtual public request': function( data, callback ) + 'virtual public request': function( data, callback, id ) { // null is a good indicator of "I have no intent to send any data"; // empty strings and objects are not, since those are valid data diff --git a/src/server/DocumentServer.js b/src/server/DocumentServer.js index 8370258..c8abb4a 100644 --- a/src/server/DocumentServer.js +++ b/src/server/DocumentServer.js @@ -48,17 +48,40 @@ const { JsonServerResponse, ServerDataApiFactory, }, + + token: { + MongoTokenDao: { MongoTokenDao }, + }, }, } = require( '../..' ); /** * Vanilla document server + * + * XXX: This is a mess, and it's only getting worse with dependencies + * instantiated everywhere. Everything should be instantiated in one place + * rather than part of them being passed in here. See controller.js. */ module.exports = Class( 'DocumentServer', { - 'public create': ( dao, logger, enc_service, origin_url, conf ) => - Promise.all( [ + /** + * Create document server + * + * See above XXX. + * + * @param {MongoServerDao} dao server DAO + * @param {Logger} logger log manager + * @param {EncryptionService} enc_service encryption service + * @param {string} origin_url HTTP_ORIGIN_URL + * @param {ConfStore} conf configuration store + * @param {MongoConnection} collection database collection + * + * @return {Promise} + */ + 'public create'( dao, logger, enc_service, origin_url, conf, collection ) + { + return Promise.all( [ conf.get( 'dapi' ), ] ).then( ([ dapi_conf ]) => Server( new JsonServerResponse.create(), @@ -68,17 +91,46 @@ module.exports = Class( 'DocumentServer', DataProcessor( bucket_filter, - ( apis, request ) => DataApiManager( - ServerDataApiFactory( - origin_url || request.getOrigin(), - request, - dapi_conf - ), - apis + ( apis, request, quote ) => this._createDapiManager( + apis, request, origin_url, dapi_conf, quote, collection ), DapiMetaSource( QuoteDataBucket ), StagingBucket ), ProgramInit() - ) ) + ) ); + }, + + + /** + * Create new DataApiManager + * + * See above XXX. + * + * @param {Object} apis API definitions + * @param {Request} request Node HTTP request + * @param {string} origin_url HTTP_ORIGIN_URL + * @param {Object} dapi_conf dapi configuration + * @param {Quote} quote current quote for request + * @param {MongoConnection} collection database collection + */ + 'private _createDapiManager'( + apis, request, origin_url, dapi_conf, quote, collection + ) + { + return DataApiManager( + ServerDataApiFactory( + origin_url || request.getOrigin(), + request, + dapi_conf, + quote.getId(), + new MongoTokenDao( + collection, + 'dapitok', + () => Math.floor( ( new Date() ).getTime() / 1000 ) + ) + ), + apis + ); + }, } ); diff --git a/src/server/Server.js b/src/server/Server.js index 6592faf..8a42ab0 100644 --- a/src/server/Server.js +++ b/src/server/Server.js @@ -1148,7 +1148,7 @@ module.exports = Class( 'Server' ) const { filtered, dapis, meta_clear } = server._dataProcessor.processDiff( - parsed_data, request, program, bucket + parsed_data, request, program, bucket, quote ); server._monitorMetadataPromise( quote, dapis, meta_clear ); diff --git a/src/server/daemon/controller.js b/src/server/daemon/controller.js index 18a2635..7e8ca54 100644 --- a/src/server/daemon/controller.js +++ b/src/server/daemon/controller.js @@ -127,47 +127,50 @@ exports.init = function( logger, enc_service, conf ) var db = _createDB( logger ); const dao = MongoServerDao( db ); - _createDocumentServer( dao, logger, enc_service, conf ).then( srv => + db.collection( 'quotes', function( err, collection ) { - server = srv; - - server_cache = _createCache( server ); - server.init( server_cache, exports.rater ); - - // TODO: temporary proof-of-concept - rating_service = RatingService.use( - RatingServicePublish( amqplib, exports.post_rate_publish, logger ) - )( - logger, dao, server, exports.rater - ); - - // TODO: exports.init needs to support callbacks; this will work, but - // only because it's unlikely that we'll get a request within - // milliseconds of coming online - _initExportService( db, function( service ) + _createDocumentServer( dao, logger, enc_service, conf, collection ).then( srv => { - c1_export_service = service; - } ); + server = srv; - server.on( 'quotePverUpdate', function( quote, program, event ) - { - // let them know that we're going to be a moment - var c = event.wait(); + server_cache = _createCache( server ); + server.init( server_cache, exports.rater ); - getCleaner( program ).clean( quote, function( err ) + // TODO: temporary proof-of-concept + rating_service = RatingService.use( + RatingServicePublish( amqplib, exports.post_rate_publish, logger ) + )( + logger, dao, server, exports.rater + ); + + // TODO: exports.init needs to support callbacks; this will work, but + // only because it's unlikely that we'll get a request within + // milliseconds of coming online + _initExportService( collection, function( service ) { - // report on our success/failure - if ( err ) - { - event.bad( err ); - } - else - { - event.good(); - } + c1_export_service = service; + } ); - // we're done - c(); + server.on( 'quotePverUpdate', function( quote, program, event ) + { + // let them know that we're going to be a moment + var c = event.wait(); + + getCleaner( program ).clean( quote, function( err ) + { + // report on our success/failure + if ( err ) + { + event.bad( err ); + } + else + { + event.good(); + } + + // we're done + c(); + } ); } ); } ); } ); @@ -203,7 +206,7 @@ function _createDB( logger ) return db; } -function _createDocumentServer( dao, logger, enc_service, conf ) +function _createDocumentServer( dao, logger, enc_service, conf, collection ) { const origin_url = process.env.HTTP_ORIGIN_URL || ''; @@ -219,52 +222,43 @@ function _createDocumentServer( dao, logger, enc_service, conf ) } return DocumentServer() - .create( dao, logger, enc_service, origin_url, conf ); + .create( dao, logger, enc_service, origin_url, conf, collection ); } -function _initExportService( db, callback ) +function _initExportService( collection, callback ) { - db.collection( 'quotes', function( err, collection ) - { - if ( collection === null ) - { - return; - } + var spoof_host = ( + ''+( + process.env.C1_EXPORT_HOST + || process.env.LV_RATE_DOMAIN + || process.env.LV_RATE_HOST + ).trim() + ); - var spoof_host = ( - ''+( - process.env.C1_EXPORT_HOST - || process.env.LV_RATE_DOMAIN - || process.env.LV_RATE_HOST - ).trim() - ); + var spoof = SessionSpoofHttpClient( http, spoof_host ); - var spoof = SessionSpoofHttpClient( http, spoof_host ); + callback( + ExportService + .use( TokenedService( + 'c1import', + new MongoTokenDao( collection, "exports", getUnixTimestamp ), + function tokgen() + { + var shasum = crypto.createHash( 'sha1' ); + shasum.update( ''+Math.random() ); - callback( - ExportService - .use( TokenedService( - 'c1import', - new MongoTokenDao( collection, "exports", getUnixTimestamp ), - function tokgen() - { - var shasum = crypto.createHash( 'sha1' ); - shasum.update( ''+Math.random() ); - - return shasum.digest( 'hex' ); - }, - function newcapturedResponse( request, callback ) - { - return UserResponse - .use( CapturedUserResponse( callback ) ) - ( request ); - } - ) ) - ( spoof ) - ); - - } ); + return shasum.digest( 'hex' ); + }, + function newcapturedResponse( request, callback ) + { + return UserResponse + .use( CapturedUserResponse( callback ) ) + ( request ); + } + ) ) + ( spoof ) + ); } diff --git a/src/server/request/DataProcessor.js b/src/server/request/DataProcessor.js index a746be3..3d9bbf6 100644 --- a/src/server/request/DataProcessor.js +++ b/src/server/request/DataProcessor.js @@ -87,10 +87,10 @@ module.exports = Class( 'DataProcessor', * * @return {Object} processed diff */ - 'public processDiff'( data, request, program, bucket ) + 'public processDiff'( data, request, program, bucket, quote ) { const filtered = this.sanitizeDiff( data, request, program ); - const dapi_manager = this._dapif( program.apis, request ); + const dapi_manager = this._dapif( program.apis, request, quote ); const staging = this._stagingCtor( bucket ); // forbidBypass will force diff generation on initQuote diff --git a/src/server/request/ServerDataApiFactory.js b/src/server/request/ServerDataApiFactory.js index 087d70e..522fb0c 100644 --- a/src/server/request/ServerDataApiFactory.js +++ b/src/server/request/ServerDataApiFactory.js @@ -20,6 +20,8 @@ */ const { Class } = require( 'easejs' ); +const crypto = require( 'crypto' ); + const { dapi: { DataApiFactory, @@ -31,6 +33,16 @@ const { store: { StoreMissError, }, + server: { + dapi: { + TokenedDataApi: { TokenedDataApi }, + }, + token: { + store: { + PersistentTokenStore: { PersistentTokenStore }, + }, + }, + }, } = require( '../..' ); @@ -58,12 +70,48 @@ module.exports = Class( 'ServerDataApiFactory' ) */ 'private _conf': null, + /** + * Document (quote) id + * @type {DocumentId} + */ + 'private _doc_id': 0, - constructor( origin, session, conf ) + + constructor( origin, session, conf, doc_id, tokdao ) { this._origin = ''+origin; this._session = session; this._conf = conf; + this._doc_id = doc_id; + this._tokdao = tokdao; + }, + + + 'override protected createDataApi'( type, desc, bucket ) + { + return new TokenedDataApi( + this.__super( type, desc, bucket ), + token_ns => new PersistentTokenStore( + this._tokdao, + this._doc_id, + token_ns, + () => this._generateTokenId() + ) + ); + }, + + + /** + * Generate random token identifier + * + * @return {string} unique token identifier + */ + 'private _generateTokenId'() + { + var shasum = crypto.createHash( 'sha1' ); + shasum.update( ''+Math.random() ); + + return shasum.digest( 'hex' ); }, diff --git a/test/dapi/AutoRetryTest.js b/test/dapi/AutoRetryTest.js index 424babe..c6d12b3 100644 --- a/test/dapi/AutoRetryTest.js +++ b/test/dapi/AutoRetryTest.js @@ -282,7 +282,7 @@ function _createStub( err, resp ) given: null, requests: 0, - 'virtual public request': function( data, callback ) + 'virtual public request': function( data, callback, id ) { this.given = data; this.requests++; diff --git a/test/dapi/DummyDataApi.js b/test/dapi/DummyDataApi.js index a33c48b..038b184 100644 --- a/test/dapi/DummyDataApi.js +++ b/test/dapi/DummyDataApi.js @@ -59,9 +59,9 @@ module.exports = Class( 'DummyDataApi' ) * * @return {DataApi} self */ - 'virtual public request'( data, callback ) + 'virtual public request'( data, callback, id ) { - this._reqCallback( data, callback ); + this._reqCallback( data, callback, id ); return this; }, } ); diff --git a/test/dapi/format/JsonResponseTest.js b/test/dapi/format/JsonResponseTest.js index 970ddeb..7f198c9 100644 --- a/test/dapi/format/JsonResponseTest.js +++ b/test/dapi/format/JsonResponseTest.js @@ -142,7 +142,7 @@ function _createStubbedDapi( err, resp ) { given: null, - 'virtual public request': function( data, callback ) + 'virtual public request': function( data, callback, id ) { this.given = data; callback( err, resp ); diff --git a/test/server/service/RatingServiceSubmitNotifyTest.js b/test/server/service/RatingServiceSubmitNotifyTest.js index e7bf4c6..1333eaf 100644 --- a/test/server/service/RatingServiceSubmitNotifyTest.js +++ b/test/server/service/RatingServiceSubmitNotifyTest.js @@ -127,7 +127,7 @@ describe( 'RatingServiceSubmitNotify', () => // warning: if an expectation fails, because of how // RatingService handles errors, it will cause the test to // _hang_ rather than throw the assertion error - request( data, callback ) + request( data, callback, id ) { expect( given_request ).to.equal( request ); expect( data ).to.deep.equal( { quote_id: quote_id } );