employer
/
liza
1
0
Fork 0
Code Releases Activity

Integrate TokenedDataApi into system 

This changes the easejs interface for DataApi, which requires adding the
param to everything.  The TS interface was created in a previous commit and
already contained this parameter.  The idea is to remove the easejs
interface in the future, but traits are a barrier to that atm.

DocumentServer and controller demonstrate the mess that we have with regards
to instantiating dependencies.  This needs to change---DocumentServer itself
was something that was started but never fully realized.  It makes this
incredibly confusing, difficult to follow, and complicates important error
handling that ought to be taking place.  It also discourages implementing
additional dependencies.

I'm not going to go through and provide a ChangeLog-style commit message for
this commit.  I'm too exhausted by this crap.
master
Mike Gerwitz 2019-10-16 14:36:54 -04:00
parent d8c065817f
commit e003abcd4b
21 changed files with 211 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/dapi/AutoRetry.js
View File

@ -121,9 +121,9 @@ module.exports = Trait( 'AutoRetry' )
*
* @return {DataApi} self
*/
'abstract override public request': function( input, callback )
'abstract override public request': function( input, callback, id )
{
this._try( input, callback, this._tries );
this._try( input, callback, id, this._tries );
return this;
},
@ -141,7 +141,7 @@ module.exports = Trait( 'AutoRetry' )
*
* @return {undefined}
*/
'private _try': function( input, callback, n )
'private _try': function( input, callback, id, n )
{
var _self = this;
@ -178,10 +178,10 @@ module.exports = Trait( 'AutoRetry' )
( n - 1 ),
function()
{
_self._try( input, callback, ( n - 1 ) );
_self._try( input, callback, id, ( n - 1 ) );
},
complete
);
} );
}, id );
}
} );

2
src/dapi/BucketDataApi.js
View File

@ -66,7 +66,7 @@ module.exports = Class( 'BucketDataApi' )
*
* @return {BucketDataApi} self
*/
'public request': function( data, callback )
'public request': function( data, callback, id )
{
var _self = this.__inst,
rows = [];

5
src/dapi/DataApi.ts
View File

@ -63,7 +63,7 @@ export interface DataApi
{
request(
data: DataApiInput,
callback: ( e: Error | null, data: DataApiResult | null ) => void,
callback: NodeCallback<DataApiResult>,
id: string,
): this;
}
@ -94,8 +94,9 @@ module.exports = Interface( 'DataApi',
*
* @param {?Object<string,string>|string} data params or post data
* @param {function(?Error,*):string} callback continuation upon reply
* @param {string} id unique dapi identifier
*
* @return {DataApi} self
*/
'public request': [ 'data', 'callback' ]
'public request': [ 'data', 'callback', 'id' ]
} );

4
src/dapi/DataApiFactory.js
View File

@ -58,7 +58,7 @@ module.exports = Class( 'DataApiFactory',
const nonempty = !!descl.static_nonempty;
const multiple = !!descl.static_multiple;
const api = this._createDataApi( type, descl, bucket );
const api = this.createDataApi( type, descl, bucket );
return RestrictedDataApi(
StaticAdditionDataApi( api, nonempty, multiple, static_data ),
@ -93,7 +93,7 @@ module.exports = Class( 'DataApiFactory',
*
* @return {DataApi}
*/
'private _createDataApi'( type, desc, bucket )
'virtual protected createDataApi'( type, desc, bucket )
{
const source = ( desc.source || '' );
const method = ( desc.method || '' );

2
src/dapi/DataApiManager.js
View File

@ -202,7 +202,7 @@ module.exports = Class( 'DataApiManager' )
_self._pendingApiCall[ id ] = undefined;
_self.emit( 'fieldLoaded', name, +index );
}
} ) )
}, id ) )
.catch( e => fc( e ) );
};

2
src/dapi/QuoteDataApi.js
View File

@ -77,7 +77,7 @@ module.exports = Class( 'QuoteDataApi' )
*
* @return {DataApi} self
*/
'public request'( data, callback )
'public request'( data, callback, id )
{
this._dapi.request( this.mapData( data ), callback );
},

2
src/dapi/RestDataApi.js
View File

@ -73,7 +73,7 @@ module.exports = Class( 'RestDataApi' )
*
* @return {RestDataApi} self
*/
'public request': function( data, callback )
'public request': function( data, callback, id )
{
var _self = this.__inst;

4
src/dapi/RestrictedDataApi.js
View File

@ -85,7 +85,7 @@ module.exports = Class( 'RestrictedDataApi' )
*
* @return {DataApi} self
*/
'virtual public request': function( data, callback )
'virtual public request': function( data, callback, id )
{
data = data || {};
callback = callback || function() {};
@ -103,7 +103,7 @@ module.exports = Class( 'RestrictedDataApi' )
err,
_self._checkResponse( response, callback )
);
} );
}, id );
},

11
src/dapi/StaticAdditionDataApi.js
View File

@ -66,13 +66,6 @@ module.exports = Class( 'StaticAdditionDataApi' )
*/
__construct: function( data_api, nonempty, multiple, static_data )
{
if ( !( Class.isA( DataApi, data_api ) ) )
{
throw Error(
'Expected object of type DataApi; given: ' + data_api
);
}
this._api = data_api;
this._static = static_data;
this._nonempty = !!nonempty;
@ -88,7 +81,7 @@ module.exports = Class( 'StaticAdditionDataApi' )
*
* @return {DataApi} self
*/
'public request': function( data, callback )
'public request': function( data, callback, id )
{
data = data || {};
callback = callback || function() {};
@ -110,7 +103,7 @@ module.exports = Class( 'StaticAdditionDataApi' )
err,
_self._unshiftData( response )
);
} );
}, id );
},

4
src/dapi/format/JsonResponse.js
View File

@ -47,14 +47,14 @@ module.exports = Trait( 'JsonResponse' )
*
* @return {DataApi} self
*/
'virtual abstract override public request': function( data, callback )
'virtual abstract override public request': function( data, callback, id )
{
var _self = this;
this.__super( data, function( err, resp )
{
_self._tryParse( err, resp, callback );
} );
}, id );
return this;
},

4
src/dapi/format/ResponseApply.js
View File

@ -65,12 +65,12 @@ module.exports = Trait( 'ResponseApply' )
*
* @return {DataApi} self
*/
'virtual abstract override public request'( data, callback )
'virtual abstract override public request'( data, callback, id )
{
this.__super( data, ( e, retdata ) =>
{
callback( e, this._dataf( retdata ) );
} );
}, id );
return this;
},

2
src/dapi/http/HttpDataApi.js
View File

@ -128,7 +128,7 @@ module.exports = Class( 'HttpDataApi' )
*
* @throws {TypeError} on validation failure
*/
'virtual public request': function( data, callback )
'virtual public request': function( data, callback, id )
{
// null is a good indicator of "I have no intent to send any data";
// empty strings and objects are not, since those are valid data

72
src/server/DocumentServer.js
View File

@ -48,17 +48,40 @@ const {
JsonServerResponse,
ServerDataApiFactory,
},
token: {
MongoTokenDao: { MongoTokenDao },
},
},
} = require( '../..' );
/**
* Vanilla document server
*
* XXX: This is a mess, and it's only getting worse with dependencies
* instantiated everywhere. Everything should be instantiated in one place
* rather than part of them being passed in here. See controller.js.
*/
module.exports = Class( 'DocumentServer',
{
'public create': ( dao, logger, enc_service, origin_url, conf ) =>
Promise.all( [
/**
* Create document server
*
* See above XXX.
*
* @param {MongoServerDao} dao server DAO
* @param {Logger} logger log manager
* @param {EncryptionService} enc_service encryption service
* @param {string} origin_url HTTP_ORIGIN_URL
* @param {ConfStore} conf configuration store
* @param {MongoConnection} collection database collection
*
* @return {Promise<Server>}
*/
'public create'( dao, logger, enc_service, origin_url, conf, collection )
{
return Promise.all( [
conf.get( 'dapi' ),
] ).then( ([ dapi_conf ]) => Server(
new JsonServerResponse.create(),
@ -68,17 +91,46 @@ module.exports = Class( 'DocumentServer',
DataProcessor(
bucket_filter,
( apis, request ) => DataApiManager(
ServerDataApiFactory(
origin_url || request.getOrigin(),
request,
dapi_conf
),
apis
( apis, request, quote ) => this._createDapiManager(
apis, request, origin_url, dapi_conf, quote, collection
),
DapiMetaSource( QuoteDataBucket ),
StagingBucket
),
ProgramInit()
) )
) );
},
/**
* Create new DataApiManager
*
* See above XXX.
*
* @param {Object} apis API definitions
* @param {Request} request Node HTTP request
* @param {string} origin_url HTTP_ORIGIN_URL
* @param {Object} dapi_conf dapi configuration
* @param {Quote} quote current quote for request
* @param {MongoConnection} collection database collection
*/
'private _createDapiManager'(
apis, request, origin_url, dapi_conf, quote, collection
)
{
return DataApiManager(
ServerDataApiFactory(
origin_url || request.getOrigin(),
request,
dapi_conf,
quote.getId(),
new MongoTokenDao(
collection,
'dapitok',
() => Math.floor( ( new Date() ).getTime() / 1000 )
)
),
apis
);
},
} );

2
src/server/Server.js
View File

@ -1148,7 +1148,7 @@ module.exports = Class( 'Server' )
const { filtered, dapis, meta_clear } =
server._dataProcessor.processDiff(
parsed_data, request, program, bucket
parsed_data, request, program, bucket, quote
);
server._monitorMetadataPromise( quote, dapis, meta_clear );

144
src/server/daemon/controller.js
View File

@ -127,47 +127,50 @@ exports.init = function( logger, enc_service, conf )
var db = _createDB( logger );
const dao = MongoServerDao( db );
_createDocumentServer( dao, logger, enc_service, conf ).then( srv =>
db.collection( 'quotes', function( err, collection )
{
server = srv;
server_cache = _createCache( server );
server.init( server_cache, exports.rater );
// TODO: temporary proof-of-concept
rating_service = RatingService.use(
RatingServicePublish( amqplib, exports.post_rate_publish, logger )
)(
logger, dao, server, exports.rater
);
// TODO: exports.init needs to support callbacks; this will work, but
// only because it's unlikely that we'll get a request within
// milliseconds of coming online
_initExportService( db, function( service )
_createDocumentServer( dao, logger, enc_service, conf, collection ).then( srv =>
{
c1_export_service = service;
} );
server = srv;
server.on( 'quotePverUpdate', function( quote, program, event )
{
// let them know that we're going to be a moment
var c = event.wait();
server_cache = _createCache( server );
server.init( server_cache, exports.rater );
getCleaner( program ).clean( quote, function( err )
// TODO: temporary proof-of-concept
rating_service = RatingService.use(
RatingServicePublish( amqplib, exports.post_rate_publish, logger )
)(
logger, dao, server, exports.rater
);
// TODO: exports.init needs to support callbacks; this will work, but
// only because it's unlikely that we'll get a request within
// milliseconds of coming online
_initExportService( collection, function( service )
{
// report on our success/failure
if ( err )
{
event.bad( err );
}
else
{
event.good();
}
c1_export_service = service;
} );
// we're done
c();
server.on( 'quotePverUpdate', function( quote, program, event )
{
// let them know that we're going to be a moment
var c = event.wait();
getCleaner( program ).clean( quote, function( err )
{
// report on our success/failure
if ( err )
{
event.bad( err );
}
else
{
event.good();
}
// we're done
c();
} );
} );
} );
} );
@ -203,7 +206,7 @@ function _createDB( logger )
return db;
}
function _createDocumentServer( dao, logger, enc_service, conf )
function _createDocumentServer( dao, logger, enc_service, conf, collection )
{
const origin_url = process.env.HTTP_ORIGIN_URL || '';
@ -219,52 +222,43 @@ function _createDocumentServer( dao, logger, enc_service, conf )
}
return DocumentServer()
.create( dao, logger, enc_service, origin_url, conf );
.create( dao, logger, enc_service, origin_url, conf, collection );
}
function _initExportService( db, callback )
function _initExportService( collection, callback )
{
db.collection( 'quotes', function( err, collection )
{
if ( collection === null )
{
return;
}
var spoof_host = (
''+(
process.env.C1_EXPORT_HOST
|| process.env.LV_RATE_DOMAIN
|| process.env.LV_RATE_HOST
).trim()
);
var spoof_host = (
''+(
process.env.C1_EXPORT_HOST
|| process.env.LV_RATE_DOMAIN
|| process.env.LV_RATE_HOST
).trim()
);
var spoof = SessionSpoofHttpClient( http, spoof_host );
var spoof = SessionSpoofHttpClient( http, spoof_host );
callback(
ExportService
.use( TokenedService(
'c1import',
new MongoTokenDao( collection, "exports", getUnixTimestamp ),
function tokgen()
{
var shasum = crypto.createHash( 'sha1' );
shasum.update( ''+Math.random() );
callback(
ExportService
.use( TokenedService(
'c1import',
new MongoTokenDao( collection, "exports", getUnixTimestamp ),
function tokgen()
{
var shasum = crypto.createHash( 'sha1' );
shasum.update( ''+Math.random() );
return shasum.digest( 'hex' );
},
function newcapturedResponse( request, callback )
{
return UserResponse
.use( CapturedUserResponse( callback ) )
( request );
}
) )
( spoof )
);
} );
return shasum.digest( 'hex' );
},
function newcapturedResponse( request, callback )
{
return UserResponse
.use( CapturedUserResponse( callback ) )
( request );
}
) )
( spoof )
);
}

4
src/server/request/DataProcessor.js
View File

@ -87,10 +87,10 @@ module.exports = Class( 'DataProcessor',
*
* @return {Object} processed diff
*/
'public processDiff'( data, request, program, bucket )
'public processDiff'( data, request, program, bucket, quote )
{
const filtered = this.sanitizeDiff( data, request, program );
const dapi_manager = this._dapif( program.apis, request );
const dapi_manager = this._dapif( program.apis, request, quote );
const staging = this._stagingCtor( bucket );
// forbidBypass will force diff generation on initQuote

50
src/server/request/ServerDataApiFactory.js
View File

@ -20,6 +20,8 @@
*/
const { Class } = require( 'easejs' );
const crypto = require( 'crypto' );
const {
dapi: {
DataApiFactory,
@ -31,6 +33,16 @@ const {
store: {
StoreMissError,
},
server: {
dapi: {
TokenedDataApi: { TokenedDataApi },
},
token: {
store: {
PersistentTokenStore: { PersistentTokenStore },
},
},
},
} = require( '../..' );
@ -58,12 +70,48 @@ module.exports = Class( 'ServerDataApiFactory' )
*/
'private _conf': null,
/**
* Document (quote) id
* @type {DocumentId}
*/
'private _doc_id': 0,
constructor( origin, session, conf )
constructor( origin, session, conf, doc_id, tokdao )
{
this._origin = ''+origin;
this._session = session;
this._conf = conf;
this._doc_id = doc_id;
this._tokdao = tokdao;
},
'override protected createDataApi'( type, desc, bucket )
{
return new TokenedDataApi(
this.__super( type, desc, bucket ),
token_ns => new PersistentTokenStore(
this._tokdao,
this._doc_id,
token_ns,
() => this._generateTokenId()
)
);
},
/**
* Generate random token identifier
*
* @return {string} unique token identifier
*/
'private _generateTokenId'()
{
var shasum = crypto.createHash( 'sha1' );
shasum.update( ''+Math.random() );
return shasum.digest( 'hex' );
},

2
test/dapi/AutoRetryTest.js
View File

@ -282,7 +282,7 @@ function _createStub( err, resp )
given: null,
requests: 0,
'virtual public request': function( data, callback )
'virtual public request': function( data, callback, id )
{
this.given = data;
this.requests++;

4
test/dapi/DummyDataApi.js
View File

@ -59,9 +59,9 @@ module.exports = Class( 'DummyDataApi' )
*
* @return {DataApi} self
*/
'virtual public request'( data, callback )
'virtual public request'( data, callback, id )
{
this._reqCallback( data, callback );
this._reqCallback( data, callback, id );
return this;
},
} );

2
test/dapi/format/JsonResponseTest.js
View File

@ -142,7 +142,7 @@ function _createStubbedDapi( err, resp )
{
given: null,
'virtual public request': function( data, callback )
'virtual public request': function( data, callback, id )
{
this.given = data;
callback( err, resp );

2
test/server/service/RatingServiceSubmitNotifyTest.js
View File

@ -127,7 +127,7 @@ describe( 'RatingServiceSubmitNotify', () =>
// warning: if an expectation fails, because of how
// RatingService handles errors, it will cause the test to
// _hang_ rather than throw the assertion error
request( data, callback )
request( data, callback, id )
{
expect( given_request ).to.equal( request );
expect( data ).to.deep.equal( { quote_id: quote_id } );