1
0
Fork 0
liza/src/server/daemon/controller.js

850 lines
23 KiB
JavaScript

/**
* Route controller
*
* Copyright (C) 2010-2019 R-T Specialty, LLC.
*
* This file is part of the Liza Data Collection Framework.
*
* liza is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @todo this is a mess of routing and glue code
*/
const {
Db: MongoDb,
Server: MongoServer,
Connection: MongoConnection,
ReplSetServers: ReplSetServers,
} = require( 'mongodb/lib/mongodb' );
const easejs = require( 'easejs' );
const regex_base = /^\/quote\/([a-z0-9-]+)\/?(?:\/(\d+)\/?(?:\/(.*))?|\/(program.js))?$/;
const regex_step = /^step\/(\d+)\/?(?:\/(post|visit))?$/;
const http = require( 'http' );
const crypto = require( 'crypto' );
var server = null;
var server_cache = null;
var rating_service = null;
const {
bucket: {
QuoteDataBucket,
delta
},
dapi: {
http: {
HttpDataApi,
HttpDataApiUrlData,
NodeHttpImpl,
SpoofedNodeHttpImpl,
},
DataApiFactory,
DataApiManager,
},
document: {
DocumentProgramFormatter,
},
field: {
FieldClassMatcher,
},
server: {
DocumentServer,
db: {
MongoServerDao: { MongoServerDao },
},
lock: {
Semaphore,
},
quote: {
ServerSideQuote: Quote,
ProgramQuoteCleaner,
},
service: {
export: {
ExportService,
},
RatingService: { RatingService },
RatingServicePublish,
TokenedService,
},
token: {
MongoTokenDao: {
MongoTokenDao
},
},
request: {
CapturedUserResponse,
SessionSpoofHttpClient,
UserResponse,
},
},
store,
} = require( '../..' );
const amqplib = require( 'amqplib' );
// read and write locks, as separate semaphores
var rlock = Semaphore(),
wlock = Semaphore();
// concurrent session flag
var sflag = {};
// TODO: kluge to get liza somewhat decoupled from lovullo (rating module)
exports.rater = {};
exports.skey = "";
exports.post_rate_publish = {};
exports.init = function( logger, enc_service, conf, env )
{
var db = _createDB( logger );
const dao = new MongoServerDao( db, env );
db.collection( 'quotes', function( err, collection )
{
_createDocumentServer( dao, logger, enc_service, conf, collection ).then( srv =>
{
server = srv;
server_cache = _createCache( server );
server.init( server_cache, exports.rater );
// TODO: temporary proof-of-concept
rating_service = easejs( RatingService ).use(
RatingServicePublish( amqplib, exports.post_rate_publish, logger )
)(
logger, dao, server, exports.rater, delta.createDelta
);
// TODO: exports.init needs to support callbacks; this will work, but
// only because it's unlikely that we'll get a request within
// milliseconds of coming online
_initExportService( collection, function( service )
{
c1_export_service = service;
} );
server.on( 'quotePverUpdate', function( quote, program, event )
{
// let them know that we're going to be a moment
var c = event.wait();
getCleaner( program ).clean( quote, function( err )
{
// report on our success/failure
if ( err )
{
event.bad( err );
}
else
{
event.good();
}
// we're done
c();
} );
} );
} );
} );
}
// TODO: Remove this and use the new MongoFactory.ts
function _createDB( logger )
{
if(process.env.LIZA_MONGODB_HA==1)
{
var mongodbPort = process.env.MONGODB_PORT || MongoConnection.DEFAULT_PORT;
var mongodbReplSet = process.env.LIZA_MONGODB_REPLSET || 'rs0';
var dbServers = new ReplSetServers(
[
new MongoServer( process.env.LIZA_MONGODB_HOST_A, +process.env.LIZA_MONGODB_PORT_A || mongodbPort),
new MongoServer( process.env.LIZA_MONGODB_HOST_B, +process.env.LIZA_MONGODB_PORT_B || mongodbPort)
],
{rs_name: mongodbReplSet, auto_reconnect: true}
);
}
else
{
var dbServers = new MongoServer(
process.env.MONGODB_HOST || '127.0.0.1',
+process.env.MONGODB_PORT || MongoConnection.DEFAULT_PORT,
{auto_reconnect: true}
);
}
var db = new MongoDb(
'program',
dbServers,
{native_parser: false, safe: false, logger: logger}
);
return db;
}
function _createDocumentServer( dao, logger, enc_service, conf, collection )
{
const origin_url = process.env.HTTP_ORIGIN_URL || '';
if ( !origin_url )
{
// this allows the system to work without configuration (e.g. for
// local development), but is really bad
logger.log( logger.PRIORITY_IMPORTANT,
"*** HTTP_ORIGIN_URL environment variable not set; " +
"system will fall back to using the origin of HTTP requests, " +
"meaning an attacker can control where server-side requests go! ***"
);
}
return DocumentServer()
.create( dao, logger, enc_service, origin_url, conf, collection );
}
function _initExportService( collection, callback )
{
var spoof_host = (
''+(
process.env.C1_EXPORT_HOST
|| process.env.LV_RATE_DOMAIN
|| process.env.LV_RATE_HOST
).trim()
);
var spoof = SessionSpoofHttpClient( http, spoof_host );
callback(
ExportService
.use( TokenedService(
'c1import',
new MongoTokenDao( collection, "exports", getUnixTimestamp ),
function tokgen()
{
var shasum = crypto.createHash( 'sha1' );
shasum.update( ''+Math.random() );
return shasum.digest( 'hex' );
},
function newcapturedResponse( request, callback )
{
return UserResponse
.use( CapturedUserResponse( callback ) )
( request );
}
) )
( spoof )
);
}
/**
* Retrieve current date as a Unix timestamp
*/
function getUnixTimestamp()
{
return Math.floor( ( new Date() ).getTime() / 1000 );
}
/**
* Create server cache
*
* TODO: This needs to be moved elsewhere; it is a stepping-stone
* kluge.
*
* @param {Server} server server containing miss methods
*
* @return {Store} cache
*/
function _createCache( server )
{
const progjs_cache = store.MemoryStore.use(
store.MissLookup( server.loadProgramFiles.bind( server ) )
)();
const step_prog_cache = store.MemoryStore.use(
store.MissLookup( program_id => Promise.resolve(
store.MemoryStore.use(
store.MissLookup(
server.loadStepHtml.bind( server, program_id )
)
)()
) )
)();
const prog_cache = store.MemoryStore.use(
store.MissLookup( server.loadProgram.bind( server ) )
)();
const cache = store.MemoryStore.use( store.Cascading )();
cache.add( 'program_js', progjs_cache );
cache.add( 'step_html', step_prog_cache );
cache.add( 'program', prog_cache );
return cache;
}
exports.reload = function()
{
// will cause all steps, programs, etc to be reloaded on demand
server_cache.clear();
server.reload( exports.rater );
};
exports.route = function( request )
{
var data;
if ( !( data = request.getUri().match( regex_base ) ) )
{
// we don't handle this URI
return Promise.resolve( false );
}
// we don't want to cache the responses, as most of them change with each
// request
request.noCache();
var program_id = data[1];
return server.getProgram( program_id )
.then( function( program )
{
return new Promise( function( resolve, reject )
{
doRoute( program, request, data, resolve, reject );
} );
} );
}
function doRoute( program, request, data, resolve, reject )
{
// store our data in more sensible vars
var program_id = data[1],
quote_id = +data[2] || 0,
cmd = data[3] || data[4] || '',
session = request.getSession();
// if we were unable to load the program class, that's a problem
if ( program === null )
{
server.sendError( request,
'Internal error. Please contact our support team for ' +
'support.' +
'<br /><br />Your information has <em>not</em> been saved!'
);
resolve( true );
return;
}
var skey = has_skey( request );
// is the user currently logged in?
if ( ( request.getSession().isLoggedIn() === false )
&& !skey
)
{
// todo: this is temporary so we don't break our current setup; remove
// this check once we can error out before we even get to this point
// (PHP current handles the initial page load)
if ( cmd !== 'program.js' )
{
session.setRedirect( '/quote/' + program_id + '/', function()
{
session.setReturnQuoteNumber( quote_id, function()
{
// peoples are trying to steal our secrets!?!!?
server.sendError(
request,
'Please <a href="/login">click here</a> to log in.'
);
} );
} );
resolve( true );
return;
}
}
// if the session key was provided, mark us as internal
if ( skey )
{
request.getSession().setAgentId( '900000' );
}
// we'll be serving all our responses as plain text
request.setContentType( 'text/plain' );
if ( data = cmd.match( regex_step ) )
{
var step_id = data[1];
var step_action = ( data[2] !== undefined ) ? data[2] : '';
switch ( step_action )
{
case 'post':
acquireWriteLock( quote_id, request, function()
{
handleRequest( function( quote )
{
server.handlePost(
step_id, request, quote, program, session
);
} );
} );
break;
case 'visit':
acquireRwLock( quote_id, request, function()
{
handleRequest( function( quote )
{
server.visitStep( step_id, request, quote );
} );
} );
break;
default:
// send the requested step to the client
acquireReadLock( quote_id, request, function()
{
handleRequest( function( quote )
{
server.sendStep(
request, quote, program, step_id, session
);
} );
} );
break;
}
}
else if ( cmd == 'init' )
{
acquireWriteLock( quote_id, request, function()
{
handleRequest( function( quote )
{
server.sendInit(
request,
quote,
program,
// for invalid quote requests
createQuoteQuick,
// concurrent access?
getConcurrentSessionUser( quote_id, session )
);
} );
} );
}
else if ( cmd == 'progdata' )
{
acquireReadLock( quote_id, request, function()
{
handleRequest( function( quote )
{
const response = UserResponse( request );
const bucket = quote.getBucket();
const class_matcher = FieldClassMatcher( program.whens );
DocumentProgramFormatter( program, class_matcher )
.format( bucket )
.then( data => response.ok( data ) )
.catch( e => response.internalError( {}, e ) );
} );
} );
}
else if ( cmd === 'mkrev' )
{
// the database operation for this is atomic and disjoint from
// anything else we're doing, so no need to acquire any sort of
// lock
handleRequest( function( quote )
{
server.createRevision( request, quote );
} );
}
// TODO: diff against other revisions as well
else if ( data = cmd.match( /^revdiffgrp\/(.*?)\/(\d+)$/ ) )
{
// similar to above; no locking needed
handleRequest( function( quote )
{
var gid = data[ 1 ] || '',
revid = +data[ 2 ] || 0;
server.diffRevisionGroup( request, program, quote, gid, revid );
} );
}
else if ( cmd == 'program.js' )
{
// no quote involved; just send the JS
server.sendProgramJs( request, program_id );
resolve( true );
return;
}
else if ( /^rate\b/.test( cmd ) )
{
// the client may have optionally requested the rate for a specific
// alias
var ratedata = cmd.match( /^rate(?:\/([a-z]+))?/ ),
alias = ratedata[ 1 ];
// request manual lock freeing; allows us to free the lock when we
// want to (since we'll be saving data to the DB async, after the
// response is already returned)
acquireWriteLock( quote_id, request, function( free )
{
// if we're performing deferred rating, it must be async;
// immediately free the locks and trust that the deferred process
// knows what it is doing and can properly handle such concurrency
alias && free();
handleRequest( function( quote )
{
var response = UserResponse( request );
rating_service.request( request, response, quote, alias )
.catch( () => {} )
.then( () => free() );
} );
}, true );
}
else if ( /^worksheet\//.test( cmd ) )
{
var wdata = cmd.match( /^worksheet\/(.+)\/([0-9]+)/ ),
supplier = wdata[ 1 ],
index = +wdata[ 2 ];
handleRequest( function( quote )
{
rating_service.serveWorksheet( request, quote, supplier, index );
} );
}
else if ( /^export\//.test( cmd ) )
{
var import_data = cmd.match( /^export\/(.+?)(?:\/(.+))?$/ ),
type = import_data[ 1 ],
subcmd = import_data[ 2 ];
// TODO: extract body
handleRequest( function( quote )
{
// TODO: support type
c1_export_service.request(
request,
UserResponse( request ),
quote,
subcmd
);
} );
}
else if ( cmd === 'quicksave' )
{
// TODO: we keep this route around only as a heartbeat, for now; the
// original purpose of this route (to save staged data) has been
// removed
handleRequest( function( quote )
{
server.sendEmptyReply( request, quote );
} );
touchSession( quote_id, session );
}
else if ( /^log\//.test( cmd ) )
{
// the "log" URI currently does absolutely nothing; ideally, we'd be
// able to post to this and log somewhere useful, but for now it
// just appears in the logs
handleRequest( function( quote )
{
server.sendEmptyReply( request, quote );
} );
}
else
{
resolve( false );
return;
}
// create a quote to represent this request
function handleRequest( operation )
{
createQuote( quote_id, program, request, operation, function( fatal )
{
// if fatal, notify the user and bail out
if ( fatal )
{
// an error occurred; quote invalid
server.sendError( request,
'There was a problem loading this quote; please contact ' +
'our support team for assistance.'
);
return;
}
// otherwise, the given quote is invalid, but we can provide a new
// one
server.sendNewQuote( request, createQuoteQuick );
} );
}
// we handled the request; don't do any additional routing
resolve( true );
}
/**
* Creates a new quote instance with the given quote id
*
* @param Integer quote_id id of the quote
* @param Program program program that the quote will be a part of
* @param Object request request to create quote
* @param Function( quote ) callback function to call when quote is ready
* @param Function( quote ) callback function to call when an error occurs
*
* @return undefined
*/
function createQuote( quote_id, program, request, callback, error_callback )
{
// if an invalid callback was given, log it to the console...that's a
// problem, since the quote won't even be returned!
callback = callback || function()
{
server.logger.log( log.PRIORITY_ERROR,
"Invalid createQuote() callback"
);
}
var bucket = QuoteDataBucket(),
metabucket = QuoteDataBucket(),
ratebucket = QuoteDataBucket(),
quote = Quote( quote_id, bucket );
quote.setMetabucket( metabucket );
quote.setRateBucket( ratebucket );
var controller = this;
return server.initQuote( quote, program, request,
function()
{
callback.call( controller, quote );
},
function()
{
error_callback.apply( controller, arguments );
}
);
}
function createQuoteQuick( id )
{
return Quote(
id,
QuoteDataBucket()
);
}
/**
* Check whether the proper skey (session key) was provided
*
* This is a basic authentication token that allows bypassing authentication
* for internal tasks (like creating quotes).
*
* XXX: A single shared secret is a terrible idea; this was intended to
* be a temporary solution. Fix this crap in favor of proper authentication
* between services.
*/
function has_skey( user_request )
{
if ( !exports.skey )
{
return false;
}
return ( user_request.getGetData().skey === exports.skey );
}
function getCleaner( program )
{
return ProgramQuoteCleaner( program );
}
/**
* Acquire a semaphore for a quote id
*
* Note that, since this controller is single-threaded, we do not have to worry
* about race conditions with regards to acquiring the lock.
*/
function acquireLock( type, id, request, c, manual )
{
type.acquire( id, function( free )
{
// automatically release the lock once the request completes (this is
// also safer, as it is hopefully immune to exceptions before lock
// release and will still work with the timeout system)
if ( !manual )
{
request.once( 'end', function()
{
free();
} );
}
// we're good!
c( free );
} );
// keep the quote session alive
touchSession( id, request.getSession() );
}
/**
* ALWAYS USE THIS FUNCTION WHEN TRYING TO ACQUIRE BOTH A READ AND A WRITE LOCK!
* Otherwise, the possibility for a deadlock is introduced if something else is
* attempting to acquire both locks in the opposite order!
*/
function acquireRwLock( id, request, c, manual )
{
acquireWriteLock( id, request, function()
{
acquireReadLock( id, request, c );
}, manual );
}
function acquireWriteLock( id, request, c, manual )
{
acquireLock( wlock, id, request, c, manual );
}
function acquireReadLock( id, request, c, manual )
{
acquireLock( rlock, id, request, c, manual );
}
function acquireWriteLockImmediate( id, request, c, manual )
{
if ( wlock.isLocked( id ) )
{
// we could not obtain the lock
c( null );
return;
}
// lock is free; acquire it
acquireWriteLock.apply( null, arguments );
}
function touchSession( id, session )
{
var cur = sflag[ id ];
// do not allow touching the session if we're not the owner
if ( cur && ( cur.agentName !== session.agentName() ) )
{
return;
}
sflag[ id ] = {
agentName: session.agentName(),
time: ( new Date() ).getTime(),
};
}
function getConcurrentSessionUser( id, session )
{
var flag = sflag[ id ];
// we have a flag; if we're the same user that created it (we check on name
// because internally we all have the same agent id), then do not consider
// this a concurrent access attempt
if ( !flag || ( flag.agentName === session.agentName() ) )
{
return '';
}
return ( flag.agentName );
}
// in the unfortunate situation where a write lock hangs, for whatever reason,
// this will ensure that it is eventually freed; note that, since this shouldn't
// ever happen, this interval is 30s, meaning that a given lock may exist for
// just under 60s
var __wlock_stale_interval = 30e3;
setInterval( function __wlock_stale_free()
{
// TODO: log properly and possibly kill the request
// TODO: if the same quote repeatedly has stale locks, perhaps the
// quote data is bad and should be locked
wlock.freeStale( __wlock_stale_interval, function( id )
{
console.log( 'Freeing stale write lock: ' + id );
} );
rlock.freeStale( __wlock_stale_interval, function( id )
{
console.log( 'Freeing stale read lock: ' + id );
} );
}, __wlock_stale_interval );
// set this to ~10s after the quicksave interval
var __sclear_interval = 70e3;
setInterval( function __sclear_timeout()
{
var now = ( new Date() ).getTime();
for ( var id in sflag )
{
// clear all session flags that have timed out
if ( ( now - sflag[ id ].time ) > __sclear_interval )
{
delete sflag[ id ];
}
}
}, __sclear_interval );