2019-11-11 15:37:01 -05:00
|
|
|
/**
|
|
|
|
* Start the Liza delta processor
|
|
|
|
*
|
|
|
|
* Copyright (C) 2010-2019 R-T Specialty, LLC.
|
|
|
|
*
|
|
|
|
* This file is part of the Liza Data Collection Framework.
|
|
|
|
*
|
|
|
|
* liza is free software: you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License as published by
|
|
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
|
|
* (at your option) any later version.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
*/
|
2019-12-02 10:00:23 -05:00
|
|
|
import { createAmqpConfig } from '../src/system/AmqpPublisher';
|
2019-11-25 17:20:56 -05:00
|
|
|
import { MongoDeltaDao } from '../src/system/db/MongoDeltaDao';
|
|
|
|
import { DeltaProcessor } from '../src/system/DeltaProcessor';
|
|
|
|
import { DeltaPublisher } from '../src/system/DeltaPublisher';
|
2019-12-02 10:00:23 -05:00
|
|
|
import { MongoCollection } from '../src/types/mongodb';
|
|
|
|
import { createAvroEncoder } from '../src/system/avro/AvroFactory';
|
|
|
|
import {
|
|
|
|
createMongoConfig,
|
|
|
|
createMongoDB,
|
|
|
|
getMongoCollection,
|
|
|
|
} from '../src/system/db/MongoFactory';
|
|
|
|
import { EventMediator } from '../src/system/EventMediator';
|
2019-11-25 17:20:56 -05:00
|
|
|
import { EventEmitter } from 'events';
|
2019-12-02 10:00:23 -05:00
|
|
|
import { StandardLogger } from '../src/system/StandardLogger';
|
|
|
|
import { MetricsCollector } from '../src/system/MetricsCollector';
|
2019-11-21 15:59:17 -05:00
|
|
|
import {
|
2019-12-02 10:00:23 -05:00
|
|
|
PrometheusFactory,
|
|
|
|
createPrometheusConfig,
|
|
|
|
} from '../src/system/PrometheusFactory';
|
|
|
|
import { AmqpConnection } from '../src/system/amqp/AmqpConnection';
|
2019-11-13 14:08:42 -05:00
|
|
|
|
|
|
|
|
2019-12-02 10:00:23 -05:00
|
|
|
const amqp_conf = createAmqpConfig( process.env );
|
|
|
|
const prom_conf = createPrometheusConfig( process.env );
|
|
|
|
const db_conf = createMongoConfig( process.env );
|
|
|
|
const db = createMongoDB( db_conf );
|
|
|
|
const process_interval_ms = +( process.env.process_interval_ms || 2000 );
|
2019-11-25 17:20:56 -05:00
|
|
|
const env = process.env.NODE_ENV || 'Unknown Environment';
|
|
|
|
const emitter = new EventEmitter();
|
2019-12-02 10:00:23 -05:00
|
|
|
const log = new StandardLogger( console, ts_ctr, env );
|
|
|
|
const amqp_connection = new AmqpConnection( amqp_conf, emitter );
|
|
|
|
const publisher = new DeltaPublisher(
|
|
|
|
emitter,
|
|
|
|
ts_ctr,
|
|
|
|
createAvroEncoder,
|
|
|
|
amqp_connection,
|
|
|
|
);
|
2019-11-25 17:20:56 -05:00
|
|
|
|
|
|
|
// Prometheus Metrics
|
|
|
|
const prom_factory = new PrometheusFactory();
|
2019-11-26 19:31:39 -05:00
|
|
|
const metrics = new MetricsCollector(
|
|
|
|
prom_factory,
|
|
|
|
prom_conf,
|
|
|
|
emitter,
|
|
|
|
process.hrtime,
|
|
|
|
);
|
2019-11-25 17:20:56 -05:00
|
|
|
|
|
|
|
// Structured logging
|
2019-12-02 10:00:23 -05:00
|
|
|
new EventMediator( log, emitter );
|
2019-11-13 14:08:42 -05:00
|
|
|
|
2019-11-21 15:59:17 -05:00
|
|
|
let process_interval: NodeJS.Timer;
|
2019-11-25 17:20:56 -05:00
|
|
|
let dao: MongoDeltaDao;
|
2019-11-21 15:59:17 -05:00
|
|
|
|
2019-12-02 10:00:23 -05:00
|
|
|
getMongoCollection( db, db_conf )
|
2019-11-26 19:31:39 -05:00
|
|
|
.then( ( conn: MongoCollection ) => { return new MongoDeltaDao( conn ); } )
|
2019-12-02 10:00:23 -05:00
|
|
|
.then( ( mongoDao: MongoDeltaDao ) => { dao = mongoDao; } )
|
|
|
|
.then( _ => amqp_connection.connect() )
|
2019-11-26 19:31:39 -05:00
|
|
|
.then( _ =>
|
2019-12-02 10:00:23 -05:00
|
|
|
{
|
|
|
|
log.info( 'Liza Delta Processor' );
|
2019-11-21 15:59:17 -05:00
|
|
|
|
2019-12-02 10:00:23 -05:00
|
|
|
handleShutdown();
|
2019-11-21 15:59:17 -05:00
|
|
|
|
2019-12-02 10:00:23 -05:00
|
|
|
const processor = new DeltaProcessor( dao, publisher, emitter );
|
|
|
|
|
|
|
|
process_interval = setInterval( () =>
|
2019-11-26 19:31:39 -05:00
|
|
|
{
|
|
|
|
processor.process();
|
2019-12-02 10:00:23 -05:00
|
|
|
|
|
|
|
dao.getErrorCount()
|
|
|
|
.then( count => { metrics.updateErrorCount( count ) } );
|
2019-11-26 19:31:39 -05:00
|
|
|
},
|
|
|
|
process_interval_ms,
|
|
|
|
);
|
|
|
|
} )
|
2019-12-02 10:00:23 -05:00
|
|
|
.catch( e =>
|
|
|
|
{
|
|
|
|
log.error( e );
|
|
|
|
process.exit( 1 );
|
|
|
|
} );
|
2019-11-21 15:59:17 -05:00
|
|
|
|
|
|
|
|
|
|
|
/**
|
2019-12-02 10:00:23 -05:00
|
|
|
* Hook shutdown events
|
2019-11-21 15:59:17 -05:00
|
|
|
*/
|
2019-12-02 10:00:23 -05:00
|
|
|
function handleShutdown(): void
|
2019-11-21 15:59:17 -05:00
|
|
|
{
|
2019-11-26 19:31:39 -05:00
|
|
|
process.on( 'SIGINT', () => { shutdown( 'SIGINT' ); } )
|
2019-12-02 10:00:23 -05:00
|
|
|
.on( 'SIGTERM', () => { shutdown( 'SIGTERM' ); } );
|
2019-11-21 15:59:17 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Perform a graceful shutdown
|
|
|
|
*
|
|
|
|
* @param signal - the signal that caused the shutdown
|
|
|
|
*/
|
|
|
|
function shutdown( signal: string ): void
|
|
|
|
{
|
2019-12-02 10:00:23 -05:00
|
|
|
log.info( 'Received ' + signal + '. Beginning graceful shutdown:' );
|
|
|
|
log.info( '...Stopping processing interval' );
|
2019-11-21 15:59:17 -05:00
|
|
|
|
|
|
|
clearInterval( process_interval );
|
|
|
|
|
2019-12-02 10:00:23 -05:00
|
|
|
log.info( '...Closing MongoDb connection' );
|
2019-11-21 15:59:17 -05:00
|
|
|
|
|
|
|
db.close( ( err, _data ) =>
|
|
|
|
{
|
|
|
|
if ( err )
|
|
|
|
{
|
2019-11-25 17:20:56 -05:00
|
|
|
console.error( ' Error closing connection: ' + err );
|
2019-11-21 15:59:17 -05:00
|
|
|
}
|
|
|
|
} );
|
|
|
|
|
2019-12-02 10:00:23 -05:00
|
|
|
log.info( '...Closing AMQP connection...' );
|
|
|
|
|
|
|
|
amqp_connection.close();
|
|
|
|
|
|
|
|
log.info( '...Stopping the metrics collector...' );
|
2019-11-21 15:59:17 -05:00
|
|
|
|
2019-12-02 10:00:23 -05:00
|
|
|
metrics.stop();
|
2019-11-21 15:59:17 -05:00
|
|
|
|
2019-12-02 10:00:23 -05:00
|
|
|
log.info( 'Shutdown complete. Exiting.' );
|
2019-11-21 15:59:17 -05:00
|
|
|
|
|
|
|
process.exit();
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2019-11-13 14:08:42 -05:00
|
|
|
/** Timestamp constructor
|
|
|
|
*
|
|
|
|
* @return a timestamp
|
|
|
|
*/
|
|
|
|
function ts_ctr(): UnixTimestamp
|
|
|
|
{
|
|
|
|
return <UnixTimestamp>Math.floor( new Date().getTime() / 1000 );
|
|
|
|
}
|