[DEV-5312] Create additional indexes and process less frequently
parent
abf76beac6
commit
b6f72b4b09
8
.env
8
.env
|
@ -14,4 +14,10 @@ PROM_PUSH_INTERVAL_MS=5000
|
||||||
PROM_BUCKETS_START=0
|
PROM_BUCKETS_START=0
|
||||||
PROM_BUCKETS_WIDTH=10
|
PROM_BUCKETS_WIDTH=10
|
||||||
PROM_BUCKETS_COUNT=10
|
PROM_BUCKETS_COUNT=10
|
||||||
PROCESS_INTERVAL_MS=2000
|
LIZA_MONGODB_HA=
|
||||||
|
LIZA_MONGODB_REPLSET=
|
||||||
|
LIZA_MONGODB_HOST_A=
|
||||||
|
LIZA_MONGODB_PORT_A=
|
||||||
|
LIZA_MONGODB_HOST_B=
|
||||||
|
LIZA_MONGODB_PORT_B=
|
||||||
|
PROCESS_INTERVAL_MS=2000
|
|
@ -48,7 +48,7 @@ const amqp_conf = createAmqpConfig( process.env );
|
||||||
const prom_conf = createPrometheusConfig( process.env );
|
const prom_conf = createPrometheusConfig( process.env );
|
||||||
const db_conf = createMongoConfig( process.env );
|
const db_conf = createMongoConfig( process.env );
|
||||||
const db = createMongoDB( db_conf );
|
const db = createMongoDB( db_conf );
|
||||||
const process_interval_ms = +( process.env.process_interval_ms || 2000 );
|
const process_interval_ms = +( process.env.process_interval_ms || 10000 );
|
||||||
const env = process.env.NODE_ENV || 'Unknown Environment';
|
const env = process.env.NODE_ENV || 'Unknown Environment';
|
||||||
const emitter = new EventEmitter();
|
const emitter = new EventEmitter();
|
||||||
const log = new StandardLogger( console, ts_ctr, env );
|
const log = new StandardLogger( console, ts_ctr, env );
|
||||||
|
|
|
@ -62,7 +62,7 @@ export function createPrometheusConfig(
|
||||||
hostname: env.PROM_HOST,
|
hostname: env.PROM_HOST,
|
||||||
port: +( env.PROM_PORT || 0 ),
|
port: +( env.PROM_PORT || 0 ),
|
||||||
env: process.env.NODE_ENV,
|
env: process.env.NODE_ENV,
|
||||||
push_interval_ms: +( process.env.PROM_PUSH_INTERVAL_MS || 5000 ),
|
push_interval_ms: +( process.env.PROM_PUSH_INTERVAL_MS || 10000 ),
|
||||||
buckets_start: +( process.env.PROM_BUCKETS_START || 0 ),
|
buckets_start: +( process.env.PROM_BUCKETS_START || 0 ),
|
||||||
buckets_width: +( process.env.PROM_BUCKETS_WIDTH || 10 ),
|
buckets_width: +( process.env.PROM_BUCKETS_WIDTH || 10 ),
|
||||||
buckets_count: +( process.env.PROM_BUCKETS_COUNT || 10 ),
|
buckets_count: +( process.env.PROM_BUCKETS_COUNT || 10 ),
|
||||||
|
|
|
@ -101,6 +101,8 @@ export class V1MessageWriter implements MessageWriter
|
||||||
const bucket_formatted = this.setDataTypes( bucket );
|
const bucket_formatted = this.setDataTypes( bucket );
|
||||||
const ratedata_formatted = this.setDataTypes( ratedata );
|
const ratedata_formatted = this.setDataTypes( ratedata );
|
||||||
const event_id = this.DELTA_MAP[ delta.type ];
|
const event_id = this.DELTA_MAP[ delta.type ];
|
||||||
|
const start_date_ms = { "long": meta.startDate * 1000 };
|
||||||
|
const last_update_ms = { "long": meta.lastUpdate * 1000 };
|
||||||
|
|
||||||
return {
|
return {
|
||||||
event: {
|
event: {
|
||||||
|
@ -111,8 +113,8 @@ export class V1MessageWriter implements MessageWriter
|
||||||
},
|
},
|
||||||
document: {
|
document: {
|
||||||
id: meta.id,
|
id: meta.id,
|
||||||
created: meta.startDate,
|
created: start_date_ms,
|
||||||
modified: meta.lastUpdate,
|
modified: last_update_ms,
|
||||||
},
|
},
|
||||||
session: {
|
session: {
|
||||||
Session: {
|
Session: {
|
||||||
|
|
|
@ -29,9 +29,9 @@ import { DaoError } from '../../error/DaoError';
|
||||||
|
|
||||||
|
|
||||||
const {
|
const {
|
||||||
Db: MongoDb,
|
Db: MongoDb,
|
||||||
Server: MongoServer,
|
Server: MongoServer,
|
||||||
ReplServers: ReplSetServers,
|
ReplSetServers: ReplSetServers,
|
||||||
} = require( 'mongodb' );
|
} = require( 'mongodb' );
|
||||||
|
|
||||||
|
|
||||||
|
@ -148,29 +148,42 @@ export function getMongoCollection(
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// initialize indexes
|
let createdCount = 0
|
||||||
collection.createIndex(
|
const checkAllCreated = (): void =>
|
||||||
[
|
{
|
||||||
['published', 1],
|
if( createdCount >= 3 )
|
||||||
['deltaError', 1],
|
|
||||||
],
|
|
||||||
true,
|
|
||||||
( e: any, _index: { [P: string]: any } ) =>
|
|
||||||
{
|
{
|
||||||
if ( e )
|
|
||||||
{
|
|
||||||
reject( new DaoError(
|
|
||||||
'Error creating index: ' + e
|
|
||||||
) );
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
resolve( collection );
|
resolve( collection );
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const cb = ( e: any, _index: { [P: string]: any } ): void =>
|
||||||
|
{
|
||||||
|
if ( e )
|
||||||
|
{
|
||||||
|
reject( new DaoError(
|
||||||
|
'Error creating index: ' + e
|
||||||
|
) );
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
createdCount++;
|
||||||
|
checkAllCreated();
|
||||||
|
};
|
||||||
|
|
||||||
|
// initialize indexes
|
||||||
|
collection.createIndex( [ ['published', 1] ], false, cb );
|
||||||
|
collection.createIndex( [ ['deltaError', 1] ], false, cb);
|
||||||
|
collection.createIndex(
|
||||||
|
[
|
||||||
|
[ 'published', 1 ],
|
||||||
|
[ 'deltaError', 1 ],
|
||||||
|
],
|
||||||
|
false,
|
||||||
|
cb
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
} );
|
} );
|
||||||
} );
|
} );
|
||||||
}
|
}
|
||||||
|
|
|
@ -371,8 +371,12 @@ describe( 'system.V1MessageWriter', () =>
|
||||||
},
|
},
|
||||||
document: {
|
document: {
|
||||||
id: doc_id,
|
id: doc_id,
|
||||||
created: startDate,
|
created: {
|
||||||
modified: lastUpdate,
|
'long': startDate * 1000
|
||||||
|
},
|
||||||
|
modified: {
|
||||||
|
'long': lastUpdate * 1000
|
||||||
|
},
|
||||||
},
|
},
|
||||||
session: {
|
session: {
|
||||||
Session: {
|
Session: {
|
||||||
|
|
Loading…
Reference in New Issue