[DEV-5312] Create additional indexes and process less frequently

.env

@@ -14,4 +14,10 @@ PROM_PUSH_INTERVAL_MS=5000
 PROM_BUCKETS_START=0
 PROM_BUCKETS_WIDTH=10
 PROM_BUCKETS_COUNT=10
-PROCESS_INTERVAL_MS=2000
+LIZA_MONGODB_HA=
+LIZA_MONGODB_REPLSET=
+LIZA_MONGODB_HOST_A=
+LIZA_MONGODB_PORT_A=
+LIZA_MONGODB_HOST_B=
+LIZA_MONGODB_PORT_B=
+PROCESS_INTERVAL_MS=2000

bin/delta-processor.ts

@@ -48,7 +48,7 @@ const amqp_conf           = createAmqpConfig( process.env );
 const prom_conf           = createPrometheusConfig( process.env );
 const db_conf             = createMongoConfig( process.env );
 const db                  = createMongoDB( db_conf );
-const process_interval_ms = +( process.env.process_interval_ms || 2000 );
+const process_interval_ms = +( process.env.process_interval_ms || 10000 );
 const env                 = process.env.NODE_ENV || 'Unknown Environment';
 const emitter             = new EventEmitter();
 const log                 = new StandardLogger( console, ts_ctr, env );

src/system/PrometheusFactory.ts

@@ -62,7 +62,7 @@ export function createPrometheusConfig(
         hostname:         env.PROM_HOST,
         port:             +( env.PROM_PORT || 0 ),
         env:              process.env.NODE_ENV,
-        push_interval_ms: +( process.env.PROM_PUSH_INTERVAL_MS || 5000 ),
+        push_interval_ms: +( process.env.PROM_PUSH_INTERVAL_MS || 10000 ),
         buckets_start:    +( process.env.PROM_BUCKETS_START || 0 ),
         buckets_width:    +( process.env.PROM_BUCKETS_WIDTH || 10 ),
         buckets_count:    +( process.env.PROM_BUCKETS_COUNT || 10 ),

src/system/avro/V1MessageWriter.ts

@@ -101,6 +101,8 @@ export class V1MessageWriter implements MessageWriter
         const bucket_formatted   = this.setDataTypes( bucket );
         const ratedata_formatted = this.setDataTypes( ratedata );
         const event_id           = this.DELTA_MAP[ delta.type ];
+        const start_date_ms      = { "long": meta.startDate * 1000 };
+        const last_update_ms     = { "long": meta.lastUpdate * 1000 };
 
         return {
             event: {
@@ -111,8 +113,8 @@ export class V1MessageWriter implements MessageWriter
             },
             document: {
                 id:       meta.id,
-                created:  meta.startDate,
-                modified: meta.lastUpdate,
+                created:  start_date_ms,
+                modified: last_update_ms,
             },
             session: {
                 Session: {

src/system/db/MongoFactory.ts

@@ -29,9 +29,9 @@ import { DaoError } from '../../error/DaoError';
 
 
 const {
-    Db:          MongoDb,
-    Server:      MongoServer,
-    ReplServers: ReplSetServers,
+    Db:             MongoDb,
+    Server:         MongoServer,
+    ReplSetServers: ReplSetServers,
 } = require( 'mongodb' );
 
 
@@ -148,29 +148,42 @@ export function getMongoCollection(
                         return;
                     }
 
-                    // initialize indexes
-                    collection.createIndex(
-                        [
-                            ['published',  1],
-                            ['deltaError', 1],
-                        ],
-                        true,
-                        ( e: any, _index: { [P: string]: any } ) =>
+                    let createdCount = 0
+                    const checkAllCreated = (): void =>
+                    {
+                        if( createdCount >= 3 )
                         {
-                            if ( e )
-                            {
-                                reject( new DaoError(
-                                    'Error creating index: ' + e
-                                ) );
-                                return;
-                            }
-
                             resolve( collection );
+                        }
+                    };
+
+                    const cb = ( e: any, _index: { [P: string]: any } ): void =>
+                    {
+                        if ( e )
+                        {
+                            reject( new DaoError(
+                                'Error creating index: ' + e
+                            ) );
                             return;
                         }
+
+                        createdCount++;
+                        checkAllCreated();
+                    };
+
+                    // initialize indexes
+                    collection.createIndex( [ ['published', 1] ], false, cb );
+                    collection.createIndex( [ ['deltaError', 1] ], false, cb);
+                    collection.createIndex(
+                        [
+                            [ 'published', 1 ],
+                            [ 'deltaError', 1 ],
+                        ],
+                        false,
+                        cb
                     );
                 }
             );
         } );
     } );
-}
+}

test/system/V1MessageWriterTest.ts

@@ -371,8 +371,12 @@ describe( 'system.V1MessageWriter', () =>
             },
             document: {
                 id:       doc_id,
-                created:  startDate,
-                modified: lastUpdate,
+                created:  {
+                    'long': startDate * 1000
+                },
+                modified: {
+                    'long': lastUpdate * 1000
+                },
             },
             session: {
                 Session: {