client: Truncate diff posted to server after first null

Before this change, since `undefined' is encoded as `null' when serialized,
there was no way for the server to disambiguate between unmodified values
and a truncation point.  For example:

  [ undefined, undefined, null, null, null ]

The above array represents two unmodified and three removed indexes.  But
this is serialzed into JSON as:

  [ null, null, null, null, null ]

It isn't possible for the server to determine what the truncation point is
from that diff.  The solution is to therefore truncate the array _before_
sending it to the server, providing a trailing null to indicate that a
truncation has occurred:

  [ null, null, null ]

The above means that the first two indexes are unmodified, and that index 2
and later should all be truncated.

* doc/client.texi (Saving to Server): New section.
* src/client/transport/XhttpQuoteTransport.js (_truncateDiff): New method to
  perform truncation.
  (getBucketDataJson): Use it.
* test/client/transport/XhttpQuoteTransportTest.js: New file with respective
  test case.

doc/client.texi

@@ -25,6 +25,7 @@
 
 @menu
 * Error Handling::
+* Saving to Server::  Posting bucket diff to the Server.
 @end menu
 
 
@@ -96,3 +97,44 @@ When any field or classification changes that is represented on the
 
 Error state is managed by
   @srcref{src/validate/ValidStateMonitor.js, ValidStateMonitor}.
+
+
+
+@node Saving to Server
+@section Saving to Server
+@helpwanted
+
+@cindex Saving
+@cindex Bucket Diff
+@cindex Bucket Truncation
+@cindex Serialization
+To save changes,
+  the client posts only the bucket diff (@pxref{Bucket Diff}) to the
+  Server (@pxref{Server}).
+Because JSON serialization encodes @code{undefined} values as @code{null}
+  (as noted in @ref{Bucket Diff}),
+    and only the null in the tail position marks the truncation point,
+    the Client first truncates the array to include only the first
+      @code{null}.@footnote{
+        The server would otherwise remove only the last index,
+          even if multiple indexes were removed.}
+An example is shown in @ref{f:client-diff}.
+
+@float Figure, f:client-diff
+@example
+  // given (two unchanged, three removed)
+  [ undefined, undefined, null, null, null ]
+
+  // encodes into JSON as (bad; represents four unchanged, one removed)
+  [ null, null, null, null, null ]
+
+  // Client truncates to (two unchanged, >=2 removed)
+  [ null, null, null ]
+@end example
+@caption{Client diff truncation}
+@end float
+
+This conversion is handled by
+  @srcrefjs{client/transport,XhttpQuoteTransport}.
+Examples can be found in the respective test case
+  @testrefjs{client/transport,XhttpQuoteTransport}.

src/client/transport/XhttpQuoteTransport.js

@@ -96,6 +96,11 @@ module.exports = Class( 'XhttpQuoteTransport' )
     /**
      * Retrieve bucket data in JSON format
      *
+     * The serialized data will have the arrays truncated at the position of
+     * the first `null`; since all `undefined` values are serialized as
+     * `"null"`, there is otherwise no way to disambiguate a truncation
+     * point.
+     *
      * Allows subtypes to override what data is retrieved from the bucket
      *
      * @param {Bucket} bucket bucket from which to retrieve data
@@ -106,9 +111,47 @@ module.exports = Class( 'XhttpQuoteTransport' )
     {
         // get a "filled" diff containing the merged values of only the fields
         // that have changed
-        var data = bucket.getFilledDiff();
+        const raw_data = bucket.getFilledDiff();
+
+        // truncated data to serialize
+        const data = {};
+
+        Object.keys( raw_data ).forEach( field =>
+            data[ field ] = this._truncateDiff( raw_data[ field ] )
+        );
 
         return JSON.stringify( data );
+    },
+
+
+    /**
+     * Truncate just after first null
+     *
+     * If there are no nulls, then the diff is returned
+     * unmodified.  Otherwise, the array is truncated at the index of the
+     * first `null` (so that a trailing `null` still exists).
+     *
+     * WARNING: This modifies DIFF; it does not return a copy!
+     *
+     * @param {Array} diff bucket diff
+     *
+     * @return {Array} possibly truncated diff
+     */
+    'private _truncateDiff'( diff )
+    {
+        const null_i = diff.findIndex( x => x === null );
+
+        // no nulls, retain as-is
+        if ( null_i === -1 )
+        {
+            return diff;
+        }
+
+        // truncate following the first null (indicating that we terminate at
+        // this position)
+        diff.length = null_i + 1;
+
+        return diff;
     }
 } );
 

test/client/transport/XhttpQuoteTransportTest.js

@@ -0,0 +1,75 @@
+/**
+ * Test case for XhttpQuoteTransport
+ *
+ *  Copyright (C) 2018 R-T Specialty, LLC.
+ *
+ *  This file is part of the Liza Data Collection Framework
+ *
+ *  Liza is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+"use strict";
+
+const expect = require( 'chai' ).expect;
+const Class  = require( 'easejs' ).Class;
+
+const { XhttpQuoteTransport: Sut } = require( '../../../' ).client.transport;
+
+
+describe( "XhttpQuoteTransport", () =>
+{
+    it( "truncates index removals", done =>
+    {
+        // before truncating
+        const bucket = { getFilledDiff: () => ( {
+            none:           [ 'no', 'truncate' ],
+            empty:          [],
+            one_null:       [ null ],
+            all_null:       [ null, null, null ],
+            tail_null:      [ 'a', 'b', null ],
+            tail_null_many: [ 'a', 'b', null, null, null ],
+            undefined_null: [ undefined, 'b', undefined, null, null ],
+
+            // this shouldn't ever happen, but let's make sure the behavior
+            // is sane anyway
+            bs: [ null, 'should', 'not', 'ever', 'happen' ],
+        } ) };
+
+        // after truncating
+        const expected_data = {
+            none:           [ 'no', 'truncate' ],
+            empty:          [],
+            one_null:       [ null ],
+            all_null:       [ null ],
+            tail_null:      [ 'a', 'b', null ],
+            tail_null_many: [ 'a', 'b', null ],
+            undefined_null: [ null, 'b', null, null ],
+            bs:             [ null ],
+        };
+
+        const stub_quote  = { visitData: c => c( bucket ) };
+
+        const mock_proxy = {
+            post( _, data )
+            {
+                expect( JSON.parse( data.data ) )
+                    .to.deep.equal( expected_data );
+
+                done();
+            },
+        };
+
+        Sut( '', mock_proxy ).send( stub_quote );
+    } );
+} );