From fafbf3921eae6c53539f4c5de24e801365ec4af3 Mon Sep 17 00:00:00 2001
From: Mike Gerwitz <mike.gerwitz@rtspecialty.com>
Date: Tue, 15 May 2018 16:47:21 -0400
Subject: [PATCH 1/4] list2typedef: Remove trailing newline from preimage

Oops---we do not want a newline as part of the preimage, which is what
`<<<' does.

* build-aux/list2typedef (output-item)[value]: Remove newline from preimage.
* build-aux/test/test-list2typedef (test-typedef-gen): Update hashes.
  (test-collision-value-check): Add new collisions.
---
 build-aux/list2typedef           |  2 +-
 build-aux/test/test-list2typedef | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/build-aux/list2typedef b/build-aux/list2typedef
index fdb5c59a..3a18cf39 100755
--- a/build-aux/list2typedef
+++ b/build-aux/list2typedef
@@ -66,7 +66,7 @@ output-item()
   local name; read -r name
 
   local -r const=$( constify "$typedef" <<< "$name" )
-  local -r value=$( hash-name "$bytes" <<< "$name" )
+  local -r value=$( echo -n "$name" | hash-name "$bytes" )
 
   assert-collision "$const" "$value"
 
diff --git a/build-aux/test/test-list2typedef b/build-aux/test/test-list2typedef
index e417c022..62a5bd8d 100755
--- a/build-aux/test/test-list2typedef
+++ b/build-aux/test/test-list2typedef
@@ -41,9 +41,9 @@ Second'\''s @     @Line
          title="FooType Type">
   <typedef name="FooType" desc="FooType">
     <enum type="integer">
-      <item name="FOOTYPE_FIRST" value="4285081399" desc="First" />
-      <item name="FOOTYPE_SECONDS_LINE" value="582715539" desc="Second'\''s @     @Line" />
-      <item name="FOOTYPE_THIRD" value="3355510722" desc="!!!THIRD!!!" />
+      <item name="FOOTYPE_FIRST" value="2706493105" desc="First" />
+      <item name="FOOTYPE_SECONDS_LINE" value="3512333918" desc="Second'\''s @     @Line" />
+      <item name="FOOTYPE_THIRD" value="519392729" desc="!!!THIRD!!!" />
     </enum>
   </typedef>
 </package>'
@@ -79,11 +79,11 @@ test-collision-value-check()
 
   # we can easily force a collision by reducing the number of bytes to 1 and
   # calculating hashes from /usr/share/dict/words; this is one example
-  local err=$( ../list2typedef Foo 1 2>&1 >/dev/null <<< 'abacist
-abatis
+  local err=$( ../list2typedef Foo 1 2>&1 >/dev/null <<< 'abase
+abate
 ' && echo 'EXPECTED FAILURE' )
 
-  [[ "$err" =~ ABATIS && "$err" =~ '44' ]] || {
+  [[ "$err" =~ ABATE && "$err" =~ '157' ]] || {
     echo 'expecting useful error message for value collion' >&2
     return 1
   }

From 2a9196d9408304567f7501f7d692be3b42cf702f Mon Sep 17 00:00:00 2001
From: Mike Gerwitz <mike.gerwitz@rtspecialty.com>
Date: Tue, 29 May 2018 10:50:31 -0400
Subject: [PATCH 2/4] Add __path-root param

* src/current/include/dslc-base.xsl (__path-root): New param.
* src/current/src/com/lovullo/dslc/DslCompiler.java
  (DslCompiler)[compile]: Resolve TAME root path.
    [_transform]: Set it.

DEV-3115
---
 src/current/include/dslc-base.xsl                 | 10 ++++++++++
 src/current/src/com/lovullo/dslc/DslCompiler.java | 11 ++++++++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/src/current/include/dslc-base.xsl b/src/current/include/dslc-base.xsl
index b7a0447a..ab501e7e 100644
--- a/src/current/include/dslc-base.xsl
+++ b/src/current/include/dslc-base.xsl
@@ -29,6 +29,16 @@
   xmlns="http://www.w3.org/1999/xhtml"
   xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
 
+<!--
+  Absolute path to root of TAME
+
+  Relative paths between files in XSLT can be flaky because most operations
+  are relative to the root stylesheet (not the filename of the stylesheet
+  that a particular line of code is executing in).  Using absolute paths
+  mitigates that definitively.
+-->
+<xsl:param name="__path-root" />
+
 
 <!--
   Package source path, stripped of its extension
diff --git a/src/current/src/com/lovullo/dslc/DslCompiler.java b/src/current/src/com/lovullo/dslc/DslCompiler.java
index 4958787f..0c33abfb 100644
--- a/src/current/src/com/lovullo/dslc/DslCompiler.java
+++ b/src/current/src/com/lovullo/dslc/DslCompiler.java
@@ -1,7 +1,7 @@
 /**
  * TAME compiler fontend
  *
- *  Copyright (C) 2016 R-T Specialty, LLC.
+ *  Copyright (C) 2016, 2018 R-T Specialty, LLC.
  *
  *  This file is part of the Liza Data Collection Framework
  *
@@ -29,6 +29,8 @@
 package com.lovullo.dslc;
 
 import java.io.*;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Map;
 import java.util.HashMap;
 import javax.xml.XMLConstants;
@@ -41,12 +43,15 @@ import org.xml.sax.SAXException;
 import org.xml.sax.SAXNotRecognizedException;
 import org.xml.sax.SAXNotSupportedException;
 
+
+// TODO: Decouple from rater/ path assumptions
 public class DslCompiler
 {
     private static class _DslCompiler
     {
         private Validator _xsd;
         private HashMap<String,Transformer> _xsl;
+        private Path _pathRoot;
 
 
         public _DslCompiler()
@@ -86,6 +91,9 @@ public class DslCompiler
                 System.exit( 4 );
             }
 
+            // root path of TAME
+            _pathRoot = Paths.get( "rater/tame" ).toRealPath();
+
             // transform to dest
             File destfile = new File( dest );
             try
@@ -134,6 +142,7 @@ public class DslCompiler
             String relroot   = new String( new char[ dircount ] ).replace( "\0", "../" );
 
             Transformer t = _xsl.get( cmd );
+            t.setParameter( "__path-root", _pathRoot.toString() );
             t.setParameter( "__srcpkg", srcpkg );
             t.setParameter( "__relroot", relroot );
             t.setParameter( "__rseed", (int)( Math.random() * 10e6 ) );

From 3f7e1fc9b8728081604df41a7401b782ae4f8eba Mon Sep 17 00:00:00 2001
From: Mike Gerwitz <mike.gerwitz@rtspecialty.com>
Date: Tue, 29 May 2018 10:53:48 -0400
Subject: [PATCH 3/4] map: Add transformation support

* src/current/compiler/map.xsl
  (lvmc:get-method-func, lvmc:value-ref, lvmc:transformation-wrap): New
    functions, partyl extracted from below.
  (lvmc:compile)[lvm:map//lvm:from]: Use `lvmc:value-ref'.
    [lvm:map//lvm:from/lvm:translate]: Add `[@key]' to match.
    [lvm:map//lvm:transform]: New match.  Ignore node entirely.
  (lvmc:concat-compile): Propagate symtable to `lvmc:compile'.
---
 src/current/compiler/map.xsl | 132 +++++++++++++++++++++++++++++------
 1 file changed, 112 insertions(+), 20 deletions(-)

diff --git a/src/current/compiler/map.xsl b/src/current/compiler/map.xsl
index 8a318ab9..b36c7129 100644
--- a/src/current/compiler/map.xsl
+++ b/src/current/compiler/map.xsl
@@ -2,7 +2,7 @@
 <!--
   Compiles map fragments to produce a map from source data to a destination
 
-  Copyright (C) 2016 R-T Specialty, LLC.
+  Copyright (C) 2016, 2018 R-T Specialty, LLC.
 
     This file is part of TAME.
 
@@ -270,6 +270,36 @@
 </template>
 
 
+<!--
+  Get name of function associated with mapping method
+
+  Note that this expands to an empty string if no processing is
+  needed.  Since functions are applied using parenthesis, this has the
+  effect of creating either a function application or a parenthesized
+  expression, the latter of which simply returns the expression untouched.
+-->
+<function name="lvmc:get-method-func" as="xs:string">
+  <param name="method" as="xs:string?" />
+
+  <choose>
+    <!-- default -->
+    <when test="not( $method ) or ( $method = 'translate' )">
+      <sequence select="''" />
+    </when>
+
+    <when test="$method = ( 'hash', 'uppercase' )">
+      <sequence select="concat( 'map_method_', $method )" />
+    </when>
+
+    <otherwise>
+      <message terminate="yes"
+               select="concat( 'error: unknown map method `',
+                               $method, '''' )" />
+    </otherwise>
+  </choose>
+</function>
+
+
 <!--
   Directly map an input to the output
 -->
@@ -752,27 +782,71 @@
 
 
 <template match="lvm:map//lvm:from" mode="lvmc:compile" priority="2">
-  <variable name="nested" as="xs:boolean"
-            select="exists( ancestor::lvm:from )" />
-
-  <text>input['</text>
-    <value-of select="@name" />
-  <text>']</text>
-
-  <choose>
-    <when test="@index">
-      <text>[</text>
-      <value-of select="@index" />
-      <text>]</text>
-    </when>
-
-    <when test="$nested">
-      <text>[curindex]</text>
-    </when>
-  </choose>
+  <sequence select="lvmc:value-ref( . )" />
 </template>
 
 
+<function name="lvmc:value-ref" as="xs:string">
+  <param name="from" as="element( lvm:from )" />
+
+  <variable name="nested" as="xs:boolean"
+            select="exists( $from/ancestor::lvm:from )" />
+
+  <variable name="name" as="xs:string"
+            select="$from/@name" />
+  <variable name="index" as="xs:string?"
+            select="$from/@index" />
+
+
+  <!-- index reference, if applicable -->
+  <variable name="index-ref" as="xs:string"
+            select="if ( $index ) then
+                        concat( '[', $index, ']' )
+                      else
+                        ''" />
+
+  <!-- additional index, if nested within another from -->
+  <variable name="nested-ref" as="xs:string"
+            select="if ( $nested ) then
+                        '[curindex]'
+                      else
+                        ''" />
+
+  <!-- compiled reference, including index and nested -->
+  <variable name="ref" as="xs:string"
+            select="concat(
+                      'input[''', $name, ''']', $index-ref, $nested-ref )" />
+
+  <!-- finally, wrap in any transformations -->
+  <sequence select="lvmc:transformation-wrap(
+                      $ref, $from/ancestor::lvm:transform )" />
+</function>
+
+
+<function name="lvmc:transformation-wrap" as="xs:string">
+  <param name="value"     as="xs:string" />
+  <param name="transform" as="element( lvm:transform )*" />
+
+<!-- transformations (if any) as function applications -->
+  <variable name="transform-methods" as="xs:string*"
+            select="for $method in $transform/@method
+                      return concat(
+                        lvmc:get-method-func( $method ),
+                        '(' )" />
+
+  <!-- closing parenthesis for each -->
+  <variable name="transform-close" as="xs:string*"
+            select="for $_ in $transform-methods
+                      return ')'" />
+
+  <!-- wrap $ref in methods and closing parentheses -->
+  <sequence select="concat(
+                      string-join( $transform-methods, '' ),
+                      $value,
+                      string-join( $transform-close, '' ) )" />
+</function>
+
+
 <template match="lvm:from/lvm:default"
           mode="lvmc:compile" priority="5">
   <param name="symtable" as="element( preproc:symtable )"
@@ -794,7 +868,11 @@
 </template>
 
 
-<template match="lvm:map//lvm:from/lvm:translate" mode="lvmc:compile" priority="5">
+<!--
+  Key/value mapping
+-->
+<template mode="lvmc:compile" priority="5"
+          match="lvm:map//lvm:from/lvm:translate[ @key ]">
   <param name="type" as="xs:string" />
 
   <text>case '</text>
@@ -807,6 +885,18 @@
 </template>
 
 
+<!--
+  Skip transformations during initial encounter
+
+  Transformations are applied within certain contexts; let those contexts
+  apply them when ready.
+-->
+<template mode="lvmc:compile" priority="3"
+          match="lvm:map//lvm:transform">
+  <apply-templates mode="lvmc:compile" />
+</template>
+
+
 <template match="lvm:translate[ element() ]"
               mode="lvmc:compile-translate" priority="5">
   <param name="symtable" as="element( preproc:symtable )"
@@ -838,6 +928,8 @@
 
         <apply-templates mode="lvmc:compile" select=".">
           <with-param name="type" select="$type" />
+          <with-param name="symtable" select="$symtable"
+                      tunnel="yes"/>
         </apply-templates>
       </for-each>
     <text>;</text>

From eef3eb85eabca531424d217068a4a919b12d5681 Mon Sep 17 00:00:00 2001
From: Mike Gerwitz <mike.gerwitz@rtspecialty.com>
Date: Tue, 29 May 2018 11:16:59 -0400
Subject: [PATCH 4/4] map: Uppercase and hash transformers

This includes a SHA256 implementation which is _not_ intended for secure
cryptographic operations; see src/js/sha256.js header for more information.

* src/current/compiler/js.xsl (compiler:static): Echo src/js/sha256.js.
  [map_method_uppercase, map_method_hash]: New functions.
* src/current/link.xsl: Include dslc-base.xsl.
* src/js/sha256.js: New file.
---
 src/current/compiler/js.xsl |  45 ++++++++++++++-
 src/current/link.xsl        |   4 +-
 src/js/sha256.js            | 112 ++++++++++++++++++++++++++++++++++++
 3 files changed, 159 insertions(+), 2 deletions(-)
 create mode 100644 src/js/sha256.js

diff --git a/src/current/compiler/js.xsl b/src/current/compiler/js.xsl
index 78afb0c4..cfc5ccfa 100644
--- a/src/current/compiler/js.xsl
+++ b/src/current/compiler/js.xsl
@@ -2,7 +2,7 @@
 <!--
   Compiles rater XML into JavaScript
 
-  Copyright (C) 2016, 2017 R-T Specialty, LLC.
+  Copyright (C) 2016, 2017, 2018 R-T Specialty, LLC.
 
     This file is part of TAME.
 
@@ -1954,8 +1954,51 @@
 
         return ret;
     }
+
+
+    /**
+     * Map each string in INPUT to uppercase
+     *
+     * @param {Array|string} input string
+     *
+     * @return {Array<number>|number} mapped value
+     */
+    function map_method_uppercase( input )
+    {
+        if ( Array.isArray( input ) )
+        {
+            return input.map( map_method_uppercase );
+        }
+
+        return input.toUpperCase();
+    }
+
+
+    /**
+     * Map each string in INPUT to an integer
+     *
+     * An integer is constructed by taking the four higher-order bytes from
+     * the SHA256 hash of the input (corresponding to eight hexadecimal digits).
+     *
+     * @param {Array|string} input preimage
+     *
+     * @return {Array<number>|number} mapped value
+     */
+    function map_method_hash( input )
+    {
+        if ( Array.isArray( input ) )
+        {
+            return input.map( map_method_hash );
+        }
+
+        const hash = sha256( input ).substr( 0, 8 );
+        return parseInt( hash, 16 );
+    }
 ]]>
 </text>
+
+<sequence select="unparsed-text(
+                    concat( $__path-root, '/src/js/sha256.js' ) )" />
 </template>
 
 </stylesheet>
diff --git a/src/current/link.xsl b/src/current/link.xsl
index b46cfefe..ac880f91 100644
--- a/src/current/link.xsl
+++ b/src/current/link.xsl
@@ -2,7 +2,7 @@
 <!--
   Entry point for linker
 
-  Copyright (C) 2016 R-T Specialty, LLC.
+  Copyright (C) 2016, 2018 R-T Specialty, LLC.
 
     This file is part of TAME.
 
@@ -24,6 +24,8 @@
   xmlns="http://www.w3.org/1999/xhtml"
   xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
 
+<xsl:include href="include/dslc-base.xsl" />
+
 <xsl:include href="compiler/linker.xsl" />
 
 </xsl:stylesheet>
diff --git a/src/js/sha256.js b/src/js/sha256.js
new file mode 100644
index 00000000..15021006
--- /dev/null
+++ b/src/js/sha256.js
@@ -0,0 +1,112 @@
+/**
+ * Source: https://raw.githubusercontent.com/geraintluff/sha256/gh-pages/sha256.js
+ * This script is in the public domain.
+ * This comment was added by R-T Specialty.
+ *
+ * N.B.: THIS IMPLEMENTATION IS NOT INTENDED FOR SECURE CRYPTOGRAPHIC
+ * OPERATIONS.  It is used as a seemingly faithful implementation of SHA256
+ * not as a cryptographically secure hash, but as, simply, a hash function
+ * with a uniform distribution.  This allows truncating to any length while
+ * still maintaining that uniformity, which is important for avoiding
+ * collisions.
+ *
+ * So, repeat: DO NOT USE FOR SENSITIVE CRYPTOGRAPHIC OPERATIONS.  This has
+ * not been audited.
+ */
+
+var sha256 = function sha256(ascii) {
+	function rightRotate(value, amount) {
+		return (value>>>amount) | (value<<(32 - amount));
+	};
+	
+	var mathPow = Math.pow;
+	var maxWord = mathPow(2, 32);
+	var lengthProperty = 'length';
+	var i, j; // Used as a counter across the whole file
+	var result = '';
+
+	var words = [];
+	var asciiBitLength = ascii[lengthProperty]*8;
+	
+	//* caching results is optional - remove/add slash from front of this line to toggle
+	// Initial hash value: first 32 bits of the fractional parts of the square roots of the first 8 primes
+	// (we actually calculate the first 64, but extra values are just ignored)
+	var hash = sha256.h = sha256.h || [];
+	// Round constants: first 32 bits of the fractional parts of the cube roots of the first 64 primes
+	var k = sha256.k = sha256.k || [];
+	var primeCounter = k[lengthProperty];
+	/*/
+	var hash = [], k = [];
+	var primeCounter = 0;
+	//*/
+
+	var isComposite = {};
+	for (var candidate = 2; primeCounter < 64; candidate++) {
+		if (!isComposite[candidate]) {
+			for (i = 0; i < 313; i += candidate) {
+				isComposite[i] = candidate;
+			}
+			hash[primeCounter] = (mathPow(candidate, .5)*maxWord)|0;
+			k[primeCounter++] = (mathPow(candidate, 1/3)*maxWord)|0;
+		}
+	}
+	
+	ascii += '\x80'; // Append '1' bit (plus zero padding)
+	while (ascii[lengthProperty]%64 - 56) ascii += '\x00'; // More zero padding
+	for (i = 0; i < ascii[lengthProperty]; i++) {
+		j = ascii.charCodeAt(i);
+		if (j>>8) return; // ASCII check: only accept characters in range 0-255
+		words[i>>2] |= j << ((3 - i)%4)*8;
+	}
+	words[words[lengthProperty]] = ((asciiBitLength/maxWord)|0);
+	words[words[lengthProperty]] = (asciiBitLength)
+	
+	// process each chunk
+	for (j = 0; j < words[lengthProperty];) {
+		var w = words.slice(j, j += 16); // The message is expanded into 64 words as part of the iteration
+		var oldHash = hash;
+		// This is now the "working hash", often labelled as variables a...g
+		// (we have to truncate as well, otherwise extra entries at the end accumulate
+		hash = hash.slice(0, 8);
+		
+		for (i = 0; i < 64; i++) {
+			var i2 = i + j;
+			// Expand the message into 64 words
+			// Used below if 
+			var w15 = w[i - 15], w2 = w[i - 2];
+
+			// Iterate
+			var a = hash[0], e = hash[4];
+			var temp1 = hash[7]
+				+ (rightRotate(e, 6) ^ rightRotate(e, 11) ^ rightRotate(e, 25)) // S1
+				+ ((e&hash[5])^((~e)&hash[6])) // ch
+				+ k[i]
+				// Expand the message schedule if needed
+				+ (w[i] = (i < 16) ? w[i] : (
+						w[i - 16]
+						+ (rightRotate(w15, 7) ^ rightRotate(w15, 18) ^ (w15>>>3)) // s0
+						+ w[i - 7]
+						+ (rightRotate(w2, 17) ^ rightRotate(w2, 19) ^ (w2>>>10)) // s1
+					)|0
+				);
+			// This is only used once, so *could* be moved below, but it only saves 4 bytes and makes things unreadble
+			var temp2 = (rightRotate(a, 2) ^ rightRotate(a, 13) ^ rightRotate(a, 22)) // S0
+				+ ((a&hash[1])^(a&hash[2])^(hash[1]&hash[2])); // maj
+			
+			hash = [(temp1 + temp2)|0].concat(hash); // We don't bother trimming off the extra ones, they're harmless as long as we're truncating when we do the slice()
+			hash[4] = (hash[4] + temp1)|0;
+		}
+		
+		for (i = 0; i < 8; i++) {
+			hash[i] = (hash[i] + oldHash[i])|0;
+		}
+	}
+	
+	for (i = 0; i < 8; i++) {
+		for (j = 3; j + 1; j--) {
+			var b = (hash[i]>>(j*8))&255;
+			result += ((b < 16) ? 0 : '') + b.toString(16);
+		}
+	}
+	return result;
+};