employer
/
tame
1
0
Fork 0
Code Packages Releases Activity

Add list2typedef and map hash support

master v2.17.0
Mike Gerwitz 2018-05-29 15:54:59 -04:00
parent e3b0baec1a eef3eb85ea
commit 1fe35cd354
8 changed files with 298 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build-aux/list2typedef
View File

@ -66,7 +66,7 @@ output-item()
local name; read -r name
local -r const=$( constify "$typedef" <<< "$name" )
local -r value=$( hash-name "$bytes" <<< "$name" )
local -r value=$( echo -n "$name" | hash-name "$bytes" )
assert-collision "$const" "$value"

12
build-aux/test/test-list2typedef
View File

@ -41,9 +41,9 @@ Second'\''s @ @Line
title="FooType Type">
<typedef name="FooType" desc="FooType">
<enum type="integer">
<item name="FOOTYPE_FIRST" value="4285081399" desc="First" />
<item name="FOOTYPE_SECONDS_LINE" value="582715539" desc="Second'\''s @ @Line" />
<item name="FOOTYPE_THIRD" value="3355510722" desc="!!!THIRD!!!" />
<item name="FOOTYPE_FIRST" value="2706493105" desc="First" />
<item name="FOOTYPE_SECONDS_LINE" value="3512333918" desc="Second'\''s @ @Line" />
<item name="FOOTYPE_THIRD" value="519392729" desc="!!!THIRD!!!" />
</enum>
</typedef>
</package>'
@ -79,11 +79,11 @@ test-collision-value-check()
# we can easily force a collision by reducing the number of bytes to 1 and
# calculating hashes from /usr/share/dict/words; this is one example
local err=$( ../list2typedef Foo 1 2>&1 >/dev/null <<< 'abacist
abatis
local err=$( ../list2typedef Foo 1 2>&1 >/dev/null <<< 'abase
abate
' && echo 'EXPECTED FAILURE' )
[[ "$err" =~ ABATIS && "$err" =~ '44' ]] || {
[[ "$err" =~ ABATE && "$err" =~ '157' ]] || {
echo 'expecting useful error message for value collion' >&2
return 1
}

45
src/current/compiler/js.xsl
View File

@ -2,7 +2,7 @@
<!--
Compiles rater XML into JavaScript
Copyright (C) 2016, 2017 R-T Specialty, LLC.
Copyright (C) 2016, 2017, 2018 R-T Specialty, LLC.
This file is part of TAME.
@ -1954,8 +1954,51 @@
return ret;
}
/**
* Map each string in INPUT to uppercase
*
* @param {Array|string} input string
*
* @return {Array<number>|number} mapped value
*/
function map_method_uppercase( input )
{
if ( Array.isArray( input ) )
{
return input.map( map_method_uppercase );
}
return input.toUpperCase();
}
/**
* Map each string in INPUT to an integer
*
* An integer is constructed by taking the four higher-order bytes from
* the SHA256 hash of the input (corresponding to eight hexadecimal digits).
*
* @param {Array|string} input preimage
*
* @return {Array<number>|number} mapped value
*/
function map_method_hash( input )
{
if ( Array.isArray( input ) )
{
return input.map( map_method_hash );
}
const hash = sha256( input ).substr( 0, 8 );
return parseInt( hash, 16 );
}
]]>
</text>
<sequence select="unparsed-text(
concat( $__path-root, '/src/js/sha256.js' ) )" />
</template>
</stylesheet>

132
src/current/compiler/map.xsl
View File

@ -2,7 +2,7 @@
<!--
Compiles map fragments to produce a map from source data to a destination
Copyright (C) 2016 R-T Specialty, LLC.
Copyright (C) 2016, 2018 R-T Specialty, LLC.
This file is part of TAME.
@ -270,6 +270,36 @@
</template>
<!--
Get name of function associated with mapping method
Note that this expands to an empty string if no processing is
needed. Since functions are applied using parenthesis, this has the
effect of creating either a function application or a parenthesized
expression, the latter of which simply returns the expression untouched.
-->
<function name="lvmc:get-method-func" as="xs:string">
<param name="method" as="xs:string?" />
<choose>
<!-- default -->
<when test="not( $method ) or ( $method = 'translate' )">
<sequence select="''" />
</when>
<when test="$method = ( 'hash', 'uppercase' )">
<sequence select="concat( 'map_method_', $method )" />
</when>
<otherwise>
<message terminate="yes"
select="concat( 'error: unknown map method `',
$method, '''' )" />
</otherwise>
</choose>
</function>
<!--
Directly map an input to the output
-->
@ -752,27 +782,71 @@
<template match="lvm:map//lvm:from" mode="lvmc:compile" priority="2">
<variable name="nested" as="xs:boolean"
select="exists( ancestor::lvm:from )" />
<text>input['</text>
<value-of select="@name" />
<text>']</text>
<choose>
<when test="@index">
<text>[</text>
<value-of select="@index" />
<text>]</text>
</when>
<when test="$nested">
<text>[curindex]</text>
</when>
</choose>
<sequence select="lvmc:value-ref( . )" />
</template>
<function name="lvmc:value-ref" as="xs:string">
<param name="from" as="element( lvm:from )" />
<variable name="nested" as="xs:boolean"
select="exists( $from/ancestor::lvm:from )" />
<variable name="name" as="xs:string"
select="$from/@name" />
<variable name="index" as="xs:string?"
select="$from/@index" />
<!-- index reference, if applicable -->
<variable name="index-ref" as="xs:string"
select="if ( $index ) then
concat( '[', $index, ']' )
else
''" />
<!-- additional index, if nested within another from -->
<variable name="nested-ref" as="xs:string"
select="if ( $nested ) then
'[curindex]'
else
''" />
<!-- compiled reference, including index and nested -->
<variable name="ref" as="xs:string"
select="concat(
'input[''', $name, ''']', $index-ref, $nested-ref )" />
<!-- finally, wrap in any transformations -->
<sequence select="lvmc:transformation-wrap(
$ref, $from/ancestor::lvm:transform )" />
</function>
<function name="lvmc:transformation-wrap" as="xs:string">
<param name="value" as="xs:string" />
<param name="transform" as="element( lvm:transform )*" />
<!-- transformations (if any) as function applications -->
<variable name="transform-methods" as="xs:string*"
select="for $method in $transform/@method
return concat(
lvmc:get-method-func( $method ),
'(' )" />
<!-- closing parenthesis for each -->
<variable name="transform-close" as="xs:string*"
select="for $_ in $transform-methods
return ')'" />
<!-- wrap $ref in methods and closing parentheses -->
<sequence select="concat(
string-join( $transform-methods, '' ),
$value,
string-join( $transform-close, '' ) )" />
</function>
<template match="lvm:from/lvm:default"
mode="lvmc:compile" priority="5">
<param name="symtable" as="element( preproc:symtable )"
@ -794,7 +868,11 @@
</template>
<template match="lvm:map//lvm:from/lvm:translate" mode="lvmc:compile" priority="5">
<!--
Key/value mapping
-->
<template mode="lvmc:compile" priority="5"
match="lvm:map//lvm:from/lvm:translate[ @key ]">
<param name="type" as="xs:string" />
<text>case '</text>
@ -807,6 +885,18 @@
</template>
<!--
Skip transformations during initial encounter
Transformations are applied within certain contexts; let those contexts
apply them when ready.
-->
<template mode="lvmc:compile" priority="3"
match="lvm:map//lvm:transform">
<apply-templates mode="lvmc:compile" />
</template>
<template match="lvm:translate[ element() ]"
mode="lvmc:compile-translate" priority="5">
<param name="symtable" as="element( preproc:symtable )"
@ -838,6 +928,8 @@
<apply-templates mode="lvmc:compile" select=".">
<with-param name="type" select="$type" />
<with-param name="symtable" select="$symtable"
tunnel="yes"/>
</apply-templates>
</for-each>
<text>;</text>

10
src/current/include/dslc-base.xsl
View File

@ -29,6 +29,16 @@
xmlns="http://www.w3.org/1999/xhtml"
xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
<!--
Absolute path to root of TAME
Relative paths between files in XSLT can be flaky because most operations
are relative to the root stylesheet (not the filename of the stylesheet
that a particular line of code is executing in). Using absolute paths
mitigates that definitively.
-->
<xsl:param name="__path-root" />
<!--
Package source path, stripped of its extension

4
src/current/link.xsl
View File

@ -2,7 +2,7 @@
<!--
Entry point for linker
Copyright (C) 2016 R-T Specialty, LLC.
Copyright (C) 2016, 2018 R-T Specialty, LLC.
This file is part of TAME.
@ -24,6 +24,8 @@
xmlns="http://www.w3.org/1999/xhtml"
xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
<xsl:include href="include/dslc-base.xsl" />
<xsl:include href="compiler/linker.xsl" />
</xsl:stylesheet>

11
src/current/src/com/lovullo/dslc/DslCompiler.java
View File

@ -1,7 +1,7 @@
/**
* TAME compiler fontend
*
* Copyright (C) 2016 R-T Specialty, LLC.
* Copyright (C) 2016, 2018 R-T Specialty, LLC.
*
* This file is part of the Liza Data Collection Framework
*
@ -29,6 +29,8 @@
package com.lovullo.dslc;
import java.io.*;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Map;
import java.util.HashMap;
import javax.xml.XMLConstants;
@ -41,12 +43,15 @@ import org.xml.sax.SAXException;
import org.xml.sax.SAXNotRecognizedException;
import org.xml.sax.SAXNotSupportedException;
// TODO: Decouple from rater/ path assumptions
public class DslCompiler
{
private static class _DslCompiler
{
private Validator _xsd;
private HashMap<String,Transformer> _xsl;
private Path _pathRoot;
public _DslCompiler()
@ -86,6 +91,9 @@ public class DslCompiler
System.exit( 4 );
}
// root path of TAME
_pathRoot = Paths.get( "rater/tame" ).toRealPath();
// transform to dest
File destfile = new File( dest );
try
@ -134,6 +142,7 @@ public class DslCompiler
String relroot = new String( new char[ dircount ] ).replace( "\0", "../" );
Transformer t = _xsl.get( cmd );
t.setParameter( "__path-root", _pathRoot.toString() );
t.setParameter( "__srcpkg", srcpkg );
t.setParameter( "__relroot", relroot );
t.setParameter( "__rseed", (int)( Math.random() * 10e6 ) );

112
src/js/sha256.js 100644
View File

@ -0,0 +1,112 @@
/**
* Source: https://raw.githubusercontent.com/geraintluff/sha256/gh-pages/sha256.js
* This script is in the public domain.
* This comment was added by R-T Specialty.
*
* N.B.: THIS IMPLEMENTATION IS NOT INTENDED FOR SECURE CRYPTOGRAPHIC
* OPERATIONS. It is used as a seemingly faithful implementation of SHA256
* not as a cryptographically secure hash, but as, simply, a hash function
* with a uniform distribution. This allows truncating to any length while
* still maintaining that uniformity, which is important for avoiding
* collisions.
*
* So, repeat: DO NOT USE FOR SENSITIVE CRYPTOGRAPHIC OPERATIONS. This has
* not been audited.
*/
var sha256 = function sha256(ascii) {
function rightRotate(value, amount) {
return (value>>>amount) | (value<<(32 - amount));
};
var mathPow = Math.pow;
var maxWord = mathPow(2, 32);
var lengthProperty = 'length';
var i, j; // Used as a counter across the whole file
var result = '';
var words = [];
var asciiBitLength = ascii[lengthProperty]*8;
//* caching results is optional - remove/add slash from front of this line to toggle
// Initial hash value: first 32 bits of the fractional parts of the square roots of the first 8 primes
// (we actually calculate the first 64, but extra values are just ignored)
var hash = sha256.h = sha256.h || [];
// Round constants: first 32 bits of the fractional parts of the cube roots of the first 64 primes
var k = sha256.k = sha256.k || [];
var primeCounter = k[lengthProperty];
/*/
var hash = [], k = [];
var primeCounter = 0;
//*/
var isComposite = {};
for (var candidate = 2; primeCounter < 64; candidate++) {
if (!isComposite[candidate]) {
for (i = 0; i < 313; i += candidate) {
isComposite[i] = candidate;
}
hash[primeCounter] = (mathPow(candidate, .5)*maxWord)|0;
k[primeCounter++] = (mathPow(candidate, 1/3)*maxWord)|0;
}
}
ascii += '\x80'; // Append '1' bit (plus zero padding)
while (ascii[lengthProperty]%64 - 56) ascii += '\x00'; // More zero padding
for (i = 0; i < ascii[lengthProperty]; i++) {
j = ascii.charCodeAt(i);
if (j>>8) return; // ASCII check: only accept characters in range 0-255
words[i>>2] |= j << ((3 - i)%4)*8;
}
words[words[lengthProperty]] = ((asciiBitLength/maxWord)|0);
words[words[lengthProperty]] = (asciiBitLength)
// process each chunk
for (j = 0; j < words[lengthProperty];) {
var w = words.slice(j, j += 16); // The message is expanded into 64 words as part of the iteration
var oldHash = hash;
// This is now the "working hash", often labelled as variables a...g
// (we have to truncate as well, otherwise extra entries at the end accumulate
hash = hash.slice(0, 8);
for (i = 0; i < 64; i++) {
var i2 = i + j;
// Expand the message into 64 words
// Used below if
var w15 = w[i - 15], w2 = w[i - 2];
// Iterate
var a = hash[0], e = hash[4];
var temp1 = hash[7]
+ (rightRotate(e, 6) ^ rightRotate(e, 11) ^ rightRotate(e, 25)) // S1
+ ((e&hash[5])^((~e)&hash[6])) // ch
+ k[i]
// Expand the message schedule if needed
+ (w[i] = (i < 16) ? w[i] : (
w[i - 16]
+ (rightRotate(w15, 7) ^ rightRotate(w15, 18) ^ (w15>>>3)) // s0
+ w[i - 7]
+ (rightRotate(w2, 17) ^ rightRotate(w2, 19) ^ (w2>>>10)) // s1
)|0
);
// This is only used once, so *could* be moved below, but it only saves 4 bytes and makes things unreadble
var temp2 = (rightRotate(a, 2) ^ rightRotate(a, 13) ^ rightRotate(a, 22)) // S0
+ ((a&hash[1])^(a&hash[2])^(hash[1]&hash[2])); // maj
hash = [(temp1 + temp2)|0].concat(hash); // We don't bother trimming off the extra ones, they're harmless as long as we're truncating when we do the slice()
hash[4] = (hash[4] + temp1)|0;
}
for (i = 0; i < 8; i++) {
hash[i] = (hash[i] + oldHash[i])|0;
}
}
for (i = 0; i < 8; i++) {
for (j = 3; j + 1; j--) {
var b = (hash[i]>>(j*8))&255;
result += ((b < 16) ? 0 : '') + b.toString(16);
}
}
return result;
};