employer
/
tame
1
0
Fork 0
Code Packages Releases Activity

linker: >50% performance improvements in both time and space 

This has a significant performance impact: processing time is cut in about
half and memory usage is reduced by more than 50%.  For example, a package
that previously took 30s and 2.1GiB of memory to link now takes 14s and less
than 900MiB of memory.
master v2.17.4
Mike Gerwitz 2018-07-16 08:41:56 -04:00
parent 844c6512c9 ea7e0e1565
commit d1e2b435a0
3 changed files with 58 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
src/current/compiler/js.xsl
View File

@ -99,9 +99,8 @@
<!-- handle defaults --> <!-- handle defaults -->
<text>init_defaults( args, params );</text> <text>init_defaults( args, params );</text>
<!-- perform classifications -->
<value-of select="$compiler:nl" /> <value-of select="$compiler:nl" />
<text>var classes = rater.classify( args, _canterm );</text> <text>var classes = {};</text>
<!-- for @external generated clases --> <!-- for @external generated clases -->
<text>var genclasses = {};</text> <text>var genclasses = {};</text>
</template> </template>
@ -110,34 +109,23 @@
<!-- allow classification of any arbitrary dataset --> <!-- allow classification of any arbitrary dataset -->
<value-of select="$compiler:nl" /> <value-of select="$compiler:nl" />
<text>rater.classify = function( args, _canterm ) {</text> <text>rater.classify = function( args, _canterm ) {</text>
<text>_canterm = ( _canterm == undefined ) ? true : !!_canterm;</text> return rater( args, _canterm ).classes;
<text> };</text>
<!-- XXX: Remove this; shouldn't be magic -->
<text>consts['__DATE_YEAR__'] = ( new Date() ).getFullYear(); </text>
<!-- object into which all classifications will be stored -->
<text>var classes = {}, genclasses = {}; </text>
<!-- TODO: We need to do something with this... -->
<text>var req_params = {}; </text>
</template> </template>
<template match="lv:package" mode="compiler:exit-classifier"> <template match="lv:package" mode="compiler:exit-classifier">
<text>return classes;</text>
<text> };</text>
<!-- TODO: make sure fromMap has actually been compiled --> <!-- TODO: make sure fromMap has actually been compiled -->
<text>rater.classify.fromMap = function( args_base, _canterm ) { </text> <text>rater.classify.fromMap = function( args_base, _canterm ) { </text>
<text>var ret = {}; </text> <text>var ret = {}; </text>
<text>rater.fromMap( args_base, function( args ) {</text> <text>rater.fromMap( args_base, function( args ) {</text>
<text>var classes = rater.classify( args, _canterm ); </text> <text>var result = rater( args, _canterm ); </text>
<text> <text>
for ( var c in classes ) for ( var c in rater.classify.classmap )
{ {
ret[ c ] = { ret[ c ] = {
is: !!classes[ c ], is: !!result.classes[ c ],
indexes: args[ rater.classify.classmap[ c ] ] indexes: result.vars[ rater.classify.classmap[ c ] ]
}; };
} }
</text> </text>
@ -221,15 +209,15 @@
<!-- for each cgen symbol (which is the classification @yields), map the <!-- for each cgen symbol (which is the classification @yields), map the
classification name (the @parent) to the cgen symbol name --> classification name (the @parent) to the cgen symbol name -->
<for-each select="$symbols[ @type='cgen' ]"> <for-each select="$symbols[ @type='class' and not( @preproc:generated ) ]">
<if test="position() > 1"> <if test="position() > 1">
<text>,</text> <text>,</text>
</if> </if>
<text>'</text> <text>'</text>
<value-of select="substring-after( @parent, ':class:' )" /> <value-of select="substring-after( @name, ':class:' )" />
<text>':'</text> <text>':'</text>
<value-of select="@name" /> <value-of select="@yields" />
<text>'</text> <text>'</text>
</for-each> </for-each>
</template> </template>

198
src/current/compiler/linker.xsl
View File

@ -2,7 +2,7 @@
<!-- <!--
Assembles code fragments into a final executable Assembles code fragments into a final executable
Copyright (C) 2016, 2017 R-T Specialty, LLC. Copyright (C) 2016, 2017, 2018 R-T Specialty, LLC.
This file is part of TAME. This file is part of TAME.
@ -57,13 +57,11 @@
<variable name="l:orig-root" as="document-node( element( lv:package ) )" <variable name="l:orig-root" as="document-node( element( lv:package ) )"
select="/" /> select="/" />
<variable name="l:process-empty" as="element( l:pstack )"> <variable name="l:process-empty" as="element( preproc:sym )*"
<l:pstack /> select="()" />
</variable>
<variable name="l:stack-empty" as="element( l:sym-stack )"> <variable name="l:stack-empty" as="element( preproc:sym )*"
<l:sym-stack /> select="()" />
</variable>
<template match="*" mode="l:link" priority="1"> <template match="*" mode="l:link" priority="1">
@ -115,7 +113,7 @@
<l:dep> <l:dep>
<!-- empty stack --> <!-- empty stack -->
<apply-templates select="preproc:symtable" mode="l:depgen"> <apply-templates select="preproc:symtable" mode="l:depgen">
<with-param name="stack" select="$l:stack-empty" as="element( l:sym-stack )" /> <with-param name="stack" select="$l:stack-empty" />
</apply-templates> </apply-templates>
</l:dep> </l:dep>
</variable> </variable>
@ -175,8 +173,8 @@
<template mode="l:depgen" as="element( preproc:sym )*" <template mode="l:depgen" as="element( preproc:sym )*"
match="preproc:symtable"> match="preproc:symtable">
<param name="stack" as="element( l:sym-stack )" <param name="stack" as="element( preproc:sym )*"
select="$l:stack-empty" /> select="()" />
<!-- we care only of the symbols used by lv:yields, from which all <!-- we care only of the symbols used by lv:yields, from which all
dependencies may be derived (if it's not derivable from the yield dependencies may be derived (if it's not derivable from the yield
@ -209,7 +207,7 @@
<variable name="result" as="element()+"> <variable name="result" as="element()+">
<call-template name="l:depgen-sym"> <call-template name="l:depgen-sym">
<with-param name="pending" select="$yields" /> <with-param name="pending" select="$yields" />
<with-param name="stack" select="$stack" as="element( l:sym-stack )" /> <with-param name="stack" select="$stack" />
</call-template> </call-template>
</variable> </variable>
@ -228,75 +226,31 @@
</template> </template>
<!-- replace marks with the symbols that they reference (note that the linker
will not add duplicates, so we needn't worry about them) -->
<template mode="l:resolv-deps" as="element( preproc:sym )"
priority="8"
match="preproc:sym[ @l:mark-inclass ]">
<!-- FIXME: I sometimes return more than one symbol! -->
<variable name="sym" as="element( preproc:sym )*"
select="root(.)/preproc:sym[
@name = current()/@name ]" />
<!-- sanity check; hopefully never necessary -->
<if test="not( $sym )">
<call-template name="log:internal-error">
<with-param name="name" select="'link'" />
<with-param name="msg">
<text>l:mark-class found for non-existing symbol </text>
<value-of select="@name" />
<text>; there is a bug in l:depgen-process-sym</text>
</with-param>
</call-template>
</if>
<!-- copy the element and mark as inclass (no need to check @src, since at
this point, such conflicts would have already resulted in an error -->
<preproc:sym>
<sequence select="$sym/@*" />
<!-- override attribute -->
<attribute name="inclass" select="'true'" />
</preproc:sym>
</template>
<!-- any now-inclass symbols should be stripped of their original position -->
<template mode="l:resolv-deps" priority="7"
match="preproc:sym[
@name = root(.)/preproc:sym[ @l:mark-inclass ]
/@name ]">
<!-- bye-bye -->
</template>
<template match="*" mode="l:resolv-deps" priority="1"> <template match="*" mode="l:resolv-deps" priority="1">
<sequence select="." /> <sequence select="." />
</template> </template>
<!-- FIXME: I want element( l:preproc:sym ), but we also have <template name="l:depgen-sym" as="element( preproc:sym )*">
l:mark-inclass -->
<template name="l:depgen-sym" as="element()*">
<param name="pending" as="element( preproc:sym )*" /> <param name="pending" as="element( preproc:sym )*" />
<param name="stack" as="element( l:sym-stack )" /> <param name="stack" as="element( preproc:sym )*" />
<param name="path" as="xs:string" <param name="path" as="xs:string"
select="''" /> select="''" />
<param name="processing" as="element( l:pstack )" <param name="processing" as="element( preproc:sym )*"
select="$l:process-empty" /> select="$l:process-empty" />
<variable name="pend-count" as="xs:integer" <variable name="pend-count" as="xs:integer"
select="count( $pending )" /> select="count( $pending )" />
<variable name="stack-count" as="xs:integer" <variable name="stack-count" as="xs:integer"
select="count( $stack/preproc:sym )" /> select="count( $stack )" />
<variable name="process-count" as="xs:integer" <variable name="process-count" as="xs:integer"
select="count( $processing/* )" /> select="count( $processing )" />
<choose> <choose>
<!-- if there are no pending symbols left, then we are done; return the <!-- if there are no pending symbols left, then we are done; return the
stack --> stack -->
<when test="$pend-count = 0"> <when test="$pend-count = 0">
<sequence select="$stack/*" /> <sequence select="$stack" />
</when> </when>
@ -321,9 +275,9 @@
<text>r - </text> <text>r - </text>
<value-of select="$cur/@name" /> <value-of select="$cur/@name" />
<text> [s:: </text> <text> [s:: </text>
<value-of select="$stack/preproc:sym/@name" /> <value-of select="$stack/@name" />
<text> ::s] [r:: </text> <text> ::s] [r:: </text>
<value-of select="$processing/preproc:sym/@name" /> <value-of select="$processing/@name" />
<text>::r]</text> <text>::r]</text>
</with-param> </with-param>
</call-template> </call-template>
@ -405,9 +359,9 @@
--> -->
<template match="preproc:sym[ @extern='true' ]" mode="l:depgen-process-sym" priority="5"> <template match="preproc:sym[ @extern='true' ]" mode="l:depgen-process-sym" priority="5">
<param name="pending" as="element( preproc:sym )*" /> <param name="pending" as="element( preproc:sym )*" />
<param name="stack" as="element( l:sym-stack )" /> <param name="stack" as="element( preproc:sym )*" />
<param name="path" as="xs:string" /> <param name="path" as="xs:string" />
<param name="processing" as="element( l:pstack )" /> <param name="processing" as="element( preproc:sym )*" />
<variable name="cur" select="." /> <variable name="cur" select="." />
@ -451,7 +405,7 @@
<text>); pulled in by: </text> <text>); pulled in by: </text>
<!-- help the user figure out how this happened --> <!-- help the user figure out how this happened -->
<for-each select="$processing/preproc:sym"> <for-each select="$processing">
<if test="position() gt 1"> <if test="position() gt 1">
<text> - </text> <text> - </text>
</if> </if>
@ -477,7 +431,7 @@
<!-- use the resolved symbol in place of the original extern --> <!-- use the resolved symbol in place of the original extern -->
<apply-templates select="$eresolv-uniq" mode="l:depgen-process-sym"> <apply-templates select="$eresolv-uniq" mode="l:depgen-process-sym">
<with-param name="pending" select="$pending" /> <with-param name="pending" select="$pending" />
<with-param name="stack" select="$stack" as="element( l:sym-stack )" /> <with-param name="stack" select="$stack" />
<with-param name="path" select="$path" /> <with-param name="path" select="$path" />
<with-param name="processing" select="$processing" /> <with-param name="processing" select="$processing" />
</apply-templates> </apply-templates>
@ -487,29 +441,19 @@
<template mode="l:depgen-process-sym" priority="1" <template mode="l:depgen-process-sym" priority="1"
match="preproc:sym"> match="preproc:sym">
<param name="pending" as="element( preproc:sym )*" /> <param name="pending" as="element( preproc:sym )*" />
<param name="stack" as="element( l:sym-stack )" /> <param name="stack" as="element( preproc:sym )*" />
<param name="path" as="xs:string" /> <param name="path" as="xs:string" />
<param name="processing" as="element( l:pstack )" /> <param name="processing" as="element( preproc:sym )*" />
<variable name="cur" as="element( preproc:sym )" <variable name="cur" as="element( preproc:sym )"
select="." /> select="." />
<!-- determines if the compile destination for these dependencies will be
within the classifier; this is the case for all class dependencies
*unless* the class is external -->
<variable name="inclass" as="xs:boolean"
select="( ( $cur/@type='class' )
and not( $cur/@extclass='true' ) )
or $processing/preproc:sym[
@type='class'
and not( @extclass='true' ) ]" />
<!-- perform circular dependency check and blow up if found (we cannot choose <!-- perform circular dependency check and blow up if found (we cannot choose
a proper linking order without a correct dependency tree); the only a proper linking order without a correct dependency tree); the only
exception is if the circular dependency is a function, since that simply exception is if the circular dependency is a function, since that simply
implies recursion, which we can handle just fine --> implies recursion, which we can handle just fine -->
<variable name="circ" as="element( preproc:sym )*" <variable name="circ" as="element( preproc:sym )*"
select="$processing/preproc:sym[ select="$processing[
@name=$cur/@name @name=$cur/@name
and @src=$cur/@src ]" /> and @src=$cur/@src ]" />
@ -531,7 +475,7 @@
<call-template name="l:depgen-sym"> <call-template name="l:depgen-sym">
<with-param name="pending" select="remove( $pending, 1 )" /> <with-param name="pending" select="remove( $pending, 1 )" />
<with-param name="processing" select="$processing" /> <with-param name="processing" select="$processing" />
<with-param name="stack" select="$stack" as="element( l:sym-stack )" /> <with-param name="stack" select="$stack" />
</call-template> </call-template>
</when> </when>
@ -539,8 +483,7 @@
<!-- process; TODO: good refactoring point; large template --> <!-- process; TODO: good refactoring point; large template -->
<otherwise> <otherwise>
<variable name="existing" as="element( preproc:sym )*" <variable name="existing" as="element( preproc:sym )*"
select="$stack/preproc:sym[ select="$stack[ @name=$cur/@name ]" />
@name=$cur/@name ]" />
<!-- TODO: this uses @name instead of @src because of map import <!-- TODO: this uses @name instead of @src because of map import
paths; decide on one or the other --> paths; decide on one or the other -->
@ -567,28 +510,17 @@
</call-template> </call-template>
</if> </if>
<!-- determine if class already exists, but needs to be marked for
inclusion within the classifier -->
<variable name="needs-class-mark" as="xs:boolean"
select="$existing
and $inclass
and not( $existing/@inclass='true' )
and not( $stack/preproc:sym[
@l:mark-inclass
and @name=$cur/@name
and @src=$cur/@src ] )" />
<!-- continue with the remainder of the symbol list --> <!-- continue with the remainder of the symbol list -->
<call-template name="l:depgen-sym"> <call-template name="l:depgen-sym">
<with-param name="pending" select="remove( $pending, 1 )" /> <with-param name="pending" select="remove( $pending, 1 )" />
<with-param name="processing" select="$processing" /> <with-param name="processing" select="$processing" />
<with-param name="stack" as="element( l:sym-stack )"> <with-param name="stack" as="element( preproc:sym )*">
<!-- if this symbol already exists on the stack, then there is no use <!-- if this symbol already exists on the stack, then there is no use
re-adding it (note that we check both the symbol name and its source re-adding it (note that we check both the symbol name and its source
since symbols could very well share a name due to exporting rules) --> since symbols could very well share a name due to exporting rules) -->
<choose> <choose>
<when test="not( $existing ) or $needs-class-mark"> <when test="not( $existing )">
<!-- does this symbol have any dependencies? --> <!-- does this symbol have any dependencies? -->
<variable name="deps" as="element( preproc:sym )*"> <variable name="deps" as="element( preproc:sym )*">
<apply-templates select="$cur" mode="l:depgen-sym" /> <apply-templates select="$cur" mode="l:depgen-sym" />
@ -607,43 +539,27 @@
<call-template name="l:dep-aug"> <call-template name="l:dep-aug">
<with-param name="cur" select="$cur" /> <with-param name="cur" select="$cur" />
<with-param name="deps" select="$deps" /> <with-param name="deps" select="$deps" />
<with-param name="inclass" select="$inclass" />
<with-param name="mypath" select="$mypath" /> <with-param name="mypath" select="$mypath" />
</call-template> </call-template>
</variable> </variable>
<l:sym-stack> <!-- process the dependencies (note that this has the effect of
<!-- process the dependencies (note that this has the effect of outputting the existing stack as well, which is why we have
outputting the existing stack as well, which is why we have not yet done so) -->
not yet done so) --> <call-template name="l:depgen-sym">
<call-template name="l:depgen-sym"> <with-param name="pending" select="$deps-aug" />
<with-param name="pending" select="$deps-aug" /> <with-param name="stack" select="$stack" />
<with-param name="stack" select="$stack" as="element( l:sym-stack )" /> <with-param name="path" select="$mypath" />
<with-param name="path" select="$mypath" /> <with-param name="processing" as="element( preproc:sym )*">
<with-param name="processing" as="element( l:pstack )"> <sequence select="$processing" />
<l:pstack> <sequence select="$cur" />
<sequence select="$processing/*" /> </with-param>
<sequence select="$cur" /> </call-template>
</l:pstack>
</with-param>
</call-template>
<!-- finally, we can output ourself --> <!-- finally, we can output ourself -->
<choose> <preproc:sym>
<!-- existing symbol needs to be marked --> <sequence select="$cur/@*" />
<when test="$needs-class-mark"> </preproc:sym>
<preproc:sym l:mark-inclass="true" name="{$cur/@name}" src="{$cur/@src}" />
</when>
<!-- new symbol -->
<otherwise>
<preproc:sym>
<sequence select="$cur/@*" />
<attribute name="inclass" select="$inclass" />
</preproc:sym>
</otherwise>
</choose>
</l:sym-stack>
</when> </when>
@ -662,8 +578,6 @@
<template name="l:dep-aug" as="element( preproc:sym )*"> <template name="l:dep-aug" as="element( preproc:sym )*">
<param name="cur" as="element( preproc:sym )" /> <param name="cur" as="element( preproc:sym )" />
<param name="deps" as="element( preproc:sym )*" /> <param name="deps" as="element( preproc:sym )*" />
<param name="inclass" as="xs:boolean"
select="false()" />
<param name="proc-barrier" as="xs:boolean" <param name="proc-barrier" as="xs:boolean"
select="false()" /> select="false()" />
<param name="parent-name" as="xs:string" <param name="parent-name" as="xs:string"
@ -709,11 +623,6 @@
<!-- set new src path --> <!-- set new src path -->
<attribute name="src" select="$newsrc" /> <attribute name="src" select="$newsrc" />
<!-- flag for inclusion into classifier, if necessary -->
<if test="$inclass">
<attribute name="inclass" select="$inclass" />
</if>
<if test="$proc-barrier"> <if test="$proc-barrier">
<attribute name="l:proc-barrier" select="'true'" /> <attribute name="l:proc-barrier" select="'true'" />
</if> </if>
@ -844,7 +753,7 @@
<value-of select="concat( $cur/@src, '/', $cur/@name )" /> <value-of select="concat( $cur/@src, '/', $cur/@name )" />
<text>): </text> <text>): </text>
<for-each select="$stack//preproc:sym"> <for-each select="$stack">
<if test="position() > 1"> <if test="position() > 1">
<text> - </text> <text> - </text>
</if> </if>
@ -991,8 +900,6 @@
<template match="lv:package" mode="l:link-classifier"> <template match="lv:package" mode="l:link-classifier">
<param name="deps" />
<call-template name="log:info"> <call-template name="log:info">
<with-param name="name" select="'link'" /> <with-param name="name" select="'link'" />
<with-param name="msg"> <with-param name="msg">
@ -1002,17 +909,7 @@
<!-- link everything that shall be a part of the classifier --> <!-- link everything that shall be a part of the classifier -->
<apply-templates select="." mode="compiler:entry-classifier" /> <apply-templates select="." mode="compiler:entry-classifier" />
<apply-templates select="." mode="l:do-link"> <!-- TODO: get rid of me completely! -->
<with-param name="symbols" select="
$deps[
@inclass='true'
and not( @type='param' )
and not( @type='type' )
and not( @type='meta' )
and not( @type='worksheet' )
]
" />
</apply-templates>
<apply-templates select="." mode="compiler:exit-classifier" /> <apply-templates select="." mode="compiler:exit-classifier" />
</template> </template>
@ -1090,8 +987,7 @@
<apply-templates select="." mode="l:do-link"> <apply-templates select="." mode="l:do-link">
<with-param name="symbols" select=" <with-param name="symbols" select="
$deps[ $deps[
not( @inclass='true' ) not( @type='param' )
and not( @type='param' )
and not( @type='type' ) and not( @type='type' )
and not( @type='func' ) and not( @type='func' )
and not( @type='meta' ) and not( @type='meta' )

3
src/current/include/preproc/template.xsl
View File

@ -1353,8 +1353,7 @@
<xsl:function name="eseq:is-expandable" as="xs:boolean" <xsl:function name="eseq:is-expandable" as="xs:boolean">
override="yes">
<xsl:param name="node" as="node()" /> <xsl:param name="node" as="node()" />
<!-- TODO: what a mess; clean me up by changing the point at which <!-- TODO: what a mess; clean me up by changing the point at which