From f2db9f12682476437fe60c4a63c6d4e93e8f2258 Mon Sep 17 00:00:00 2001
From: Mike Gerwitz <mike.gerwitz@rtspecialty.com>
Date: Fri, 13 Jul 2018 13:08:52 -0400
Subject: [PATCH 1/3] linker: Remove classifier vs. calculation distinction

This is something that I thought would be useful back in the day when TAME
was in its infancy, but it is not important.  Rather than having the linker
spend time trying to figure out what symbols belong in the classifier---and
rather than keeping that complexity around---this simplifies things by
making the existing `classify' method simply perform _all_ calculations, and
then yield only the classification portion of the result.

This isn't a problem in practice because, if we only desire the use of a
classifier, then we create a "supplier" that only uses classifications and
has no other dependencies.  The end result is, as far as we care, the same.

* src/current/compiler/js.xsl (compiler:entry-rater)[lv:package]: Initialize
    `classes' rather than invoking classifier
  (compiler:entry-classifier)[lv:package]: Invoke all calculations and
    return only classes to provide equivalent behavior.
  (compiler:exit-classifier): Post-process classifications from calculation
    results, iterating through classmap.
  (compiler:classifier-yields-map)[lv:package]: Output all classifications
    that are not generated.  This differs slightly from the original
    implementation in that it includes all non-generated classes rather than
    just classes that have a non-generated `@yields'; this distinction is
    important since `compiler:exit-classifier' is now using it to produce a
    classification result set that doesn't contain all the generated
    stuff (since it didn't before, and shouldn't now).
* src/current/compiler/linker.xsl: Update copyright year.
  (l:resolv-deps)[preproc:sym[@l:mark-inclass]]: Remove template.
  (l:resolv-deps)[preproc:sym...@l:mark-inclass...]: Remove template.
  (l:depgen-sym): Set type of result to `element(preproc:sym)', since
    `l:mark-inclass' is no longer produced.
    [inclass, needs-class-mark]: Remove variables and all instances where
      they are used.
  (l:dep-aug)[inclass]: Remove param.  Stop producing `@inclass' attribute.
  (l:link-classifier)[lv:package]: Do not process any dependencies.  This
    can be removed entirely in the future since it now only produces static
    code, which we can perhaps combine with a different block.
  (l:link-rater)[lv:package]: Remove mention of `inclass' for dependencies;
    all dependencies will now be compiled into this block.
---
 src/current/compiler/js.xsl     |  32 +++------
 src/current/compiler/linker.xsl | 113 +++-----------------------------
 2 files changed, 18 insertions(+), 127 deletions(-)

diff --git a/src/current/compiler/js.xsl b/src/current/compiler/js.xsl
index cfc5ccfa..f975f375 100644
--- a/src/current/compiler/js.xsl
+++ b/src/current/compiler/js.xsl
@@ -99,9 +99,8 @@
       <!-- handle defaults -->
       <text>init_defaults( args, params );</text>
 
-      <!-- perform classifications -->
       <value-of select="$compiler:nl" />
-      <text>var classes = rater.classify( args, _canterm );</text>
+      <text>var classes = {};</text>
       <!-- for @external generated clases -->
       <text>var genclasses = {};</text>
 </template>
@@ -110,34 +109,23 @@
   <!-- allow classification of any arbitrary dataset -->
   <value-of select="$compiler:nl" />
   <text>rater.classify = function( args, _canterm ) {</text>
-    <text>_canterm = ( _canterm == undefined ) ? true : !!_canterm;</text>
-
-    <!-- XXX: Remove this; shouldn't be magic -->
-    <text>consts['__DATE_YEAR__'] = ( new Date() ).getFullYear(); </text>
-
-    <!-- object into which all classifications will be stored -->
-    <text>var classes = {}, genclasses = {}; </text>
-
-    <!-- TODO: We need to do something with this... -->
-    <text>var req_params = {}; </text>
+    return rater( args, _canterm ).classes;
+  <text> };</text>
 </template>
 
 <template match="lv:package" mode="compiler:exit-classifier">
-    <text>return classes;</text>
-  <text> };</text>
-
   <!-- TODO: make sure fromMap has actually been compiled -->
   <text>rater.classify.fromMap = function( args_base, _canterm ) { </text>
     <text>var ret = {}; </text>
     <text>rater.fromMap( args_base, function( args ) {</text>
-      <text>var classes = rater.classify( args, _canterm ); </text>
+      <text>var result = rater( args, _canterm ); </text>
 
       <text>
-        for ( var c in classes )
+        for ( var c in rater.classify.classmap )
         {
           ret[ c ] = {
-            is: !!classes[ c ],
-            indexes: args[ rater.classify.classmap[ c ] ]
+            is: !!result.classes[ c ],
+            indexes: result.vars[ rater.classify.classmap[ c ] ]
           };
         }
       </text>
@@ -221,15 +209,15 @@
 
   <!-- for each cgen symbol (which is the classification @yields), map the
        classification name (the @parent) to the cgen symbol name -->
-  <for-each select="$symbols[ @type='cgen' ]">
+  <for-each select="$symbols[ @type='class' and not( @preproc:generated ) ]">
     <if test="position() > 1">
       <text>,</text>
     </if>
 
     <text>'</text>
-      <value-of select="substring-after( @parent, ':class:' )" />
+      <value-of select="substring-after( @name, ':class:' )" />
     <text>':'</text>
-      <value-of select="@name" />
+      <value-of select="@yields" />
     <text>'</text>
   </for-each>
 </template>
diff --git a/src/current/compiler/linker.xsl b/src/current/compiler/linker.xsl
index 0f22cc2a..cb7b4e25 100644
--- a/src/current/compiler/linker.xsl
+++ b/src/current/compiler/linker.xsl
@@ -2,7 +2,7 @@
 <!--
   Assembles code fragments into a final executable
 
-  Copyright (C) 2016, 2017 R-T Specialty, LLC.
+  Copyright (C) 2016, 2017, 2018 R-T Specialty, LLC.
 
     This file is part of TAME.
 
@@ -228,56 +228,12 @@
 </template>
 
 
-<!-- replace marks with the symbols that they reference (note that the linker
-     will not add duplicates, so we needn't worry about them) -->
-<template mode="l:resolv-deps" as="element( preproc:sym )"
-          priority="8"
-          match="preproc:sym[ @l:mark-inclass ]">
-  <!-- FIXME: I sometimes return more than one symbol! -->
-  <variable name="sym" as="element( preproc:sym )*"
-            select="root(.)/preproc:sym[
-                      @name = current()/@name ]" />
-
-  <!-- sanity check; hopefully never necessary -->
-  <if test="not( $sym )">
-    <call-template name="log:internal-error">
-      <with-param name="name" select="'link'" />
-      <with-param name="msg">
-        <text>l:mark-class found for non-existing symbol </text>
-        <value-of select="@name" />
-        <text>; there is a bug in l:depgen-process-sym</text>
-      </with-param>
-    </call-template>
-  </if>
-
-  <!-- copy the element and mark as inclass (no need to check @src, since at
-       this point, such conflicts would have already resulted in an error -->
-  <preproc:sym>
-    <sequence select="$sym/@*" />
-
-    <!-- override attribute -->
-    <attribute name="inclass" select="'true'" />
-  </preproc:sym>
-</template>
-
-
-<!-- any now-inclass symbols should be stripped of their original position -->
-<template mode="l:resolv-deps" priority="7"
-          match="preproc:sym[
-                   @name = root(.)/preproc:sym[ @l:mark-inclass ]
-                             /@name ]">
-  <!-- bye-bye -->
-</template>
-
-
 <template match="*" mode="l:resolv-deps" priority="1">
   <sequence select="." />
 </template>
 
 
-<!-- FIXME: I want element( l:preproc:sym ), but we also have
-     l:mark-inclass -->
-<template name="l:depgen-sym" as="element()*">
+<template name="l:depgen-sym" as="element( preproc:sym )*">
   <param name="pending" as="element( preproc:sym )*" />
   <param name="stack" as="element( l:sym-stack )" />
   <param name="path" as="xs:string"
@@ -494,16 +450,6 @@
   <variable name="cur" as="element( preproc:sym )"
             select="." />
 
-  <!-- determines if the compile destination for these dependencies will be
-       within the classifier; this is the case for all class dependencies
-       *unless* the class is external -->
-  <variable name="inclass" as="xs:boolean"
-            select="( ( $cur/@type='class' )
-                      and not( $cur/@extclass='true' ) )
-                    or $processing/preproc:sym[
-                      @type='class'
-                      and not( @extclass='true' ) ]" />
-
   <!-- perform circular dependency check and blow up if found (we cannot choose
        a proper linking order without a correct dependency tree); the only
        exception is if the circular dependency is a function, since that simply
@@ -567,17 +513,6 @@
         </call-template>
       </if>
 
-      <!-- determine if class already exists, but needs to be marked for
-           inclusion within the classifier -->
-      <variable name="needs-class-mark" as="xs:boolean"
-                select="$existing
-                          and $inclass
-                          and not( $existing/@inclass='true' )
-                          and not( $stack/preproc:sym[
-                                     @l:mark-inclass
-                                     and @name=$cur/@name
-                                     and @src=$cur/@src ] )" />
-
       <!-- continue with the remainder of the symbol list -->
       <call-template name="l:depgen-sym">
         <with-param name="pending" select="remove( $pending, 1 )" />
@@ -588,7 +523,7 @@
                re-adding it (note that we check both the symbol name and its source
                since symbols could very well share a name due to exporting rules) -->
           <choose>
-            <when test="not( $existing ) or $needs-class-mark">
+            <when test="not( $existing )">
               <!-- does this symbol have any dependencies? -->
               <variable name="deps" as="element( preproc:sym )*">
                 <apply-templates select="$cur" mode="l:depgen-sym" />
@@ -607,7 +542,6 @@
                 <call-template name="l:dep-aug">
                   <with-param name="cur" select="$cur" />
                   <with-param name="deps" select="$deps" />
-                  <with-param name="inclass" select="$inclass" />
                   <with-param name="mypath" select="$mypath" />
                 </call-template>
               </variable>
@@ -629,20 +563,9 @@
                 </call-template>
 
                 <!-- finally, we can output ourself -->
-                <choose>
-                  <!-- existing symbol needs to be marked -->
-                  <when test="$needs-class-mark">
-                    <preproc:sym l:mark-inclass="true" name="{$cur/@name}" src="{$cur/@src}" />
-                  </when>
-
-                  <!-- new symbol -->
-                  <otherwise>
-                    <preproc:sym>
-                      <sequence select="$cur/@*" />
-                      <attribute name="inclass" select="$inclass" />
-                    </preproc:sym>
-                  </otherwise>
-                </choose>
+                <preproc:sym>
+                  <sequence select="$cur/@*" />
+                </preproc:sym>
               </l:sym-stack>
             </when>
 
@@ -662,8 +585,6 @@
 <template name="l:dep-aug" as="element( preproc:sym )*">
   <param name="cur" as="element( preproc:sym )" />
   <param name="deps" as="element( preproc:sym )*" />
-  <param name="inclass" as="xs:boolean"
-         select="false()" />
   <param name="proc-barrier" as="xs:boolean"
          select="false()" />
   <param name="parent-name" as="xs:string"
@@ -709,11 +630,6 @@
       <!-- set new src path -->
       <attribute name="src" select="$newsrc" />
 
-      <!-- flag for inclusion into classifier, if necessary -->
-      <if test="$inclass">
-        <attribute name="inclass" select="$inclass" />
-      </if>
-
       <if test="$proc-barrier">
         <attribute name="l:proc-barrier" select="'true'" />
       </if>
@@ -991,8 +907,6 @@
 
 
 <template match="lv:package" mode="l:link-classifier">
-  <param name="deps" />
-
   <call-template name="log:info">
     <with-param name="name" select="'link'" />
     <with-param name="msg">
@@ -1002,17 +916,7 @@
 
   <!-- link everything that shall be a part of the classifier -->
   <apply-templates select="." mode="compiler:entry-classifier" />
-  <apply-templates select="." mode="l:do-link">
-    <with-param name="symbols" select="
-        $deps[
-          @inclass='true'
-          and not( @type='param' )
-          and not( @type='type' )
-          and not( @type='meta' )
-          and not( @type='worksheet' )
-        ]
-      " />
-  </apply-templates>
+  <!-- TODO: get rid of me completely! -->
   <apply-templates select="." mode="compiler:exit-classifier" />
 </template>
 
@@ -1090,8 +994,7 @@
   <apply-templates select="." mode="l:do-link">
     <with-param name="symbols" select="
         $deps[
-          not( @inclass='true' )
-          and not( @type='param' )
+          not( @type='param' )
           and not( @type='type' )
           and not( @type='func' )
           and not( @type='meta' )

From d624ee6d7eb4fa1c1c53d50271f3f591264853f5 Mon Sep 17 00:00:00 2001
From: Mike Gerwitz <mike.gerwitz@rtspecialty.com>
Date: Fri, 13 Jul 2018 23:34:43 -0400
Subject: [PATCH 2/3] linker: Use sequences for stacks (instead of trees)

This has a significant performance impact: processing time is cut in about
half and memory usage is reduced by more than 50%.  For example, a
package that previously took 30s and 2.1GiB of memory to link now takes
14s and less than 900MiB of memory.

I had tried to perform this optimization a couple years ago but was
thwarted (I think) by the classifier markers.  The previous commit did away
with those.  I'm encouraged by the gains from the low-hanging fruit.

* src/current/compiler/linker.xsl
  (l:process-empty, l:stack-empty): Convert from l:pstack and
    l:sym-stack (respectively) to empty preproc:sym sequences.
  (l:depgen-process-sym)[preproc:sym]: Append to sequence rather than
    outputting new l:sym-stack tree.
  Update all annotations and uses accordingly.
---
 src/current/compiler/linker.xsl | 89 +++++++++++++++------------------
 1 file changed, 41 insertions(+), 48 deletions(-)

diff --git a/src/current/compiler/linker.xsl b/src/current/compiler/linker.xsl
index cb7b4e25..65fb57f8 100644
--- a/src/current/compiler/linker.xsl
+++ b/src/current/compiler/linker.xsl
@@ -57,13 +57,11 @@
 <variable name="l:orig-root" as="document-node( element( lv:package ) )"
           select="/" />
 
-<variable name="l:process-empty" as="element( l:pstack )">
-  <l:pstack />
-</variable>
+<variable name="l:process-empty" as="element( preproc:sym )*"
+          select="()" />
 
-<variable name="l:stack-empty" as="element( l:sym-stack )">
-  <l:sym-stack />
-</variable>
+<variable name="l:stack-empty" as="element( preproc:sym )*"
+          select="()" />
 
 
 <template match="*" mode="l:link" priority="1">
@@ -115,7 +113,7 @@
     <l:dep>
       <!-- empty stack -->
       <apply-templates select="preproc:symtable" mode="l:depgen">
-        <with-param name="stack" select="$l:stack-empty" as="element( l:sym-stack )" />
+        <with-param name="stack" select="$l:stack-empty" />
       </apply-templates>
     </l:dep>
   </variable>
@@ -175,8 +173,8 @@
 
 <template mode="l:depgen" as="element( preproc:sym )*"
           match="preproc:symtable">
-  <param name="stack" as="element( l:sym-stack )"
-         select="$l:stack-empty" />
+  <param name="stack" as="element( preproc:sym )*"
+         select="()" />
 
   <!-- we care only of the symbols used by lv:yields, from which all
        dependencies may be derived (if it's not derivable from the yield
@@ -209,7 +207,7 @@
   <variable name="result" as="element()+">
     <call-template name="l:depgen-sym">
       <with-param name="pending" select="$yields" />
-      <with-param name="stack" select="$stack" as="element( l:sym-stack )" />
+      <with-param name="stack" select="$stack" />
     </call-template>
   </variable>
 
@@ -235,24 +233,24 @@
 
 <template name="l:depgen-sym" as="element( preproc:sym )*">
   <param name="pending" as="element( preproc:sym )*" />
-  <param name="stack" as="element( l:sym-stack )" />
+  <param name="stack" as="element( preproc:sym )*" />
   <param name="path" as="xs:string"
          select="''" />
-  <param name="processing" as="element( l:pstack )"
+  <param name="processing" as="element( preproc:sym )*"
          select="$l:process-empty" />
 
   <variable name="pend-count" as="xs:integer"
             select="count( $pending )" />
   <variable name="stack-count" as="xs:integer"
-            select="count( $stack/preproc:sym )" />
+            select="count( $stack )" />
   <variable name="process-count" as="xs:integer"
-            select="count( $processing/* )" />
+            select="count( $processing )" />
 
   <choose>
     <!-- if there are no pending symbols left, then we are done; return the
          stack -->
     <when test="$pend-count = 0">
-      <sequence select="$stack/*" />
+      <sequence select="$stack" />
     </when>
 
 
@@ -277,9 +275,9 @@
             <text>r - </text>
             <value-of select="$cur/@name" />
             <text> [s:: </text>
-            <value-of select="$stack/preproc:sym/@name" />
+            <value-of select="$stack/@name" />
             <text> ::s] [r:: </text>
-            <value-of select="$processing/preproc:sym/@name" />
+            <value-of select="$processing/@name" />
             <text>::r]</text>
           </with-param>
         </call-template>
@@ -361,9 +359,9 @@
 -->
 <template match="preproc:sym[ @extern='true' ]" mode="l:depgen-process-sym" priority="5">
   <param name="pending" as="element( preproc:sym )*" />
-  <param name="stack" as="element( l:sym-stack )" />
+  <param name="stack" as="element( preproc:sym )*" />
   <param name="path" as="xs:string" />
-  <param name="processing" as="element( l:pstack )" />
+  <param name="processing" as="element( preproc:sym )*" />
 
   <variable name="cur" select="." />
 
@@ -407,7 +405,7 @@
         <text>); pulled in by: </text>
 
         <!-- help the user figure out how this happened -->
-        <for-each select="$processing/preproc:sym">
+        <for-each select="$processing">
           <if test="position() gt 1">
             <text> - </text>
           </if>
@@ -433,7 +431,7 @@
   <!-- use the resolved symbol in place of the original extern -->
   <apply-templates select="$eresolv-uniq" mode="l:depgen-process-sym">
     <with-param name="pending" select="$pending" />
-    <with-param name="stack" select="$stack" as="element( l:sym-stack )" />
+    <with-param name="stack" select="$stack" />
     <with-param name="path" select="$path" />
     <with-param name="processing" select="$processing" />
   </apply-templates>
@@ -443,9 +441,9 @@
 <template mode="l:depgen-process-sym" priority="1"
               match="preproc:sym">
   <param name="pending" as="element( preproc:sym )*" />
-  <param name="stack" as="element( l:sym-stack )" />
+  <param name="stack" as="element( preproc:sym )*" />
   <param name="path" as="xs:string" />
-  <param name="processing" as="element( l:pstack )" />
+  <param name="processing" as="element( preproc:sym )*" />
 
   <variable name="cur" as="element( preproc:sym )"
             select="." />
@@ -455,7 +453,7 @@
        exception is if the circular dependency is a function, since that simply
        implies recursion, which we can handle just fine -->
   <variable name="circ" as="element( preproc:sym )*"
-            select="$processing/preproc:sym[
+            select="$processing[
                       @name=$cur/@name
                       and @src=$cur/@src ]" />
 
@@ -477,7 +475,7 @@
       <call-template name="l:depgen-sym">
         <with-param name="pending" select="remove( $pending, 1 )" />
         <with-param name="processing" select="$processing" />
-        <with-param name="stack" select="$stack" as="element( l:sym-stack )"  />
+        <with-param name="stack" select="$stack" />
       </call-template>
     </when>
 
@@ -485,8 +483,7 @@
     <!-- process; TODO: good refactoring point; large template -->
     <otherwise>
       <variable name="existing" as="element( preproc:sym )*"
-                select="$stack/preproc:sym[
-                          @name=$cur/@name ]" />
+                select="$stack[ @name=$cur/@name ]" />
 
       <!-- TODO: this uses @name instead of @src because of map import
            paths; decide on one or the other -->
@@ -518,7 +515,7 @@
         <with-param name="pending" select="remove( $pending, 1 )" />
         <with-param name="processing" select="$processing" />
 
-        <with-param name="stack" as="element( l:sym-stack )">
+        <with-param name="stack" as="element( preproc:sym )*">
           <!-- if this symbol already exists on the stack, then there is no use
                re-adding it (note that we check both the symbol name and its source
                since symbols could very well share a name due to exporting rules) -->
@@ -546,27 +543,23 @@
                 </call-template>
               </variable>
 
-              <l:sym-stack>
-                <!-- process the dependencies (note that this has the effect of
-                     outputting the existing stack as well, which is why we have
-                     not yet done so) -->
-                <call-template name="l:depgen-sym">
-                  <with-param name="pending" select="$deps-aug" />
-                  <with-param name="stack" select="$stack" as="element( l:sym-stack )" />
-                  <with-param name="path" select="$mypath" />
-                  <with-param name="processing" as="element( l:pstack )">
-                    <l:pstack>
-                      <sequence select="$processing/*" />
-                      <sequence select="$cur" />
-                    </l:pstack>
-                  </with-param>
-                </call-template>
+              <!-- process the dependencies (note that this has the effect of
+                   outputting the existing stack as well, which is why we have
+                   not yet done so) -->
+              <call-template name="l:depgen-sym">
+                <with-param name="pending" select="$deps-aug" />
+                <with-param name="stack" select="$stack" />
+                <with-param name="path" select="$mypath" />
+                <with-param name="processing" as="element( preproc:sym )*">
+                  <sequence select="$processing" />
+                  <sequence select="$cur" />
+                </with-param>
+              </call-template>
 
-                <!-- finally, we can output ourself -->
-                <preproc:sym>
-                  <sequence select="$cur/@*" />
+              <!-- finally, we can output ourself -->
+              <preproc:sym>
+                <sequence select="$cur/@*" />
                 </preproc:sym>
-              </l:sym-stack>
             </when>
 
 
@@ -760,7 +753,7 @@
       <value-of select="concat( $cur/@src, '/', $cur/@name )" />
       <text>): </text>
 
-      <for-each select="$stack//preproc:sym">
+      <for-each select="$stack">
         <if test="position() > 1">
           <text> - </text>
         </if>

From ea7e0e1565e1a8dbf7b500c071379ac40e7c898d Mon Sep 17 00:00:00 2001
From: Mike Gerwitz <mike.gerwitz@rtspecialty.com>
Date: Fri, 13 Jul 2018 23:49:18 -0400
Subject: [PATCH 3/3] preproc: Remove @override from eseq:is-expandable
 function

This was throwing a warning in non-ancient versions of Saxon.  It does not
need to be there, nor should it be, nor do I know why it was put there.

* src/current/include/preproc/template.xsl (eseq:is-expandable): Remove
  @override.
---
 src/current/include/preproc/template.xsl | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/current/include/preproc/template.xsl b/src/current/include/preproc/template.xsl
index e0294e7a..bc63fe65 100644
--- a/src/current/include/preproc/template.xsl
+++ b/src/current/include/preproc/template.xsl
@@ -1353,8 +1353,7 @@
 
 
 
-<xsl:function name="eseq:is-expandable" as="xs:boolean"
-              override="yes">
+<xsl:function name="eseq:is-expandable" as="xs:boolean">
   <xsl:param name="node" as="node()" />
 
   <!-- TODO: what a mess; clean me up by changing the point at which