From 2af0daceb320b59d13e312c6c9b65ae04e7a615b Mon Sep 17 00:00:00 2001
From: Mike Gerwitz <gerwitzm@lovullo.com>
Date: Wed, 12 Jul 2017 15:21:43 -0400
Subject: [PATCH 1/2] Remove @keep support from linker

And everything else.

This is a big (important) change; it addresses one of the greatest
pains of the system.

Keeps were added during the DSL rewrite (to support symbols and such)
to work around the issue that there was no symbol-driven map; it
allowed symbols to persist disjoint from the `__yield' dependency
graph so that they could be mapped back and used by external systems.

The problem with that is that it's both messy (coupling the concept of
external dependencies with the actual code) and difficult to work
with.  It had a huge performance impact on the linker for two reasons:

  - Checking whether a package had already been seen and importing the
    keeps on first visit was expensive because of tree searching and
    manipulation; and
  - _every_ keep was imported and processed by the linker, even if it
    wouldn't end up being used by a particular program.

The later especially had huge performance impacts on the entire
system.

The entire dependency graph is now map-driven, with the exception of
the implicit `__yield' (which will eventually be moved into the map as
well and the magic `lv:yield' removed in favor of a template).

Performance-wise: our largest program ("dwelling") has many thousands
of symbols and the largest package imported the majority of them, many
of them unneeded, as the result of @keep subgraphs.  Compilation of
the largest package within that (for the UI) took about a minute and a
half and ate up ~6GiB of RAM, for what really is a trivial task of
resolving externs, some basic symbol processing, a topological sort,
and ordering code fragments.

After this change, it takes ~15s and less than 2GiB of RAM.  Still a
lot---and more improvements can be made---but much, much better.

@keep and friends was left in rater.xsd so that nothing breaks while
code is cleaned up; it'll be removed in the future.

* src/current/compiler/linker.xsl: Remove @keep support.
* src/current/dot/attr-keep.xsl: Remove now-unneeded template.
* src/current/dot/defnode.xsl: Remove @keep and related.
* src/current/include/preproc/eligclass.xsl: Remove @keep and related.
* src/current/include/preproc/expand.xsl: Remove @keep and related.
* src/current/include/preproc/macros.xsl: Remove @keep and related.
* src/current/include/preproc/symtable.xsl: Remove @keep and related.
* src/current/rater.xsd: Add TODO to remove @keep and friends.
---
 src/current/compiler/linker.xsl           | 184 ++--------------------
 src/current/dot/attr-keep.xsl             |  50 ------
 src/current/dot/defnode.xsl               |   1 -
 src/current/include/preproc/eligclass.xsl |   5 -
 src/current/include/preproc/expand.xsl    |   5 -
 src/current/include/preproc/macros.xsl    |   4 +-
 src/current/include/preproc/symtable.xsl  |  29 +---
 src/current/rater.xsd                     |   2 +-
 8 files changed, 24 insertions(+), 256 deletions(-)
 delete mode 100644 src/current/dot/attr-keep.xsl

diff --git a/src/current/compiler/linker.xsl b/src/current/compiler/linker.xsl
index 23cef9ae..93eb0944 100644
--- a/src/current/compiler/linker.xsl
+++ b/src/current/compiler/linker.xsl
@@ -181,23 +181,21 @@
        symbol, then it must not be used); note that lv:yield actually compiles
        into a special symbol ___yield -->
   <variable name="yields" as="element( preproc:sym )+">
-    <copy-of select="preproc:sym[ @name='___yield' ]" />
+    <!-- TOOD: this shouldn't be a magical exception; map it -->
+    <sequence select="preproc:sym[ @name='___yield' ]" />
 
-    <!-- also include anything derivable from any @keep symbol, either local
-         or imported -->
-    <copy-of select="preproc:sym[ @keep='true' ]" />
-
-    <!-- TODO: these should be included as a consequence of the linking
-         process, not as an exception -->
-    <copy-of select="
+    <!-- TODO: messy; refactor this symbol situation -->
+    <!-- this should be the sole source of outputs and, consequently,
+         dependencies -->
+    <sequence select="
         preproc:sym[
           @type='map' or @type='map:head' or @type='map:tail'
           or @type='retmap' or @type='retmap:head' or @type='retmap:tail'
         ]
       " />
 
-    <!-- TODO: same as above -->
-    <copy-of select="preproc:sym[ @name='___worksheet' ]" />
+    <!-- TODO: also should not be an exception -->
+    <sequence select="preproc:sym[ @name='___worksheet' ]" />
   </variable>
 
   <!-- start at the top of the table and begin processing each symbol
@@ -325,81 +323,9 @@
         </call-template>
       </if>
 
-      <variable name="pkg-seen" as="xs:boolean"
-                select="(
-                          ( $cur/@src = '' or not( $cur/@src ) )
-                          and $stack/preproc:pkg-seen/@src = ''
-                        )
-                        or $cur/@src = $stack/preproc:pkg-seen/@src" />
-
-      <variable name="newpending" as="element( l:pending )">
-        <l:pending>
-          <sequence select="$pending" />
-
-          <!-- if this is the first time seeing this package, then pend its
-               @keep's for processing -->
-          <if test="not( $pkg-seen )">
-            <message select="'[link] found package ', $cur/@src" />
-
-            <variable name="document" as="element( lv:package )"
-                      select="if ( not( $cur/@src or $cur/@src = '' ) ) then
-                                $l:orig-root/lv:package
-                              else
-                                document( concat( $cur/@src, '.xmlo' ),
-                                          $l:orig-root )
-                                  /lv:package" />
-
-            <variable name="keeps" as="element( preproc:sym )*" select="
-                  $document/preproc:symtable/preproc:sym[
-                    (
-                      @keep='true'
-                      or ( $l:orig-root/lv:package/@auto-keep-imports='true'
-                           and ( @type = 'class'
-                                 or @type = 'cgen' ) )
-                    )
-                    and not(
-                      $l:orig-root/lv:package/@no-extclass-keeps='true'
-                      and @extclass='true'
-                    )
-                    and not( @name=$pending/@name )
-                    and not( @name=$stack/preproc:sym/@name )
-                  ]
-              " />
-
-            <variable name="keepdeps" as="element( preproc:sym )*">
-              <call-template name="l:dep-aug">
-                <with-param name="cur" select="$cur" />
-                <with-param name="deps" select="$keeps" />
-                <with-param name="proc-barrier" select="true()" />
-                <with-param name="parent-name"
-                  select="concat( 'package ', $cur/@src )" />
-              </call-template>
-            </variable>
-
-            <sequence select="$keepdeps" />
-          </if>
-        </l:pending>
-      </variable>
-
-
-      <variable name="stack-seen" as="element( l:sym-stack )">
-        <l:sym-stack>
-          <if test="not( $pkg-seen )">
-            <sequence select="$stack/*" />
-            <preproc:pkg-seen src="{$cur/@src}" />
-          </if>
-        </l:sym-stack>
-      </variable>
-
-      <variable name="newstack" as="element( l:sym-stack )"
-                select="if ( $pkg-seen ) then
-                          $stack
-                        else
-                          $stack-seen" />
-
       <apply-templates select="$cur" mode="l:depgen-process-sym">
-        <with-param name="pending" select="$newpending/*" />
-        <with-param name="stack" select="$newstack" />
+        <with-param name="pending" select="$pending" />
+        <with-param name="stack" select="$stack" />
         <with-param name="path" select="$path" />
         <with-param name="processing" select="
             if ( $cur/@l:proc-barrier = 'true' )
@@ -610,11 +536,13 @@
                 select="$stack/preproc:sym[
                           @name=$cur/@name ]" />
 
+      <!-- TODO: this uses @name instead of @src because of map import
+           paths; decide on one or the other -->
       <variable name="src-conflict" as="element( preproc:sym )*"
-                select="if ( not( $cur/@src ) or $cur/@src = '' ) then
+                select="if ( not( $cur/@name ) or $cur/@name = '' ) then
                           ()
                         else
-                          $existing[ not( @src = $cur/@src ) ]" />
+                          $existing[ not( @name = $cur/@name ) ]" />
 
       <if test="$src-conflict">
         <call-template name="log:error">
@@ -622,7 +550,7 @@
           <with-param name="msg">
             <text>symbol name is not unique: `</text>
             <value-of select="@name" />
-            <text>' found in</text>
+            <text>' found in </text>
             <value-of select="$cur/@src" />
 
             <for-each select="$src-conflict">
@@ -805,20 +733,13 @@
 
 
 <!-- TODO: some better way. -->
-<template match="preproc:sym[ starts-with( @type, 'map' ) or starts-with( @type, 'retmap' ) ]"
+<template match="preproc:sym[ starts-with( @type, 'map' ) ]"
   mode="l:depgen-sym" priority="7">
 
   <!-- do not process deps -->
 </template>
 
 
-<template mode="l:depgen-sym" as="element()*"
-          match="preproc:pkg-seen"
-          priority="5">
-  <sequence select="." />
-</template>
-
-
 <template mode="l:depgen-sym" as="element( preproc:sym )*"
           match="preproc:sym[
                    @type='const' ]"
@@ -1384,78 +1305,7 @@
 
 
 <template match="preproc:sym" mode="l:map" priority="5">
-  <param name="symtable" as="element( l:dep )" />
-  <param name="type" as="xs:string"
-         select="'input'" />
-  <param name="from" as="xs:string"
-         select="'destination'" />
-  <param name="ignore-error" as="xs:boolean"
-         select="false()" />
-
-  <variable name="name" as="xs:string"
-            select="@name" />
-  <variable name="src" as="xs:string"
-            select="@src" />
-
-  <!-- map symbols must always be remote -->
-  <variable name="pkg" as="element( lv:package )"
-            select="document( concat( @src, '.xmlo' ), . )
-                      /lv:package" />
-
-  <!-- get map symbol dependencies -->
-  <variable name="deps" as="element( preproc:sym-dep )*"
-            select="$pkg/preproc:sym-deps/
-                      preproc:sym-dep[ @name=$name ]" />
-
-  <if test="not( $deps )">
-    <call-template name="log:internal-error">
-      <with-param name="name" select="'link'" />
-      <with-param name="msg">
-        <text>could not locate symbol dependencies: </text>
-        <value-of select="concat( @src, '/', @name )" />
-      </with-param>
-    </call-template>
-  </if>
-
-  <!-- FIXME: we should not have to check for @yields here; we may
-       have to require imports in the map to satisfy normalization
-       before-hand -->
-  <variable name="unknown" as="element( preproc:sym-ref )*"
-            select="$deps/preproc:sym-ref[
-                      not( @name=$symtable/preproc:sym/@name
-                           or @name=$symtable/preproc:sym/@yields ) ]" />
-
-  <choose>
-    <!-- ensure that every dependency is known (we only care that the symbol
-         actually exists and is an input) -->
-    <when test="$unknown and not( $ignore-error )">
-      <for-each select="$unknown">
-        <call-template name="log:error">
-          <with-param name="terminate" select="'no'" />
-          <with-param name="name" select="'link'" />
-          <with-param name="msg">
-            <value-of select="$type" />
-            <text> map </text>
-            <value-of select="$from" />
-            <text> </text>
-              <value-of select="@name" />
-            <text> is not a known </text>
-              <value-of select="$type" />
-            <text> field for </text>
-            <value-of select="concat( $src, '/', $name )" />
-            <text>; ensure that it exists and is either used or has @keep set</text>
-          </with-param>
-        </call-template>
-      </for-each>
-
-      <l:map-error />
-    </when>
-
-    <!-- good to go; link symbol -->
-    <otherwise>
-      <apply-templates select="." mode="l:link-deps" />
-    </otherwise>
-  </choose>
+  <apply-templates select="." mode="l:link-deps" />
 </template>
 
 </stylesheet>
diff --git a/src/current/dot/attr-keep.xsl b/src/current/dot/attr-keep.xsl
deleted file mode 100644
index d2776748..00000000
--- a/src/current/dot/attr-keep.xsl
+++ /dev/null
@@ -1,50 +0,0 @@
-<?xml version="1.0" encoding="ISO-8859-1"?>
-<!--
-  Styles node based on keep flag
-
-  Copyright (C) 2016 LoVullo Associates, Inc.
-
-    This file is part of TAME.
-
-    TAME is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-    General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see
-    <http://www.gnu.org/licenses/>.
--->
-<xsl:stylesheet version="2.0"
-  xmlns="http://www.w3.org/1999/xhtml"
-  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-
-  xmlns:dot="http://www.lovullo.com/calc/dot"
-  xmlns:preproc="http://www.lovullo.com/rater/preproc">
-
-
-<!--
-  External nodes should be styled as such
--->
-<xsl:template mode="dot:attr-keep" priority="5" match="
-    *[ @keep='true' ]
-  ">
-
-  <dot:attr name="fontcolor">red</dot:attr>
-</xsl:template>
-
-
-
-<!--
-  Default node attributes
--->
-<xsl:template match="preproc:sym" mode="dot:defnode-keep" priority="1">
-</xsl:template>
-
-</xsl:stylesheet>
-
diff --git a/src/current/dot/defnode.xsl b/src/current/dot/defnode.xsl
index 1cf6501a..6e350df3 100644
--- a/src/current/dot/defnode.xsl
+++ b/src/current/dot/defnode.xsl
@@ -92,7 +92,6 @@
         <xsl:apply-templates select="." mode="dot:attr-extern" />
         <xsl:apply-templates select="." mode="dot:attr-color" />
         <xsl:apply-templates select="." mode="dot:attr-shape" />
-        <xsl:apply-templates select="." mode="dot:attr-keep" />
       </xsl:with-param>
     </xsl:call-template>
   </xsl:variable>
diff --git a/src/current/include/preproc/eligclass.xsl b/src/current/include/preproc/eligclass.xsl
index 5fc9353e..974fb7a2 100644
--- a/src/current/include/preproc/eligclass.xsl
+++ b/src/current/include/preproc/eligclass.xsl
@@ -101,11 +101,6 @@
   <lv:classify as="{$as}" yields="{$yields}"
     desc="{@name} package is eligible">
 
-    <!-- TODO: this should really be a compile-time value -->
-    <xsl:if test="@keep-elig-class = 'true'">
-      <xsl:attribute name="keep" select="'true'" />
-    </xsl:if>
-
     <!-- each of our imported packages' elig classes must be truthful -->
     <xsl:apply-templates mode="preproc:gen-elig-class-matches"
       select="lv:import">
diff --git a/src/current/include/preproc/expand.xsl b/src/current/include/preproc/expand.xsl
index c8b24598..f023df15 100644
--- a/src/current/include/preproc/expand.xsl
+++ b/src/current/include/preproc/expand.xsl
@@ -500,11 +500,6 @@
     <xsl:apply-templates mode="preproc:expand"
                          select="@*" />
 
-    <!-- force @keep on @terminate -->
-    <xsl:if test="@terminate='true'">
-      <xsl:attribute name="keep" select="'true'" />
-    </xsl:if>
-
     <!-- copy everything else -->
     <xsl:apply-templates mode="preproc:expand" />
   </xsl:copy>
diff --git a/src/current/include/preproc/macros.xsl b/src/current/include/preproc/macros.xsl
index dd836c9d..8d991a49 100644
--- a/src/current/include/preproc/macros.xsl
+++ b/src/current/include/preproc/macros.xsl
@@ -461,9 +461,7 @@
 
     <!-- ensure that this name does not already exist -->
     <xsl:if test="not( /lv:*/lv:classify[ @as=$genas ] )">
-      <!-- TODO: We're flagging as @keep for now due to gclass needs, but this
-           should be removed later -->
-      <lv:classify as="{$genas}" desc="{@desc}" keep="true">
+      <lv:classify as="{$genas}" desc="{@desc}">
         <lv:match on="{@name}" value="TRUE" />
       </lv:classify>
     </xsl:if>
diff --git a/src/current/include/preproc/symtable.xsl b/src/current/include/preproc/symtable.xsl
index e9a7c619..432c3abd 100644
--- a/src/current/include/preproc/symtable.xsl
+++ b/src/current/include/preproc/symtable.xsl
@@ -517,7 +517,6 @@
   <xsl:param name="orig-root" />
   <xsl:param name="package" select="@package" />
   <xsl:param name="export" select="@export" />
-  <xsl:param name="ignore-keep" select="@ignore-keep" />
   <xsl:param name="no-extclass" select="@no-extclass" />
   <xsl:param name="keep-classes" select="@keep-classes" />
 
@@ -593,7 +592,7 @@
         <xsl:when test="@pollute='true'
                         and @local='true'
                         and not( @extern='true' )">
-          <xsl:sequence select="@name, @src, @pollute, @keep, @parent, @extclass" />
+          <xsl:sequence select="@name, @src, @pollute, @parent, @extclass" />
         </xsl:when>
 
         <!-- copy all the symbol information -->
@@ -633,23 +632,6 @@
         </xsl:choose>
       </xsl:attribute>
 
-      <!-- keep manipulation: *always* keep classes if requested, even if
-           @ignore-keep is provided; in the case of the latter, unsets @keep -->
-      <xsl:choose>
-        <!-- keep classes when requested -->
-        <xsl:when test="
-          ( @type = 'class' or @type = 'cgen' )
-          and $keep-classes = 'true'">
-
-          <xsl:attribute name="keep" select="'false'" />
-        </xsl:when>
-
-        <!-- demolish @keep if requested -->
-        <xsl:when test="$ignore-keep = 'true'">
-          <xsl:attribute name="keep" select="'false'" />
-        </xsl:when>
-      </xsl:choose>
-
       <!-- children should always be copied, unless poluting -->
       <xsl:if test="not( @pollute='true' and @local='true' )">
         <xsl:sequence select="preproc:*" />
@@ -663,7 +645,7 @@
   <xsl:variable name="external"  select="boolean( @external='true' )" />
 
   <preproc:sym name="{@yields}" type="rate"
-    extclass="{$external}" keep="{boolean( @keep )}"
+    extclass="{$external}"
     local="{@local}" dtype="float" dim="0" tex="{@sym}" />
 
   <xsl:apply-templates mode="preproc:symtable" />
@@ -706,7 +688,7 @@
 <xsl:template match="c:*[ @generates ]" mode="preproc:symtable" priority="5">
   <xsl:variable name="parent" select="ancestor::lv:rate" />
 
-  <preproc:sym name="{@generates}" keep="{boolean( $parent/@keep )}"
+  <preproc:sym name="{@generates}"
     parent="{$parent/@yields}"
     type="gen" dtype="float" dim="1" desc="{@desc}" tex="{@sym}" />
 
@@ -718,10 +700,9 @@
 <xsl:template match="lv:classify" mode="preproc:symtable" priority="5">
   <xsl:variable name="external"  select="boolean( @external='true' )" />
   <xsl:variable name="terminate" select="boolean( @terminate='true' )" />
-  <xsl:variable name="keep"      select="boolean( @keep='true' )" />
 
   <preproc:sym name=":class:{@as}"
-    extclass="{$external}" terminate="{$terminate}" keep="{$keep}"
+    extclass="{$external}" terminate="{$terminate}"
     type="class" dim="?" desc="{@desc}" yields="{@yields}"
     orig-name="{@as}">
 
@@ -733,7 +714,7 @@
        to avoid scanning separate object files for such common information)
        -->
   <xsl:if test="@yields">
-    <preproc:sym name="{@yields}" keep="{$keep}"
+    <preproc:sym name="{@yields}"
       parent=":class:{@as}"
       extclass="{$external}" terminate="{$terminate}"
       type="cgen" dtype="boolean" dim="?" desc="{@desc}">
diff --git a/src/current/rater.xsd b/src/current/rater.xsd
index d1bac472..f27d75a7 100644
--- a/src/current/rater.xsd
+++ b/src/current/rater.xsd
@@ -1,5 +1,5 @@
 <?xml version="1.0"?>
-
+<!-- TODO: Remove @keep -->
 <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
   targetNamespace="http://www.lovullo.com/rater"
   xmlns="http://www.lovullo.com/rater"

From dbb368ab34c43e9513e16e3ede77132d42822378 Mon Sep 17 00:00:00 2001
From: Mike Gerwitz <gerwitzm@lovullo.com>
Date: Wed, 12 Jul 2017 15:42:53 -0400
Subject: [PATCH 2/2] macros.xsl: Remove unnused param boolean class generation

A classification used to be generated for each param for convenience;
this has been deprecated by the progui package generation (when `q:'
is used).

* src/current/include/preproc/macros.xsl: Remove template.
---
 src/current/include/preproc/macros.xsl | 34 --------------------------
 1 file changed, 34 deletions(-)

diff --git a/src/current/include/preproc/macros.xsl b/src/current/include/preproc/macros.xsl
index 8d991a49..e58f247c 100644
--- a/src/current/include/preproc/macros.xsl
+++ b/src/current/include/preproc/macros.xsl
@@ -434,38 +434,4 @@
   </lv:rate>
 </xsl:template>
 
-
-<!--
-  Generates a classifier for each boolean param
-
-  This is for convenience; a boolean can esssentially be considered its own
-  classifier, so let's generate one to cut down on the amount of code.
-
-  Technically not a macro, but needs to be done before preproc:expand.
-
-  XXX: Get rid of me!  Now unused!
--->
-<xsl:template match="lv:params[ not( @preproc:processed ) ]"
-  mode="preproc:macros" priority="5">
-
-  <xsl:copy>
-    <xsl:attribute name="preproc:processed" select="'true'" />
-    <xsl:sequence select="@*" />
-
-    <xsl:apply-templates mode="preproc:macros" />
-  </xsl:copy>
-
-  <xsl:for-each select="lv:param[ @type='boolean' ]">
-    <xsl:variable name="as" select="translate( @name, '_', '-' )" />
-    <xsl:variable name="genas" select="concat( 'is-', $as )" />
-
-    <!-- ensure that this name does not already exist -->
-    <xsl:if test="not( /lv:*/lv:classify[ @as=$genas ] )">
-      <lv:classify as="{$genas}" desc="{@desc}">
-        <lv:match on="{@name}" value="TRUE" />
-      </lv:classify>
-    </xsl:if>
-  </xsl:for-each>
-</xsl:template>
-
 </xsl:stylesheet>