Experimental guided TCO

This implements TCO in the XSLT compiler by requiring a human to manually
indicate when a recursive call is in tail position.  This was somewhat
urgently needed to resolve stack exhaustion on large rate tables.

TAMER will do this properly by determining itself whether a call is in tail
position.  Until then, this will serve as a test for this type of feature.

RELEASES.md

@@ -14,6 +14,20 @@ commits that introduce the changes.  To make a new release, run
 `tools/mkrelease`, which will handle updating the heading for you.
 
 
+NEXT
+====
+Compiler
+--------
+- Experimental guided tail call optimizations (TCO) added to XSLT-based
+  compiler, allowing a human to manually indicate recursive calls in tail
+  position.
+  - This is undocumented and should only be used by `tame-core`.  The
+    experimental warning will be removed in future releases if the behavior
+    proves to be sound.
+  - TAMER will add support for proper tail calls that will be detected
+    automatically.
+
+
 v17.4.3 (2020-07-02)
 ====================
 This release fixes a bug caused by previous refactoring that caused

src/current/compiler/js-calc.xsl

@@ -854,7 +854,7 @@
 
   @return generated function application
 -->
-<template match="c:apply" mode="compile-calc">
+<template match="c:apply" mode="compile-calc" priority="5">
   <variable name="name" select="@name" />
   <variable name="self" select="." />
 
@@ -904,6 +904,92 @@
 </template>
 
 
+<!--
+  Whether the given function supports tail call optimizations (TCO)
+
+  This is an experimental feature that must be explicitly requested.
+-->
+<function name="compiler:function-supports-tco" as="xs:boolean">
+  <param name="func" as="element( lv:function )" />
+
+  <sequence select="exists( $func/lv:param[
+                              @name='__experimental_guided_tco' ] )" />
+</function>
+
+
+<!--
+  Whether a recursive function application is marked as being in tail
+  position within a function supporting TCO
+
+  A human must determined if a recursive call is in tail position, and
+  hopefully the human is not wrong.
+-->
+<function name="compiler:apply-uses-tco" as="xs:boolean">
+  <param name="apply" as="element( c:apply )" />
+
+  <variable name="ancestor-func" as="element( lv:function )?"
+            select="$apply/ancestor::lv:function" />
+
+  <sequence select="exists( $apply/c:arg[ @name='__experimental_guided_tco' ] )
+                   and $apply/@name = $ancestor-func/@name
+                   and compiler:function-supports-tco( $ancestor-func ) " />
+</function>
+
+
+<!--
+  Experimental guided tail call optimization (TCO)
+
+  When the special param `__experimental_guided_tco' is defined and set to a
+  true value, the recursive call instead overwrites the original function
+  arguments and returns a dummy value.  The function's trampoline is then
+  responsible for re-invoking the function's body.
+
+  Note that this only applies to self-recursive functions; mutual recursion
+  is not recognized.
+
+  By forcing a human to specify whether a recursive call is in tail
+  position, we free ourselves from having to track tail position within this
+  nightmare of a compiler; we can figure this out properly in TAMER.
+-->
+<template mode="compile-calc" priority="7"
+          match="c:apply[ compiler:apply-uses-tco( . ) ]">
+  <variable name="name" select="@name" />
+  <variable name="self" select="." />
+
+  <message select="concat('warning: ', $name, ' recursing with experimental guided TCO')" />
+
+  <variable name="arg-prefix" select="concat( ':', $name, ':' )" />
+
+  <!-- reassign function arguments -->
+  <for-each select="
+      root(.)/preproc:symtable/preproc:sym[
+        @type='func'
+        and @name=$name
+      ]/preproc:sym-ref
+    ">
+
+    <variable name="pname" select="substring-after( @name, $arg-prefix )" />
+    <variable name="arg" select="$self/c:arg[@name=$pname]" />
+
+    <!-- if the call specified this argument, then use it -->
+    <if test="$arg">
+      <sequence select="concat( '/*TCO*/', $pname, '=' )" />
+      <apply-templates select="$arg/c:*[1]" mode="compile" />
+      <text>,</text>
+    </if>
+  </for-each>
+
+  <!-- return value, which doesn't matter since it won't be used -->
+  <text>0</text>
+
+  <!-- don't support c:when here; not worth the effort -->
+  <if test="./c:when">
+    <message terminate="yes"
+             select="'c:when unsupported on TCO c:apply: ', ." />
+  </if>
+</template>
+
+
 <template match="c:when" mode="compile-calc">
   <!-- note that if we have multiple c:whens, they'll be multiplied together by
        whatever calls this, so we're probably fine -->

src/current/compiler/js.xsl

@@ -1022,6 +1022,10 @@
   will return the result of its expression (represented by a calculation in the
   XML).
 
+  If the special param __experimental_guided_tco is defined, recursive calls
+  to the same function can set it to a true value to perform tail call
+  optimization (TCO).  See js-calc.xsl for more information.
+
   @return generated function
 -->
 <template match="lv:function" mode="compile">
@@ -1041,13 +1045,32 @@
 
   <text>) {</text>
 
-  <text>return ( </text>
+  <variable name="tco" as="xs:boolean"
+            select="compiler:function-supports-tco( . )" />
+
+  <if test="$tco">
+    <message select="concat('warning: ', @name, ' enabled experimental guided TCO')" />
+  </if>
+
+  <!-- top of this function's trampoline, if TCO was requested -->
+  <if test="$tco">
+      <text>do{__experimental_guided_tco=0;</text>
+  </if>
+
+  <text>var fresult = ( </text>
     <!-- begin calculation generation (there should be only one calculation node
          as a child, so only it will be considered) -->
     <apply-templates select="./c:*[1]" mode="compile" />
   <text> );</text>
 
-  <text>} </text>
+  <!-- bottom of this function's trampoline, if TCO was requested; if the
+       flag is set (meaning a relevant tail call was hit), jump back to
+       the beginning of the function  -->
+  <if test="$tco">
+    <text>}while(__experimental_guided_tco);</text>
+  </if>
+
+  <text>return fresult;} </text>
 </template>
 
 

src/current/include/preproc/expand.xsl

@@ -247,6 +247,7 @@
           not(
             @name=$overrides/@name
             or @name=$self/@*/local-name()
+            or starts-with( @name, '__experimental_' )
           )
         ]
       ">