WIP

This restores the previous behavior (from before a couple commits ago), but
adds a warning.

We have a GitLab pipeline internally where we break apart certain
directories into separate jobs to parallelize them.  There is a bug relating
to build and retaining artifacts for rate tables that only manifests itself
for Summary Pages, and that needs to be resolved before this is removed.

DEV-15153

bin/dslc.in

@@ -36,7 +36,7 @@ rater-path()
 }
 
 
-CLASSPATH="$CLASSPATH:@DSLC_CLASSPATH@:$dslc_jar" \
-  "@JAVA@" @JAVA_OPTS@ $JAVA_OPTS \
+export CLASSPATH="$CLASSPATH:@DSLC_CLASSPATH@:$dslc_jar"
+exec "@JAVA@" @JAVA_OPTS@ $JAVA_OPTS \
   com.lovullo.dslc.DslCompiler \
   "$( rater-path )"

bin/tame

@@ -22,13 +22,14 @@ set -euo pipefail
 declare mypath; mypath=$( dirname "$( readlink -f "$0" )" )
 readonly mypath
 
-declare -ri EX_NOTAMED=1  # tried to start tamed or runner but failed
-declare -ri EX_STALLED=2  # runner stalled and could not recover
-declare -ri EX_NORUN=3    # no available runners
-declare -ri EX_DLOCK=4    # failed to get a lock to start tamed
-declare -ri EX_BLOCK=5    # failed to get a lock for busy runner check
-declare -ri EX_NODONE=6   # tamed did not provide a DONE with exit code
-declare -ri EX_USAGE=64   # incorrect usage; sysexits.h
+declare -ri EX_NOTAMED=1    # tried to start tamed or runner but failed
+declare -ri EX_STALLED=2    # runner stalled and could not recover
+declare -ri EX_NORUN=3      # no available runners
+declare -ri EX_DLOCK=4      # failed to get a lock to start tamed
+declare -ri EX_BLOCK=5      # failed to get a lock for busy runner check
+declare -ri EX_NODONE=6     # tamed did not provide a DONE with exit code
+declare -ri EX_UNEXPECTED=7 # runner terminated unexpectedly (see `tamed`)
+declare -ri EX_USAGE=64     # incorrect usage; sysexits.h
 
 # maximum amount of time in seconds to wait for runner to ack
 # before forcibly restarting it
@@ -93,7 +94,7 @@ command-runner()
   # if not, then it may have stalled for some reason
   verify-runner-ack "$*" < "$base/1" || {
     echo "warning: failed runner $id ack; requesting reload" >&2
-    kill -HUP "$pid"
+    reload-runner-and-wait "$base" "$id" "$pid"
 
     # give some extra time in case the host is under high load
     sleep "$TAME_CMD_WAITTIME"
@@ -213,7 +214,7 @@ spawn-runner-and-wait()
 
   # wait on the expected id
   local -ri nextid=$(( maxid + 1 ))
-  wait-for-runner "$root" "$nextid"
+  wait-for-runner "$root/$nextid" "$nextid"
 
   echo "$nextid"
 }
@@ -314,6 +315,33 @@ verify-runner()
 }
 
 
+# Request a runner reload and wait for the reload to complete.
+#
+# Before requesting the reload, a `reloading` flag will be set on the
+# runner.  This flag is expected to be cleared by tamed once the runner has
+# been restarted in response to SIGHUP.
+#
+# See `wait-for-runner` for more information on waiting for the flag.
+reload-runner-and-wait() {
+  local -r base="${1?Missing base}"
+  local -ri id="${2?Missing id}"
+  local -ri pid="${3?Missing pid}"
+
+  # mark the runner as unavailable
+  touch "$base/reloading"
+
+  # issue a reload request to the runner
+  kill -HUP "$pid"
+
+  # we must not continue before the runner has been reloaded, otherwise we
+  # may issue new requests to a process that is being killed
+  wait-for-runner "$base" "$id" || {
+    echo "error: failed to reload runner $id" >&2
+    exit "$EX_STALLED"
+  }
+}
+
+
 # Wait for command acknowledgment from runner
 #
 # The runner must respond within TAME_CMD_WAITTIME seconds
@@ -323,6 +351,13 @@ verify-runner-ack()
 {
   local -r cmd="${1?Missing command}"
 
+  if [ -n "${TAME_DEBUG_RANDOM_NOACK:-}" ]; then
+    if [ $((RANDOM % ${TAME_DEBUG_RANDOM_NOACK:-1})) -eq 0 ]; then
+      echo "debug: TAME_DEBUG_RANDOM_NOACK triggered" >&2
+      return 2
+    fi
+  fi
+
   read -t"$TAME_CMD_WAITTIME" -r ack || return
   test "COMMAND $cmd" == "$ack" || {
     # TODO check for ack mismatch once output race condition is fixed
@@ -336,9 +371,12 @@ verify-runner-ack()
 # Assumes that the runner is ready once the pidfile becomes
 # available.  Polls for a maximum of six seconds before giving up
 # and exiting with a non-zero status.
+#
+# This will also wait for the clearing of the `reloading` flag set by
+# `reload-runner-and-wait`, as part of that same time.
 wait-for-runner()
 {
-  local -r root=${1?Missing root}
+  local -r base=${1?Missing base}
   local -r id=${2?Missing runner id}
 
   # we could use inotify, but that is not installed by default
@@ -346,7 +384,10 @@ wait-for-runner()
   # another dependency (give up after 6 seconds)
   local -i i=12
   while test $((i--)); do
-    test ! -f "$root/$id/pid" || return 0
+    if [ -f "$base/pid" -a ! -f "$base/reloading" ]; then
+      return 0
+    fi
+
     sleep 0.5
   done
 
@@ -416,7 +457,7 @@ _start-tamed()
   # wait for tamed even if it was already started (just in
   # case this script was executed right after tamed started
   # but before it is done initializing)
-  wait-for-runner "$root" 0
+  wait-for-runner "$root/0" 0
 }
 
 
@@ -499,6 +540,11 @@ Environment Variables:
                        (see also --verbose)
   TAME_CMD_WAITTIME  number of seconds to wait for ack from
                        runner (default 3)
+
+Debug Environment Variables:
+  TAME_DEBUG_RANDOM_NOACK  randomly (1-in-N, where N is the value)
+                             simulate a failed runner ACK to test
+                             the runner restart system
 EOF
 
   exit $EX_USAGE

bin/tamed

@@ -129,16 +129,43 @@ spawn-runner()
   # monitor runner usage and kill when inactive
   stall-monitor "$base" &
 
-  # loop to restart runner in case of crash
-  while true; do
-    declare -i job=0
-    trap 'kill -INT $job' HUP
-    "$mypath/dslc" < "$base/0" &> "$base/1" & job=$!
 
-    declare -i status=0
-    wait -n 2>/dev/null || status=$?
-    echo "warning: runner $id exited with code $status; restarting" >&2
-  done &
+  # loop to restart runner in case of crash
+  (
+    declare -i job=0
+    trap force-job-reload HUP
+
+    force-job-reload() {
+      kill -9 $job
+    }
+
+    while true; do
+      # if this runner is busy, then it must have terminated while
+      # processing (otherwise the client `tame` would have marked it as
+      # available); let's act on its behalf so that the client sees that we
+      # failed (which we'll represent with error code 2).
+      declare -i busy=$(< "$base/busy")
+      if runner-is-busy "$base" && ! runner-is-reloading; then
+        inject-runner-unexpected-exit "$base" "$id"
+      fi
+
+      # store the time that the runner was started so that we can later
+      # determine if it should be restarted to forcefully reclaim memory
+      date +%s > "$base/created-ts"
+
+      "$mypath/dslc" < "$base/0" &> "$base/1" & job=$!
+      runner-done-reloading
+
+      declare -i status=0
+      wait "$job" 2>/dev/null || status=$?
+
+      # 129 = signal (128) + HUP (1), which is an explicit reload request
+      # that we need not report
+      if [ "$status" -ne 129 ]; then
+        echo "warning: runner $id exited with code $status (pid $job); restarting" >&2
+      fi
+    done
+  ) &
 
   echo "$!" > "$base/pid"
 
@@ -149,6 +176,64 @@ spawn-runner()
 }
 
 
+# Whether the runner at the provided base is busy
+runner-is-busy() {
+  local -r base="$root/$id"
+
+  declare -i busy=$(< "$base/busy")
+  test "$busy" -eq 1
+}
+
+
+# Whether the runner at the provided base is flagged as having a reload
+# request
+runner-is-reloading() {
+  local -r base="$root/$id"
+
+  test -f "$base/reloading"
+}
+
+
+# Clear the runner's `reloading` flag, if any.
+#
+# This flag is set by the `tame` client before requesting a
+# reload.  Clearing this flag allows the client to observe that reloading
+# the runner is complete and requests may be issued.
+runner-done-reloading() {
+  local -r base="$root/$id"
+
+  rm -f "$base/reloading"
+}
+
+
+# Inject an exit code into the runner's output stream indicating an
+# unexpected exit
+#
+# The string `DONE n` is normally output at the end of a runner's
+# compilation (via `dslc`), where `n` is the exit code.  But if the runner
+# terminates before compilation completes (e.g. is OOM-killed), then it will
+# never have the chance to do so, leaving the client waiting for a
+# response.  If the client is not checking for stalls (due to
+# configuration), it may hang indefinitely.
+#
+# This function will inject a message into the output stream of the runner
+# as if `dslc` itself replied so that the `tame` client can observe a
+# failure and react accordingly.  This uses the `tame` `EX_UNEXPECTED` exit
+# code.
+#
+# This also outputs a warning to stderr.
+inject-runner-unexpected-exit() {
+  local -r base="${1?Missing base}"
+  local -ri id="${2?Missing id}"
+
+  echo "warning: runner $id exited unexpectedly" >&2
+
+  # TODO: Worth a shared file with `tame`?
+  local -ri EX_UNEXPECTED=7
+  echo "DONE $EX_UNEXPECTED" > "$base/1"
+}
+
+
 # Monitor the given runner runtab and append to the aggregate runtab
 #
 # The aggregate runtab is append-only and has a row-level lock to support

core/vector/table.xml

@@ -160,7 +160,11 @@
       <const name="{@__tid@}_RATE_TABLE"
              type="float"
              desc="{@__tname@} table; {@__desc@}"
-             values="@data@" />
+             values="-">
+        <!-- `@values="-"` above tells TAME to read the value from the
+             child text node -->
+        <param-copy name="@data@" />
+      </const>
     </if>
     <unless name="@data@">
       <const name="{@__tid@}_RATE_TABLE"

src/current/compiler/js-calc.xsl

@@ -133,6 +133,8 @@
 
 <template mode="js-name-ref" priority="5"
           match="c:sum[@of]|c:product[@of]">
+  <param name="symtable-map" as="map(*)" tunnel="yes" />
+
   <variable name="of" select="@of" />
   <variable name="func" select="ancestor::lv:function" />
 
@@ -142,10 +144,9 @@
     <!-- is @of a function param? -->
     <when test="
         $func
-        and root(.)/preproc:symtable/preproc:sym[
-          @type='lparam'
-          and @name=concat( ':', $func/@name, ':', $of )
-        ]
+        and $symtable-map( concat( ':', $func/@name, ':', $of ) )[
+              @type='lparam'
+            ]
       ">
 
       <value-of select="@of" />
@@ -157,13 +158,7 @@
     </when>
 
     <!-- maybe a constant? -->
-    <when test="
-        root(.)/preproc:symtable/preproc:sym[
-          @type='const'
-          and @name=$of
-        ]
-      ">
-
+    <when test="$symtable-map( $of )[ @type = 'const' ]">
       <text>C['</text>
         <value-of select="@of" />
       <text>']</text>
@@ -477,6 +472,8 @@
 <!-- TODO: this should really be decoupled -->
 <!-- TODO: does not properly support matrices -->
 <template match="c:value-of[ ancestor::lv:match ]" mode="compile-calc" priority="5">
+  <param name="symtable-map" as="map(*)" tunnel="yes" />
+
   <param name="noindex" as="xs:boolean" tunnel="yes"
          select="false()" />
 
@@ -485,8 +482,7 @@
   <choose>
     <!-- scalar -->
     <when test="
-        $noindex
-          or root(.)/preproc:symtable/preproc:sym[ @name=$name ]/@dim = '0'
+        $noindex or $symtable-map( $name )/@dim = '0'
       ">
       <apply-templates select="." mode="compile-calc-value" />
     </when>
@@ -502,196 +498,126 @@
 
 
 <!--
-  Generate JS representing the value of a function argument
+  Generate code for a `c:value-of`.
 
-  This will match whenever value-of is used on a name that matches any function
-  parameter.
-
-  XXX: We want to remain decoupled from lv if possible.
-
-  @return generated JS representing argument value or 0
+  This combines what was previously a number of individual templates into a
+  large `choose`; this is important to avoid duplicate symbol table lookups
+  and expensive XPaths as Saxon attempts to determine which template
+  matches.
 -->
 <template mode="compile-calc-value"
-  match="c:*[@name=ancestor::lv:function/lv:param/@name]">
+          match="*" priority="9">
+  <param name="symtable-map" as="map(*)" tunnel="yes" />
 
-  <!-- use the argument passed to the function -->
-  <apply-templates select="." mode="compile-calc-index">
-    <with-param name="value" select="@name" />
-  </apply-templates>
-</template>
+  <variable name="value-sym" as="element( preproc:sym )?"
+            select="$symtable-map( @name )" />
 
+  <variable name="name" as="xs:string" select="@name" />
 
-<!--
-  Using value from let expressions
--->
-<template mode="compile-calc-value"
-  match="c:*[ @name=ancestor::c:let/c:values/c:value/@name ]">
-
-  <!-- compile the value with the index (if any) -->
-  <apply-templates select="." mode="compile-calc-index">
-    <with-param name="value" select="@name" />
-  </apply-templates>
-</template>
-
-
-<!--
-  Generate JS representing the value of a global constant
-
-  Since constants are intended only to prevent magic values during development
-  (and are not required at runtime), the value of the constant will be placed
-  directly into the compiled code. However, we will *not* do this if the
-  constant is a set, since its index may be determined at runtime.
-
-  Note that "magic" constants' values are not inlined.
-
-  @return quoted constant value
--->
-<template mode="compile-calc-value"
-  match="
-    c:*[
-      @name=root(.)/preproc:symtable/preproc:sym[
-        @type='const'
-        and @dim='0'
-      ]/@name
-    ]
-  ">
-
-  <variable name="name" select="@name" />
-  <variable name="sym"
-    select="root(.)/preproc:symtable/preproc:sym[ @name=$name ]" />
-
-  <!-- it is expected that validations prior to compiling will prevent JS
-       injection here -->
   <choose>
-    <!-- "magic" constants should not have their values inlined -->
-    <when test="$sym/@magic='true'">
-      <text>C['</text>
-        <value-of select="@name" />
-      <text>']</text>
+    <!-- global scalar constant (@value is only defined when @dim is 0) -->
+    <when test="$value-sym[ @type = 'const' and @dim = '0' ]">
+      <variable name="value" as="xs:string"
+                select="$value-sym/@value" />
+
+      <!-- note: this used to check for @magic, which has long since been
+           removed -->
+      <sequence select="$value" />
+    </when>
+
+    <!-- non-scalar constants are large and are stored statically for
+         reference rather than being inlined  -->
+    <when test="$value-sym[ @type = 'const' and not( @dim = '0' ) ]">
+      <variable name="value">
+        <text>C['</text>
+          <value-of select="$name" />
+        <text>']</text>
+      </variable>
+
+      <apply-templates select="." mode="compile-calc-index">
+        <with-param name="value" select="$value" />
+      </apply-templates>
+    </when>
+
+    <!-- TODO: is this worth a tunnel instead of `ancestor::`? -->
+    <!-- generator index -->
+    <when test="$name = ancestor::c:*[ @of ]/@index">
+      <!-- TODO: This shouldn't be true anymore; is this cast still needed? -->
+      <!-- depending on how the index is generated, it could be a string, so
+           we must cast it -->
+      <text>+</text>
+      <value-of select="$name" />
+    </when>
+
+    <!-- function argument -->
+    <when test="$name = ancestor::lv:function/lv:param/@name">
+      <!-- use the argument passed to the function -->
+      <apply-templates select="." mode="compile-calc-index">
+          <with-param name="value" select="$name" />
+      </apply-templates>
+    </when>
+
+    <!-- let expression value -->
+    <when test="$name = ancestor::c:let/c:values/c:value/@name">
+      <!-- compile the value with the index (if any) -->
+      <apply-templates select="." mode="compile-calc-index">
+        <with-param name="value" select="$name" />
+      </apply-templates>
     </when>
 
-    <!-- @value should be defined when @dim=0 -->
     <otherwise>
-      <value-of select="$sym/@value" />
+      <variable name="dim" as="xs:string"
+                select="$value-sym/@dim" />
+
+      <variable name="value">
+        <text>A['</text>
+          <value-of select="@name" />
+        <text>']</text>
+      </variable>
+
+      <!-- the awkward double-negatives are intentional, since @index may not
+           exist; here's what we're doing:
+
+          - If it's not a set, then indexes are irrelevant; always cast scalars
+          - Otherwise
+            - If an index was provided and it is not a matrix, cast
+            - Otherwise
+              - If two indexes were provided and it is a matrix, cast
+      -->
+      <!-- N.B. it is important to do this outside the value variable, otherwise the
+           cast may be done at the incorrect time -->
+      <if test="
+          (
+            $dim='0'
+            or (
+              (
+                ( @index and not( @index = '' ) )
+                or ( ./c:index )
+              )
+              and not( $dim='2' )
+            )
+            or (
+              ( $dim='2' )
+              and ./c:index[2]
+            )
+          )
+          and not(
+            parent::c:arg
+            and not( @index )
+          )
+        ">
+
+        <text>+</text>
+      </if>
+
+      <apply-templates select="." mode="compile-calc-index">
+        <with-param name="value" select="$value" />
+      </apply-templates>
     </otherwise>
   </choose>
 </template>
 
 
-<!--
-  Generates JS representing the value of a constant as part of a set
-
-  Since the index of constant sets can be determined at runtime, we need to
-  store all possible values. As such, we shouldn't repeat ourselves by inlining
-  all possible values; instead, we'll reference a pre-generated set of values
-  for the particular constant.
-
-  @return generated code representing value of a variable, or 0 if undefined
--->
-<template mode="compile-calc-value"
-  match="
-    c:*[
-      @name=root(.)/preproc:symtable/preproc:sym[
-        @type='const'
-        and not( @dim='0' )
-      ]/@name
-    ]
-  ">
-
-  <variable name="value">
-    <text>C['</text>
-      <value-of select="@name" />
-    <text>']</text>
-  </variable>
-
-  <apply-templates select="." mode="compile-calc-index">
-    <with-param name="value" select="$value" />
-  </apply-templates>
-</template>
-
-
-<!--
-  Generate JS representing the value of a generated index
-
-  @return generated code associated with the value of the generated index
--->
-<template mode="compile-calc-value"
-  match="c:*[ @name = ancestor::c:*[ @of ]/@index ]">
-
-  <!-- depending on how the index is generated, it could be a string, so we must
-       cast it -->
-  <text>+</text>
-  <value-of select="@name" />
-</template>
-
-
-<!--
-  Generates JS representing the value of a variable
-
-  If the variable is undefined, the value will be considered to be 0 (this is
-  especially important for the summation of sets within this implementation).
-  That is: a value will never be considered undefined.
-
-  @return generated code representing value of a variable, or 0 if undefined
--->
-<template match="c:*" mode="compile-calc-value">
-  <variable name="name" select="@name" />
-  <variable name="pkg" as="element( lv:package )"
-                select="root(.)" />
-
-  <variable name="dim"
-                select="$pkg/preproc:symtable/preproc:sym[ @name=$name ]/@dim" />
-
-  <!-- retrieve the value, casting to a number (to avoid potentially nasty
-       string concatenation bugs instead of integer/floating point arithmetic)
-       as long as we're either not a set, or provide an index for the set -->
-  <variable name="value">
-    <text>A['</text>
-      <value-of select="@name" />
-    <text>']</text>
-  </variable>
-
-  <!-- the awkward double-negatives are intentional, since @index may not
-       exist; here's what we're doing:
-
-      - If it's not a set, then indexes are irrelevant; always cast scalars
-      - Otherwise
-        - If an index was provided and it is not a matrix, cast
-        - Otherwise
-          - If two indexes were provided and it is a matrix, cast
-  -->
-  <!-- N.B. it is important to do this outside the value variable, otherwise the
-       cast may be done at the incorrect time -->
-  <if test="
-      (
-        $dim='0'
-        or (
-          (
-            ( @index and not( @index = '' ) )
-            or ( ./c:index )
-          )
-          and not( $dim='2' )
-        )
-        or (
-          ( $dim='2' )
-          and ./c:index[2]
-        )
-      )
-      and not(
-        parent::c:arg
-        and not( @index )
-      )
-    ">
-
-    <text>+</text>
-  </if>
-
-  <apply-templates select="." mode="compile-calc-index">
-    <with-param name="value" select="$value" />
-  </apply-templates>
-</template>
-
-
 <!--
   Include the index if one was provided
 
@@ -712,11 +638,16 @@
   @return index (including brackets), if one was provided
 -->
 <template match="c:*" mode="compile-calc-index">
+  <param name="symtable-map" as="map(*)" tunnel="yes" />
   <param name="value" />
+
   <variable name="index" select="@index" />
 
   <choose>
-    <when test="@index">
+    <when test="$index">
+      <variable name="index-sym" as="element( preproc:sym )?"
+                select="$symtable-map( $index )" />
+
       <text>(</text>
       <value-of select="$value" />
 
@@ -736,13 +667,11 @@
         </when>
 
         <!-- scalar constant -->
-        <when test="@index = root(.)/preproc:symtable/preproc:sym
-                          [ @type='const'
-                            and @dim='0' ]
-                          /@name">
-          <value-of select="root(.)/preproc:symtable
-                                  /preproc:sym[ @name=$index ]
-                                  /@value" />
+        <when test="$index-sym[ @type='const' and @dim='0' ]">
+          <variable name="value" as="xs:string"
+                    select="$index-sym/@value" />
+
+          <value-of select="$value" />
         </when>
 
         <!-- otherwise, it's a variable -->
@@ -849,8 +778,12 @@
   @return generated function application
 -->
 <template match="c:apply" mode="compile-calc" priority="5">
+  <param name="symtable-map" as="map(*)" tunnel="yes" />
+
   <variable name="name" select="@name" />
   <variable name="self" select="." />
+  <variable name="func-sym" as="element( preproc:sym )"
+            select="$symtable-map( $name )[ @type = 'func' ]" />
 
   <call-template name="calc-compiler:gen-func-name">
     <with-param name="name" select="@name" />
@@ -862,13 +795,7 @@
 
   <!-- generate argument list in the order that the arguments are expected (they
        can be entered in the XML in any order) -->
-  <for-each select="
-      root(.)/preproc:symtable/preproc:sym[
-        @type='func'
-        and @name=$name
-      ]/preproc:sym-ref
-    ">
-
+  <for-each select="$func-sym/preproc:sym-ref">
     <text>, </text>
 
     <variable name="pname" select="substring-after( @name, $arg-prefix )" />
@@ -947,8 +874,12 @@
 -->
 <template mode="compile-calc" priority="7"
           match="c:apply[ compiler:apply-uses-tco( . ) ]">
+  <param name="symtable-map" as="map(*)" tunnel="yes" />
+
   <variable name="name" select="@name" />
   <variable name="self" select="." />
+  <variable name="func-sym" as="element( preproc:sym )"
+            select="$symtable-map( $name )[ @type = 'func' ]" />
 
   <message select="concat('warning: ', $name, ' recursing with experimental guided TCO')" />
 
@@ -958,13 +889,7 @@
 
   <!-- reassign function arguments -->
   <variable name="args" as="element(c:arg)*">
-    <for-each select="
-        root(.)/preproc:symtable/preproc:sym[
-          @type='func'
-          and @name=$name
-        ]/preproc:sym-ref
-      ">
-
+    <for-each select="$func-sym/preproc:sym-ref">
       <variable name="pname" select="substring-after( @name, $arg-prefix )" />
       <variable name="arg" select="$self/c:arg[@name=$pname]" />
 

src/current/compiler/js.xsl

@@ -41,8 +41,6 @@
 
 <!-- legacy classification system -->
 <include href="js-legacy.xsl" />
-<!-- and whether to enable it (set to non-empty string) -->
-<param name="legacy-classify" select="''" />
 
 <!-- calculation compiler -->
 <include href="js-calc.xsl" />
@@ -474,7 +472,7 @@
   <param name="const" as="element( lv:const )" />
 
   <variable name="values-def" as="xs:string?"
-            select="$const/@values" />
+            select="compiler:const-values( $const )" />
 
   <choose>
     <when test="$values-def and contains( $values-def, ';' )">
@@ -489,6 +487,21 @@
 </function>
 
 
+<function name="compiler:const-values" as="xs:string?">
+  <param name="const" as="element( lv:const )" />
+
+  <!-- @values="-", a convention from command-line programs where '-' means
+       "read from stdin", will take the value from the child text of the
+       constant; this is done because Saxon performs very, very poorly on
+       huge single-line attributes (e.g. 60s for ~20MiB single-line
+       attribute) -->
+  <sequence select="if ( $const/@values = '-' ) then
+                        $const/text()
+                      else
+                        $const/@values" />
+</function>
+
+
 <!--
   Produce a sequence of items
 
@@ -507,7 +520,9 @@
 
     <when test="$set/@values and $allow-values">
       <sequence select="tokenize(
-                          normalize-space( $set/@values ), ',' )" />
+                          normalize-space(
+                            compiler:const-values( $set ) ),
+                          ',' )" />
     </when>
 
     <otherwise>
@@ -641,20 +656,6 @@
 </function>
 
 
-<template mode="compile" priority="6"
-          match="lv:classify[ compiler:use-legacy-classify() ]">
-  <param name="symtable-map" as="map(*)" tunnel="yes" />
-
-  <sequence select="concat(
-                      $compiler:nl,
-                      '/*!lc*/',
-                      string-join(
-                        compiler:compile-classify-legacy( $symtable-map, . ),
-                        '' ),
-                      '/*lc!*/' )" />
-</template>
-
-
 <template match="lv:classify" mode="compile" priority="5">
   <param name="symtable-map" as="map(*)" tunnel="yes" />
 
@@ -664,8 +665,7 @@
 
 <template mode="compile" priority="8"
           match="lv:classify[
-                   @preproc:inline='true'
-                   and not( compiler:use-legacy-classify() ) ]">
+                   @preproc:inline='true' ]">
   <!-- emit nothing; it'll be inlined at the match site -->
 </template>
 
@@ -708,14 +708,6 @@
 </function>
 
 
-<function name="compiler:use-legacy-classify" as="xs:boolean">
-  <variable name="flagname" as="xs:string"
-            select="'___feature-newclassify'" />
-
-  <sequence select="$legacy-classify != ''" />
-</function>
-
-
 <function name="compiler:compile-classify-assign" as="xs:string">
   <param name="symtable-map" as="map(*)" />
   <param name="classify" as="element( lv:classify )" />
@@ -2476,39 +2468,19 @@
 ]]>
 </text>
 
-<choose>
-  <when test="compiler:use-legacy-classify()">
-    <text>
-    function div(x, y)
-    {
-        return x / y;
-    }
+<text>
+function div(x, y)
+{
+    if (y === 0) return 0;
+    return x / y;
+}
 
-    function pow(x, p)
-    {
-        return Math.pow(x, p);
-    }
-    </text>
-  </when>
-
-  <!-- x/0=0 introduced with new classification system; see commit message
-       for more detailed information -->
-  <otherwise>
-    <text>
-    function div(x, y)
-    {
-        if (y === 0) return 0;
-        return x / y;
-    }
-
-    function pow(x, p)
-    {
-        if (x === 0) return 0;
-        return Math.pow(x, p);
-    }
-    </text>
-  </otherwise>
-</choose>
+function pow(x, p)
+{
+    if (x === 0) return 0;
+    return Math.pow(x, p);
+}
+</text>
 
 <sequence select="unparsed-text(
                     concat( $__path-root, '/src/js/sha256.js' ) )" />

src/current/compiler/map.xsl

@@ -319,7 +319,7 @@
 <template match="lvm:pass[ root(.)/@lvmc:type = 'retmap' ]"
           mode="preproc:depgen" priority="6">
   <preproc:sym-dep name=":retmap:{@name}">
-    <preproc:sym-ref name="{@name}" lax="true" />
+    <preproc:sym-ref name="{@name}" />
   </preproc:sym-dep>
 </template>
 
@@ -378,7 +378,7 @@
                           and root(.)/@lvmc:type = 'retmap' ]"
               mode="preproc:depgen" priority="6">
   <preproc:sym-dep name=":retmap:{@to}">
-    <preproc:sym-ref name="{@from}" lax="true" />
+    <preproc:sym-ref name="{@from}" />
   </preproc:sym-dep>
 </template>
 
@@ -596,7 +596,7 @@
               mode="preproc:depgen" priority="6">
   <preproc:sym-dep name=":retmap:{@to}">
     <for-each select=".//lvm:from">
-      <preproc:sym-ref name="{@name}" lax="true" />
+      <preproc:sym-ref name="{@name}" />
     </for-each>
   </preproc:sym-dep>
 </template>

src/current/compiler/validate.xsl

@@ -270,29 +270,15 @@
 
 
 <template match="lv:match[@pattern]" mode="lvv:validate-match" priority="9">
-  <choose>
-    <!-- warn of upcoming removal -->
-    <when test="compiler:use-legacy-classify()">
-      <message select="concat( 'warning: ',
-                               ancestor::lv:classify/@as,
-                               ': lv:match[@pattern] support is deprecated ',
-                               'and is removed with the new classification ',
-                               'system; use lookup tables instead' )" />
-    </when>
-
-    <!-- @pattern support removed in the new classification system -->
-    <otherwise>
-      <call-template name="lvv:error">
-        <with-param name="desc" select="'lv:match[@pattern] support removed'" />
-        <with-param name="refnode" select="." />
-        <with-param name="content">
-          <text>use lookup tables in place of @pattern in `</text>
-          <value-of select="parent::lv:classify/@as" />
-          <text>'</text>
-        </with-param>
-      </call-template>
-    </otherwise>
-  </choose>
+  <call-template name="lvv:error">
+    <with-param name="desc" select="'lv:match[@pattern] support removed'" />
+    <with-param name="refnode" select="." />
+    <with-param name="content">
+      <text>use lookup tables in place of @pattern in `</text>
+      <value-of select="parent::lv:classify/@as" />
+      <text>'</text>
+    </with-param>
+  </call-template>
 </template>
 
 

src/current/include/depgen.xsl

@@ -39,15 +39,14 @@
             exclude-result-prefixes="ext util xs">
 
 
-<variable name="tex-defaults">
-  <preproc:syms>
-    <preproc:sym value="\alpha" vec="A" />
-    <preproc:sym value="\beta" vec="B" />
-    <preproc:sym value="\gamma" vec="\Gamma" />
-    <preproc:sym value="x" vec="X" />
-    <preproc:sym value="y" vec="Y" />
-    <preproc:sym value="z" vec="Z" />
-  </preproc:syms>
+  <!-- see preproc:tex-gen#3 -->
+<variable name="tex-defaults" as="element( preproc:tex )+">
+  <preproc:tex value="\alpha" vec="A" />
+  <preproc:tex value="\beta" vec="B" />
+  <preproc:tex value="\gamma" vec="\Gamma" />
+  <preproc:tex value="x" vec="X" />
+  <preproc:tex value="y" vec="Y" />
+  <preproc:tex value="z" vec="Z" />
 </variable>
 
 
@@ -108,48 +107,37 @@
     <for-each select="$deps, $deps//preproc:sym-dep">
       <variable name="sym-name" as="xs:string"
                     select="@name" />
-
       <variable name="cursym" as="element( preproc:sym )?"
                     select="$symtable-map( $sym-name )" />
 
       <if test="not( $cursym )">
-        <message select="." />
         <message terminate="yes"
                      select="concat( 'internal error: ',
                                      'cannot find symbol in symbol table: ',
                                      '`', $sym-name, '''' )" />
       </if>
 
-      <!-- do not output duplicates (we used to not output references
-           to ourselves, but we are now retaining them, since those
-           data are useful) -->
-      <variable name="uniq" select="
-          preproc:sym-ref[
-            not( @name=preceding-sibling::preproc:sym-ref/@name )
-          ]
-        " />
-
-      <!-- symbols must not have themselves as their own dependency -->
-      <if test="$uniq[ not( $cursym/@allow-circular = 'true' )
-                           and ( @name = $cursym/@name
-                                 or @parent = $cursym/@name ) ]">
-        <message terminate="yes"
+      <preproc:sym-dep name="{@name}">
+        <!-- @tex provided an non-empty, or function -->
+        <for-each-group select="preproc:sym-ref"
+                        group-by="@name">
+          <!-- symbols must not have themselves as their own dependency -->
+          <if test="not( $cursym/@allow-circular = 'true' )
+                      and ( @name = $cursym/@name
+                            or @parent = $cursym/@name )">
+            <message terminate="yes"
                      select="concat( '[preproc] !!! fatal: symbol ',
                                      $cursym/@name,
                                      ' references itself ',
                                      '(circular dependency)' )" />
-      </if>
+          </if>
 
-      <!-- grab the original source symbol for these references and augment them
-           with any additional dependency metadata -->
-      <variable name="syms-rtf">
-        <for-each select="$uniq">
           <variable name="name" select="@name" />
           <variable name="sym" as="element( preproc:sym )?"
                         select="$symtable-map( $name )" />
 
           <!-- we should never have this problem. -->
-          <if test="not( $sym ) and not( @lax='true' )">
+          <if test="not( $sym )">
             <message terminate="yes">
               <text>[depgen] internal error: </text>
               <text>could not locate dependency symbol `</text>
@@ -159,140 +147,84 @@
             </message>
           </if>
 
-          <!-- copy and augment (we set @name because $sym/@name may not exist
-               if @lax) -->
-          <preproc:sym name="{@name}">
-            <if test="$sym">
-              <sequence select="$sym/@*" />
-            </if>
-
-            <preproc:meta>
-              <!-- retain type -->
-              <sequence select="$sym/@type" />
-              <sequence select="$sym/@dim" />
-
-              <!-- copy any additional metadata -->
-              <sequence select="@*[ not( local-name() = 'name' ) ]" />
-            </preproc:meta>
-          </preproc:sym>
-        </for-each>
-      </variable>
-
-      <variable name="syms" select="$syms-rtf/preproc:sym" />
-
-      <!-- only applicable if the symbol is @lax and the symbol was not
-           found in the local symbol table -->
-      <variable name="lax" select="
-          $uniq[
-            @lax='true'
-            and not( @name=$syms/@name )
-          ]
-        " />
-
-      <preproc:sym-dep name="{@name}">
-        <!-- process symbols that have not been found in the local symbol
-             table (only applicable when cursym is @lax) -->
-        <for-each select="$lax">
-          <!-- the @lax flag here is simply to denote that this symbol may not
-               actually exist and that ignoring the check was explicitly
-               requested (and not a bug in the depgen process) -->
-          <preproc:sym-ref name="{@name}" lax="true">
-            <sequence select="preproc:meta/@*" />
-          </preproc:sym-ref>
-        </for-each>
-
-        <!-- @tex provided an non-empty, or function -->
-        <for-each select="
-          $syms[
-            ( @tex and not( @tex='' ) )
-            or @type='func'
-          ]">
-
-          <choose>
-            <!-- even if function, @tex overrides symbol -->
-            <when test="@tex and not( @tex='' )">
-              <preproc:sym-ref tex="{@tex}">
-                <sequence select="@*" />
-                <sequence select="preproc:meta/@*" />
-              </preproc:sym-ref>
-            </when>
-
-            <!-- must be a function; use its name -->
-            <otherwise>
-              <preproc:sym-ref>
-                <sequence select="@*" />
-                <sequence select="preproc:meta/@*" />
-
-                <attribute name="tex">
-                  <text>\textrm{</text>
-                    <value-of select="@name" />
-                  <text>}</text>
-                </attribute>
-              </preproc:sym-ref>
-            </otherwise>
-          </choose>
-        </for-each>
-
-        <!-- no @tex, @tex empty, no function -->
-        <for-each select="
-          $syms[
-            ( not( @tex ) or @tex='' )
-            and not( @type='func' )
-          ]">
-
-          <variable name="name" select="@name" />
-          <variable name="sym" select="." />
-
           <preproc:sym-ref>
             <!-- minimal attribute copy (avoid data duplication as much as
                  possible to reduce modification headaches later on) -->
-            <sequence select="@name, @parent" />
-            <sequence select="preproc:meta/@*" />
+            <sequence select="$sym/@name,
+                              $sym/@parent,
+                              $sym/@type,
+                              $sym/@dim,
+                              $sym/@tex" />
 
-            <!-- assign a symbol -->
-            <variable name="pos" select="position()" />
-            <attribute name="tex">
-              <variable name="texsym" select="
-                  $tex-defaults/preproc:syms/preproc:sym[
-                    position() = $pos
-                  ]
-                " />
+            <!-- copy any additional metadata -->
+            <sequence select="@*[ not( local-name() = 'name' ) ]" />
 
-              <choose>
-                <when test="$sym/@tex and not( $sym/@tex='' )">
-                  <value-of select="$sym/@tex" />
-                </when>
+            <variable name="tex" as="xs:string?"
+                      select="if ( @tex ) then @tex else $sym/@tex" />
 
-                <!-- scalar/vector default -->
-                <when test="$texsym and number( $sym/@dim ) lt 2">
-                  <value-of select="$texsym/@value" />
-                </when>
-
-                <!-- matrix default -->
-                <when test="$texsym">
-                  <value-of select="$texsym/@vec" />
-                </when>
-
-                <!-- no default available; generate one -->
-                <otherwise>
-                  <value-of select="
-                      if ( number( $sym/@dim ) lt 2 ) then '\theta'
-                      else '\Theta'
-                    " />
-                  <text>_{</text>
-                    <value-of select="$pos" />
-                  <text>}</text>
-                </otherwise>
-              </choose>
-            </attribute>
+            <attribute name="tex" select="
+              if ( $tex and not( $tex = '' ) ) then
+                  $tex
+                else
+                  preproc:tex-gen( @name,
+                                   @type,
+                                   number( $sym/@dim ),
+                                   position() )" />
           </preproc:sym-ref>
-        </for-each>
+        </for-each-group>
       </preproc:sym-dep>
     </for-each>
   </preproc:sym-deps>
 </template>
 
 
+<!--
+  Generate a default TeX symbol given a symbol's relative position within
+  some context.
+
+  Generally, the relative position is with respect to a parent
+  identifier.  The TeX symbols are typically used in a let list to visually
+  define variables that are later used in rendered expressions (on e.g. the
+  Summary Page).
+-->
+<function name="preproc:tex-gen" as="xs:string">
+  <param name="name" as="xs:string" />
+  <param name="type" as="xs:string" />
+  <param name="dim" as="xs:double" />
+  <param name="relpos" as="xs:integer" />
+
+  <!-- TODO: TAMER'll probably address this before it's fixed here, but this
+       may fail to generate a symbol! -->
+  <variable name="texsym" as="element( preproc:tex )?"
+            select="$tex-defaults[ $relpos ]" />
+
+  <choose>
+    <when test="$type = 'func'">
+      <sequence select="concat( '\textrm{', $name, '}' )" />
+    </when>
+
+    <!-- scalar/vector default -->
+    <when test="$texsym and $dim lt 2">
+      <sequence select="string( $texsym/@value )" />
+    </when>
+
+    <!-- matrix default -->
+    <when test="$texsym">
+      <sequence select="string( $texsym/@vec )" />
+    </when>
+
+    <otherwise>
+      <variable name="theta" as="xs:string"
+                select="if ( $dim lt 2 ) then
+                            '\theta'
+                          else
+                            '\Theta'" />
+      <sequence select="concat( $theta, '_{', $relpos, '}' )" />
+    </otherwise>
+  </choose>
+</function>
+
+
 <template mode="preproc:depgen" priority="7"
               match="lv:rate">
   <preproc:sym-dep name="{@yields}">

src/current/include/dslc-base.xsl

@@ -26,7 +26,9 @@
   template that is intended for use with dslc should include this.
 -->
 <stylesheet version="2.0"
-            xmlns="http://www.w3.org/1999/XSL/Transform">
+            xmlns="http://www.w3.org/1999/XSL/Transform"
+            xmlns:xs="http://www.w3.org/2001/XMLSchema"
+            xmlns:preproc="http://www.lovullo.com/rater/preproc">
 
 <!--
   Absolute path to root of TAME
@@ -71,14 +73,25 @@
 -->
 <param name="__relroot" />
 
-
 <!--
-  Random value that may be used to seed random values
+  A package-unique string
 
-  XSLT is deterministic and does not offer support for generating random values;
-  its generate-id() function is not sufficient for cross-package generation.
+  You should use `preproc:pkg-generate-id` instead of this value directly.
+
+  This value is deterministic, derived from `__srcpkg`, and so will not
+  change between runs; it can be used to generate identifier names that are
+  unique across packages, which is not something that we can rely on
+  `generate-id()` for on its own.
+
+  In practice, this can be concatenated with other generated strings,
+  including `generate-id()`-derived strings.
+
+  _There is no guarantee that this string will begin with a letter_, so you
+  should generate your identifiers accordingly.
+
+  See `DslCompiler.java` for implementation.
 -->
-<param name="__rseed" />
+<param name="__pkguniq" as="xs:string" />
 
 
 <!--
@@ -114,4 +127,12 @@
   </choose>
 </template>
 
+<function name="preproc:pkg-generate-id" as="xs:string">
+  <param name="refnode" as="node()" />
+
+  <sequence select="concat(
+                      '_pu', $__pkguniq, '_',
+                      generate-id( $refnode ) )" />
+</function>
+
 </stylesheet>