bin/: Server/client build scripts

These scripts allow the TAME compiler stack to be invoked naturally, rather
than requiring the use of a Makefile today.  This will not only allow users
to more easily invoke the compiler, but will also allow us to invoke TAME
naturally from Makefile and remove the klugery that has existed for so
long.

This users a server/client architecture in order to mitigate the startup
cost of the JVM.  More documentation will follow.

Note that there are a bunch of symlinks in rater/---this is a transition
step to allow the build to continue working as it did before, which relies
on a directory structure that exists outside of this repository.  This will
be cleaned up in the future.

* .gitignore (bin/dslc): Add ignore for generated file.
* bin/dslc.in: New script to encapsulate Java invocation.
* bin/tame: New script (client).
* bin/tamed: New script (server).
* configure.ac (JAVA_OPTS, DSLC_CLASSPATH, AUTOGENERATED): New variables for
  dslc.in.  Output bin/dslc.
* rater/README.md: Note that this symlink mess is temporary.
* rater/c1map: New symlink for dslc assumptions.
* rater/c1map.xsl: Likewise.
* rater/calc.xsd: Likewise.
* rater/compile.xsl: Likewise.
* rater/compiler: Likewise.
* rater/dot.xsl: Likewise.
* rater/include: Likewise.
* rater/link.xsl: Likewise.
* rater/standalone.xsl: Likewise.
* rater/summary.xsl: Likewise.
* rater/tame: Likewise (warning: circular symlink).
* src/current/src/com/lovullo/dslc/DslCompiler.java (_DslCompiler)[compile]:
  Output `DONE' lines.

.gitignore

@@ -7,6 +7,7 @@
 *.info
 
 # autotools- and configure-generated
+/bin/dslc
 /Makefile.in
 /Makefile
 /aclocal.m4

bin/dslc.in

@@ -0,0 +1,28 @@
+#!/bin/bash
+# Listen for TAME commands (compilers, linker, etc)
+#
+#   Copyright (C) 2018 R-T Specialty, LLC.
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# @AUTOGENERATED@
+##
+
+declare -r mypath=$( dirname "$( readlink -f "$0" )" )
+declare -r dslc_jar="$mypath/../src/current/src/dslc.jar"
+
+
+CLASSPATH="$CLASSPATH:@DSLC_CLASSPATH@:$dslc_jar" \
+  "@JAVA@" @JAVA_OPTS@ \
+  com.lovullo.dslc.DslCompiler

bin/tame

@@ -0,0 +1,220 @@
+#!/bin/bash
+# Client for TAME daemon (tamed)
+#
+#   Copyright (C) 2018 R-T Specialty, LLC.
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+##
+
+set -euo pipefail
+
+declare -r mypath=$( dirname "$( readlink -f "$0" )" )
+
+declare -ri EX_NOTAMED=1  # tried to start tamed but failed
+declare -ri EX_USAGE=64   # incorrect usage; sysexits.h
+
+
+# Send a single command to a runner and observe the result
+#
+# stdin will be directed to the runner.  stdout of the runner will be
+# echoed until a line beginning with "DONE" is found, after which this
+# procedure will return with the exit code indicated by the runner.
+command-runner()
+{
+  local -ri id="${1?Missing id}"
+  local -r root="${2?Missing root run path}"
+  shift 2
+
+  local -r base="$root/$id"
+  local -ri pid=$( cat "$base/pid" )
+
+  # TODO flock
+
+  verify-runner "$base" "$pid"
+
+  # all remaining arguments are passed to the runner
+  echo "$@" > "$base/0"
+
+  # output lines from runner until we reach a line stating "DONE"
+  while read line; do
+    # don't parse words in the initial read because we may be
+    # dealing with a lot of lines
+    if [ "${line:0:5}" == "DONE " ]; then
+      read _ code _ <<< "$line"
+      return "$code"
+    fi
+
+    echo "$line"
+  done < "$base/1"
+}
+
+
+# Verify that a runner is available
+#
+# If the runner is offline or not owned by $UID, then exit with
+# a non-zero status.
+verify-runner()
+{
+  local -r base="${1?Missing base}"
+  local -ri pid="${2?Missing pid}"
+
+  ps "$pid" &>/dev/null || {
+    echo "error: runner $id ($pid) is offline!" >&2
+    exit "$EX_NOTAMED"
+  }
+
+  test -O "$base/0" || {
+    echo "error: runner $id ($pid) is not owned by $USER!" >&2
+    exit "$EX_NOTAMED"
+  }
+}
+
+
+# Wait somewhat impatiently for tamed
+#
+# Assumes that tamed's runner 0 is running once the pidfile becomes
+# available.  Polls for a maximum of six seconds before giving up
+# and exiting with a non-zero status.
+wait-for-tamed()
+{
+  local -r base="${1?Missing base}"
+
+  # we could use inotify, but that is not installed by default
+  # on Debian systems, so let's just poll rather than introduce
+  # another dependency (give up after 6 seconds)
+  local -i i=12
+  while test $((i--)); do
+    test ! -f "$base/0/pid" || return 0
+    sleep 0.5
+  done
+
+  # still not available
+  echo 'error: tamed still unavailable; giving up' >&2
+  exit "$EX_NOTAMED"
+}
+
+
+# Start tamed if it is not already running
+#
+# If tamed is already running, nothing will happen; otherwise, start
+# tamed and wait impatiently for the runner to become available.
+#
+# Even if tamed is started, wait for runner 0 to become available;
+# this ensures that tamed is initialized even if this script is run
+# after tamed is started but before it has fully come online (e.g
+# parallel make).
+start-tamed()
+{
+  local -r root="${1?Missing root}"
+
+  local -ri pid=$( cat "$root/pid" 2>/dev/null )
+
+  ps "$pid" &>/dev/null || {
+    echo "starting tamed at $root..."
+
+    # tell tamed to clean up so that we eliminate race conditions
+    # with wait-for-tamed (this will also kill any stray processes
+    # that a previous tamed may have spawned but didn't get the
+    # chance to clean up)
+    kill-tamed "$root" || true
+
+    # start tamed and allow it to persist for future commands
+    "$mypath/tamed" "$root" & disown
+  }
+
+  # wait for tamed even if it was already started (just in
+  # case this script was executed right after tamed started
+  # but before it is done initializing)
+  wait-for-tamed "$root"
+}
+
+
+# Kill tamed
+#
+# Ask tamed to kill itself.
+kill-tamed()
+{
+  local -r root="${1?Missing root}"
+
+  "$mypath/tamed" --kill "$root"
+}
+
+
+# Filter dslc output to essential information
+#
+# The original output of dslc is quite noisy; this filters it down
+# to only errors and warnings.
+#
+# Eventually, dslc out to be modified to handle filtering its own
+# output rather than wasting cycles doing this filtering.
+saneout()
+{
+  awk ' \
+    /^~~~~\[begin /,/^~~~~\[end / { next } \
+    /^rm / { next } \
+    /^Exception|^\t+at / { \
+      if ( /^E/ ) { \
+        print; \
+        print "Stack trace written to .runlog"; \
+      } \
+      next; \
+     } \
+    /([Ww]arning|[Nn]otice)[: ]/ { printf "\033[0;33m"; w++; out=1; } \
+    /[Ff]atal:/ { printf "\033[0;31m"; out=1; } \
+    /!|[Ee]rror:/ { printf "\033[0;31m"; e++; out=1; } \
+    /internal:/ { printf "\033[0;35m"; out=1; } \
+    /internal error:/ { printf "\033[1m"; out=1; } \
+    /^[^[]/ || out { print; printf "\033[0;0m"; out=0; } \
+  '
+}
+
+
+# Output usage information and exit
+usage()
+{
+  cat <<EOF
+Usage: $0 cmdline
+Or:    $0 --kill
+Send command line CMDLINE to a tamed runner.  Start tamed if
+not already running.
+
+Options:
+  --kill   kill tamed
+  --help   show this message
+EOF
+
+  exit $EX_USAGE
+}
+
+
+# Run tame
+main()
+{
+  local -r root=/run/user/$UID/tamed
+
+  test $# -gt 0 || usage
+
+  case "${1:-}" in
+    --kill) kill-tamed "$root"; exit;;
+    --help) usage;;
+  esac
+
+  start-tamed "$root"
+
+  # for now we only support a single runner
+  command-runner 0 "$root" "$@" | saneout
+}
+
+main "$@"
+

bin/tamed

@@ -0,0 +1,217 @@
+#!/bin/bash
+# Daemon for accepting TAME commands (compilers, linker, etc)
+#
+#   Copyright (C) 2018 R-T Specialty, LLC.
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+##
+
+set -euo pipefail
+
+declare -r mypath=$( dirname "$( readlink -f "$0" )" )
+
+declare -ri EX_RUNNING=1
+declare -ri EX_USAGE=64      # incorrect usage; sysexits.h
+declare -ri EX_CANTCREAT=73  # cannot create file; sysexits.h
+
+# set by `main', global for `cleanup'
+declare root=
+
+
+# Create FIFOs for runner
+#
+# The FIFOs are intended to be attached to stderr and stdout
+# of the runner and will be created relative to the given
+# root path ROOT.
+#
+# If a FIFO cannot be created, exit with EX_CANTCREAT.
+mkfifos()
+{
+  local -r root="${1?Missing root path}"
+
+  mkdir -p "$root"
+
+  # note that there's no stderr; see `add-runner'
+  for n in 0 1; do
+    rm -f "$root-$n"
+
+    mkfifo -m 0600 "$root/$n" || {
+      echo "fatal: failed to create FIFO at $in"
+      exit $EX_CANTCREAT
+    }
+  done
+}
+
+
+# Spawn a runner
+#
+# A new runner is created by spawning dslc and attaching
+# new FIFOs under the given id ID relative to the given
+# run path ROOT.  The PID of the runner will be stored
+# alongside the FIFOs in a pidfile `pid'.
+spawn-runner()
+{
+  local -ri id="${1?Missing id}"
+  local -r root="${2?Missing root run path}"
+
+  local -r base="$root/$id"
+
+  mkfifos "$base"
+
+  # TODO: should we separate back out std{out,err}?
+  # XXX: why does dslc quit (with a 0 exit code) occsionally? stdin?
+  while true; do
+    "$mypath/dslc" < <( persistent-cat "$base/0" ) \
+                   >"$base/1" \
+                   2>&1
+    echo "warning: runner $id exited with code $?; restarting"
+  done &
+
+  echo "$!" > "$base/pid"
+
+  echo "runner $id ($!): $base"
+}
+
+
+# Persistently read commands from FIFO IN
+#
+# This will continue to read from the FIFO as long as it is
+# readable.  This is necessary since SIGPIPE gets sent to
+# processes reading/writing from/to the FIFO whenever a
+# process detaches from it.
+persistent-cat()
+{
+  local -r in="${1?Missing input path}"
+
+  while test -r "$in"; do
+    read -r < "$in" || return
+    echo "$REPLY"
+  done
+}
+
+
+# Exit if tamed is already running at path ROOT
+#
+# If tamed is already running at ROOT, exit with status
+# EX_RUNNING; otherwise, do nothing except output a warning
+# if a stale pid file exists.
+abort-if-running()
+{
+  local -r root="${1?Missing root rundir}"
+
+  local -ri pid=$( cat "$root/pid" 2>/dev/null )
+
+  test "$pid" -gt 0 || return 0
+
+  ! ps "$pid" &>/dev/null || {
+    echo "fatal: tamed is already running at $root (pid $pid)!"
+    exit $EX_RUNNING
+  }
+
+  test -z "$pid" || {
+    echo "warning: clearing stale tamed (pid $pid)"
+  }
+}
+
+
+# Kill running tamed at path ROOT
+#
+# If no pidfile is found at ROOT, do nothing.  This sends a
+# signal only to the parent tamed process, _not_ individual
+# runners; the target tamed is expected to clean up itself.
+# Consequently, if a tamed terminated abnormally without
+# cleaning up, this will not solve that problem.
+kill-running()
+{
+  local -r root="${1?Missing root}"
+
+  local -r pid=$( cat "$root"/pid 2>/dev/null )
+
+  test -n "$pid" || return 0
+
+  echo "killing tamed at $root ($pid)..."
+  kill "$pid"
+}
+
+
+# Clean up child processes before exit
+#
+# This should be called before exit (perhaps by a trap).  Kills
+# the entire process group.
+#
+# Do not attach this to a SIGTERM trap or it will infinitely
+# recurse.
+cleanup()
+{
+  echo "killing remaining runners..."
+
+  rm -rf "$root"
+  kill 0
+}
+
+
+# Output usage information and exit
+usage()
+{
+  cat <<EOF
+Usage: $0 [--kill] [runpath]
+Start tamed and runners.  Do not fork into background process.
+
+The default value of RUNPATH is \`/run/user/$UID/tamed'.
+
+Only one runner is currently supported.
+
+Options:
+  --kill   kill a runing tamed at path RUNPATH
+  --help   show this message
+EOF
+
+  exit $EX_USAGE
+}
+
+
+# Run tamed
+main()
+{
+  local kill=
+  case "${1:-}" in
+    --kill) kill=1; shift;;
+    --help) usage;;
+  esac
+
+  root="${1:-/run/user/$UID/tamed}"
+
+  # kill if requested
+  test -z "$kill" || {
+    kill-running "$root"
+    exit
+  }
+
+  abort-if-running "$root"
+
+  # clean up background processes before we exit
+  trap exit TERM
+  trap cleanup EXIT
+
+  # start fresh
+  rm -rf "$root"; mkdir -p "$root"
+  echo $$ > "$root/pid"
+
+  # only a single runner for now
+  spawn-runner 0 "$root"
+
+  wait -n
+}
+
+main "$@"

configure.ac

@@ -40,6 +40,7 @@ AC_SUBST(REV, m4_argn(3, ver_split))
 AC_SUBST(SUFFIX, m4_argn(4, ver_split))
 
 AC_ARG_VAR([JAVA], [The Java executable])
+AC_ARG_VAR([JAVA_OPTS], [Java options])
 AC_CHECK_PROGS(JAVA, [java])
 
 AC_ARG_VAR([SAXON_CP], [Saxon class path])
@@ -52,7 +53,16 @@ AS_IF(test ! -d "$HOXSL",
   AC_MSG_ERROR([hoxsl path '$HOXSL' does not exist!]))
 AC_MSG_RESULT(found)
 
+# BC with RATER_CLASSPATH
+DSLC_CLASSPATH="${DSLC_CLASSPATH:-$RATER_CLASSPATH}"
+AC_SUBST(DSLC_CLASSPATH, [$DSLC_CLASSPATH])
+
+AC_SUBST([AUTOGENERATED],
+         ["THIS FILE IS AUTOGENERATED!  DO NOT MODIFY!  See *.in."])
+
 AC_CONFIG_FILES([Makefile doc/Makefile src/init.xsl VERSION])
+AC_CONFIG_FILES([bin/dslc],
+                [chmod +x bin/dslc])
 
 AC_OUTPUT
 

rater/README.md

@@ -0,0 +1,6 @@
+# Compatibility Directory
+
+This directory exists for compatibility with earlier build scripts and path
+assumptions before TAME was extracted into its own repository.  A full
+transition will remove this directory.
+

rater/c1map

@@ -0,0 +1 @@
+../src/current/c1map

rater/c1map.xsl

@@ -0,0 +1 @@
+../src/current/c1map.xsl

rater/calc.xsd

@@ -0,0 +1 @@
+../src/current/calc.xsd

rater/compile.xsl

@@ -0,0 +1 @@
+../src/current/compile.xsl

rater/compiler

@@ -0,0 +1 @@
+../src/current/compiler

rater/dot.xsl

@@ -0,0 +1 @@
+../src/current/dot.xsl

rater/include

@@ -0,0 +1 @@
+../src/current/include/

rater/link.xsl

@@ -0,0 +1 @@
+../src/current/link.xsl

rater/standalone.xsl

@@ -0,0 +1 @@
+../src/current/standalone.xsl

rater/summary.xsl

@@ -0,0 +1 @@
+../src/current/summary.xsl

rater/tame

@@ -0,0 +1 @@
+../

src/current/src/com/lovullo/dslc/DslCompiler.java

@@ -105,12 +105,17 @@ public class DslCompiler
                     new StreamResult( new File( dest ) ),
                     params
                 );
+
+                // TODO: more unique identifier
+                System.err.println( "DONE 0 " + dest );
             }
             catch ( Exception e )
             {
                 // delete the output file; it's garbage
                 destfile.delete();
 
+                System.err.println( "DONE 1 " + dest );
+
                 // be verbose and unprofessional.
                 throw e;
             }