bin/: Server/client build scripts

These scripts allow the TAME compiler stack to be invoked naturally, rather
than requiring the use of a Makefile today.  This will not only allow users
to more easily invoke the compiler, but will also allow us to invoke TAME
naturally from Makefile and remove the klugery that has existed for so
long.

This users a server/client architecture in order to mitigate the startup
cost of the JVM.  More documentation will follow.

Note that there are a bunch of symlinks in rater/---this is a transition
step to allow the build to continue working as it did before, which relies
on a directory structure that exists outside of this repository.  This will
be cleaned up in the future.

* .gitignore (bin/dslc): Add ignore for generated file.
* bin/dslc.in: New script to encapsulate Java invocation.
* bin/tame: New script (client).
* bin/tamed: New script (server).
* configure.ac (JAVA_OPTS, DSLC_CLASSPATH, AUTOGENERATED): New variables for
  dslc.in.  Output bin/dslc.
* rater/README.md: Note that this symlink mess is temporary.
* rater/c1map: New symlink for dslc assumptions.
* rater/c1map.xsl: Likewise.
* rater/calc.xsd: Likewise.
* rater/compile.xsl: Likewise.
* rater/compiler: Likewise.
* rater/dot.xsl: Likewise.
* rater/include: Likewise.
* rater/link.xsl: Likewise.
* rater/standalone.xsl: Likewise.
* rater/summary.xsl: Likewise.
* rater/tame: Likewise (warning: circular symlink).
* src/current/src/com/lovullo/dslc/DslCompiler.java (_DslCompiler)[compile]:
  Output `DONE' lines.
master
Mike Gerwitz 2018-10-08 23:14:19 -04:00
parent 4ad0c5d1be
commit cf57857ce5
19 changed files with 2485 additions and 0 deletions

1
.gitignore vendored
View File

@ -7,6 +7,7 @@
*.info *.info
# autotools- and configure-generated # autotools- and configure-generated
/bin/dslc
/Makefile.in /Makefile.in
/Makefile /Makefile
/aclocal.m4 /aclocal.m4

28
bin/dslc.in 100644
View File

@ -0,0 +1,28 @@
#!/bin/bash
# Listen for TAME commands (compilers, linker, etc)
#
# Copyright (C) 2018 R-T Specialty, LLC.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# @AUTOGENERATED@
##
declare -r mypath=$( dirname "$( readlink -f "$0" )" )
declare -r dslc_jar="$mypath/../src/current/src/dslc.jar"
CLASSPATH="$CLASSPATH:@DSLC_CLASSPATH@:$dslc_jar" \
"@JAVA@" @JAVA_OPTS@ \
com.lovullo.dslc.DslCompiler

220
bin/tame 100755
View File

@ -0,0 +1,220 @@
#!/bin/bash
# Client for TAME daemon (tamed)
#
# Copyright (C) 2018 R-T Specialty, LLC.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
##
set -euo pipefail
declare -r mypath=$( dirname "$( readlink -f "$0" )" )
declare -ri EX_NOTAMED=1 # tried to start tamed but failed
declare -ri EX_USAGE=64 # incorrect usage; sysexits.h
# Send a single command to a runner and observe the result
#
# stdin will be directed to the runner. stdout of the runner will be
# echoed until a line beginning with "DONE" is found, after which this
# procedure will return with the exit code indicated by the runner.
command-runner()
{
local -ri id="${1?Missing id}"
local -r root="${2?Missing root run path}"
shift 2
local -r base="$root/$id"
local -ri pid=$( cat "$base/pid" )
# TODO flock
verify-runner "$base" "$pid"
# all remaining arguments are passed to the runner
echo "$@" > "$base/0"
# output lines from runner until we reach a line stating "DONE"
while read line; do
# don't parse words in the initial read because we may be
# dealing with a lot of lines
if [ "${line:0:5}" == "DONE " ]; then
read _ code _ <<< "$line"
return "$code"
fi
echo "$line"
done < "$base/1"
}
# Verify that a runner is available
#
# If the runner is offline or not owned by $UID, then exit with
# a non-zero status.
verify-runner()
{
local -r base="${1?Missing base}"
local -ri pid="${2?Missing pid}"
ps "$pid" &>/dev/null || {
echo "error: runner $id ($pid) is offline!" >&2
exit "$EX_NOTAMED"
}
test -O "$base/0" || {
echo "error: runner $id ($pid) is not owned by $USER!" >&2
exit "$EX_NOTAMED"
}
}
# Wait somewhat impatiently for tamed
#
# Assumes that tamed's runner 0 is running once the pidfile becomes
# available. Polls for a maximum of six seconds before giving up
# and exiting with a non-zero status.
wait-for-tamed()
{
local -r base="${1?Missing base}"
# we could use inotify, but that is not installed by default
# on Debian systems, so let's just poll rather than introduce
# another dependency (give up after 6 seconds)
local -i i=12
while test $((i--)); do
test ! -f "$base/0/pid" || return 0
sleep 0.5
done
# still not available
echo 'error: tamed still unavailable; giving up' >&2
exit "$EX_NOTAMED"
}
# Start tamed if it is not already running
#
# If tamed is already running, nothing will happen; otherwise, start
# tamed and wait impatiently for the runner to become available.
#
# Even if tamed is started, wait for runner 0 to become available;
# this ensures that tamed is initialized even if this script is run
# after tamed is started but before it has fully come online (e.g
# parallel make).
start-tamed()
{
local -r root="${1?Missing root}"
local -ri pid=$( cat "$root/pid" 2>/dev/null )
ps "$pid" &>/dev/null || {
echo "starting tamed at $root..."
# tell tamed to clean up so that we eliminate race conditions
# with wait-for-tamed (this will also kill any stray processes
# that a previous tamed may have spawned but didn't get the
# chance to clean up)
kill-tamed "$root" || true
# start tamed and allow it to persist for future commands
"$mypath/tamed" "$root" & disown
}
# wait for tamed even if it was already started (just in
# case this script was executed right after tamed started
# but before it is done initializing)
wait-for-tamed "$root"
}
# Kill tamed
#
# Ask tamed to kill itself.
kill-tamed()
{
local -r root="${1?Missing root}"
"$mypath/tamed" --kill "$root"
}
# Filter dslc output to essential information
#
# The original output of dslc is quite noisy; this filters it down
# to only errors and warnings.
#
# Eventually, dslc out to be modified to handle filtering its own
# output rather than wasting cycles doing this filtering.
saneout()
{
awk ' \
/^~~~~\[begin /,/^~~~~\[end / { next } \
/^rm / { next } \
/^Exception|^\t+at / { \
if ( /^E/ ) { \
print; \
print "Stack trace written to .runlog"; \
} \
next; \
} \
/([Ww]arning|[Nn]otice)[: ]/ { printf "\033[0;33m"; w++; out=1; } \
/[Ff]atal:/ { printf "\033[0;31m"; out=1; } \
/!|[Ee]rror:/ { printf "\033[0;31m"; e++; out=1; } \
/internal:/ { printf "\033[0;35m"; out=1; } \
/internal error:/ { printf "\033[1m"; out=1; } \
/^[^[]/ || out { print; printf "\033[0;0m"; out=0; } \
'
}
# Output usage information and exit
usage()
{
cat <<EOF
Usage: $0 cmdline
Or: $0 --kill
Send command line CMDLINE to a tamed runner. Start tamed if
not already running.
Options:
--kill kill tamed
--help show this message
EOF
exit $EX_USAGE
}
# Run tame
main()
{
local -r root=/run/user/$UID/tamed
test $# -gt 0 || usage
case "${1:-}" in
--kill) kill-tamed "$root"; exit;;
--help) usage;;
esac
start-tamed "$root"
# for now we only support a single runner
command-runner 0 "$root" "$@" | saneout
}
main "$@"

217
bin/tamed 100755
View File

@ -0,0 +1,217 @@
#!/bin/bash
# Daemon for accepting TAME commands (compilers, linker, etc)
#
# Copyright (C) 2018 R-T Specialty, LLC.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
##
set -euo pipefail
declare -r mypath=$( dirname "$( readlink -f "$0" )" )
declare -ri EX_RUNNING=1
declare -ri EX_USAGE=64 # incorrect usage; sysexits.h
declare -ri EX_CANTCREAT=73 # cannot create file; sysexits.h
# set by `main', global for `cleanup'
declare root=
# Create FIFOs for runner
#
# The FIFOs are intended to be attached to stderr and stdout
# of the runner and will be created relative to the given
# root path ROOT.
#
# If a FIFO cannot be created, exit with EX_CANTCREAT.
mkfifos()
{
local -r root="${1?Missing root path}"
mkdir -p "$root"
# note that there's no stderr; see `add-runner'
for n in 0 1; do
rm -f "$root-$n"
mkfifo -m 0600 "$root/$n" || {
echo "fatal: failed to create FIFO at $in"
exit $EX_CANTCREAT
}
done
}
# Spawn a runner
#
# A new runner is created by spawning dslc and attaching
# new FIFOs under the given id ID relative to the given
# run path ROOT. The PID of the runner will be stored
# alongside the FIFOs in a pidfile `pid'.
spawn-runner()
{
local -ri id="${1?Missing id}"
local -r root="${2?Missing root run path}"
local -r base="$root/$id"
mkfifos "$base"
# TODO: should we separate back out std{out,err}?
# XXX: why does dslc quit (with a 0 exit code) occsionally? stdin?
while true; do
"$mypath/dslc" < <( persistent-cat "$base/0" ) \
>"$base/1" \
2>&1
echo "warning: runner $id exited with code $?; restarting"
done &
echo "$!" > "$base/pid"
echo "runner $id ($!): $base"
}
# Persistently read commands from FIFO IN
#
# This will continue to read from the FIFO as long as it is
# readable. This is necessary since SIGPIPE gets sent to
# processes reading/writing from/to the FIFO whenever a
# process detaches from it.
persistent-cat()
{
local -r in="${1?Missing input path}"
while test -r "$in"; do
read -r < "$in" || return
echo "$REPLY"
done
}
# Exit if tamed is already running at path ROOT
#
# If tamed is already running at ROOT, exit with status
# EX_RUNNING; otherwise, do nothing except output a warning
# if a stale pid file exists.
abort-if-running()
{
local -r root="${1?Missing root rundir}"
local -ri pid=$( cat "$root/pid" 2>/dev/null )
test "$pid" -gt 0 || return 0
! ps "$pid" &>/dev/null || {
echo "fatal: tamed is already running at $root (pid $pid)!"
exit $EX_RUNNING
}
test -z "$pid" || {
echo "warning: clearing stale tamed (pid $pid)"
}
}
# Kill running tamed at path ROOT
#
# If no pidfile is found at ROOT, do nothing. This sends a
# signal only to the parent tamed process, _not_ individual
# runners; the target tamed is expected to clean up itself.
# Consequently, if a tamed terminated abnormally without
# cleaning up, this will not solve that problem.
kill-running()
{
local -r root="${1?Missing root}"
local -r pid=$( cat "$root"/pid 2>/dev/null )
test -n "$pid" || return 0
echo "killing tamed at $root ($pid)..."
kill "$pid"
}
# Clean up child processes before exit
#
# This should be called before exit (perhaps by a trap). Kills
# the entire process group.
#
# Do not attach this to a SIGTERM trap or it will infinitely
# recurse.
cleanup()
{
echo "killing remaining runners..."
rm -rf "$root"
kill 0
}
# Output usage information and exit
usage()
{
cat <<EOF
Usage: $0 [--kill] [runpath]
Start tamed and runners. Do not fork into background process.
The default value of RUNPATH is \`/run/user/$UID/tamed'.
Only one runner is currently supported.
Options:
--kill kill a runing tamed at path RUNPATH
--help show this message
EOF
exit $EX_USAGE
}
# Run tamed
main()
{
local kill=
case "${1:-}" in
--kill) kill=1; shift;;
--help) usage;;
esac
root="${1:-/run/user/$UID/tamed}"
# kill if requested
test -z "$kill" || {
kill-running "$root"
exit
}
abort-if-running "$root"
# clean up background processes before we exit
trap exit TERM
trap cleanup EXIT
# start fresh
rm -rf "$root"; mkdir -p "$root"
echo $$ > "$root/pid"
# only a single runner for now
spawn-runner 0 "$root"
wait -n
}
main "$@"

View File

@ -40,6 +40,7 @@ AC_SUBST(REV, m4_argn(3, ver_split))
AC_SUBST(SUFFIX, m4_argn(4, ver_split)) AC_SUBST(SUFFIX, m4_argn(4, ver_split))
AC_ARG_VAR([JAVA], [The Java executable]) AC_ARG_VAR([JAVA], [The Java executable])
AC_ARG_VAR([JAVA_OPTS], [Java options])
AC_CHECK_PROGS(JAVA, [java]) AC_CHECK_PROGS(JAVA, [java])
AC_ARG_VAR([SAXON_CP], [Saxon class path]) AC_ARG_VAR([SAXON_CP], [Saxon class path])
@ -52,7 +53,16 @@ AS_IF(test ! -d "$HOXSL",
AC_MSG_ERROR([hoxsl path '$HOXSL' does not exist!])) AC_MSG_ERROR([hoxsl path '$HOXSL' does not exist!]))
AC_MSG_RESULT(found) AC_MSG_RESULT(found)
# BC with RATER_CLASSPATH
DSLC_CLASSPATH="${DSLC_CLASSPATH:-$RATER_CLASSPATH}"
AC_SUBST(DSLC_CLASSPATH, [$DSLC_CLASSPATH])
AC_SUBST([AUTOGENERATED],
["THIS FILE IS AUTOGENERATED! DO NOT MODIFY! See *.in."])
AC_CONFIG_FILES([Makefile doc/Makefile src/init.xsl VERSION]) AC_CONFIG_FILES([Makefile doc/Makefile src/init.xsl VERSION])
AC_CONFIG_FILES([bin/dslc],
[chmod +x bin/dslc])
AC_OUTPUT AC_OUTPUT

6
rater/README.md 100644
View File

@ -0,0 +1,6 @@
# Compatibility Directory
This directory exists for compatibility with earlier build scripts and path
assumptions before TAME was extracted into its own repository. A full
transition will remove this directory.

1
rater/c1map 120000
View File

@ -0,0 +1 @@
../src/current/c1map

1
rater/c1map.xsl 120000
View File

@ -0,0 +1 @@
../src/current/c1map.xsl

1
rater/calc.xsd 120000
View File

@ -0,0 +1 @@
../src/current/calc.xsd

View File

@ -0,0 +1 @@
../src/current/compile.xsl

1
rater/compiler 120000
View File

@ -0,0 +1 @@
../src/current/compiler

1
rater/dot.xsl 120000
View File

@ -0,0 +1 @@
../src/current/dot.xsl

1
rater/include 120000
View File

@ -0,0 +1 @@
../src/current/include/

1
rater/link.xsl 120000
View File

@ -0,0 +1 @@
../src/current/link.xsl

1987
rater/rater.xsd 100644

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1 @@
../src/current/standalone.xsl

View File

@ -0,0 +1 @@
../src/current/summary.xsl

1
rater/tame 120000
View File

@ -0,0 +1 @@
../

View File

@ -105,12 +105,17 @@ public class DslCompiler
new StreamResult( new File( dest ) ), new StreamResult( new File( dest ) ),
params params
); );
// TODO: more unique identifier
System.err.println( "DONE 0 " + dest );
} }
catch ( Exception e ) catch ( Exception e )
{ {
// delete the output file; it's garbage // delete the output file; it's garbage
destfile.delete(); destfile.delete();
System.err.println( "DONE 1 " + dest );
// be verbose and unprofessional. // be verbose and unprofessional.
throw e; throw e;
} }