tame/bin/tamed

#!/bin/bash
# Daemon for accepting TAME commands (compilers, linker, etc)
#
#   Copyright (C) 2014-2023 Ryan Specialty, LLC.
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
##

set -euo pipefail

declare mypath; mypath=$( dirname "$( readlink -f "$0" )" )
readonly mypath

declare -ri EX_RUNNING=1
declare -ri EX_NOTRUNNING=2  # tamed is not running
declare -ri EX_RUNTAB_LOCK=3 # failed to acquire aggregate runtab lock
declare -ri EX_RUNTAB_OUT=4  # failed to write to aggregate runtab
declare -ri EX_USAGE=64      # incorrect usage; sysexits.h
declare -ri EX_CANTCREAT=73  # cannot create file; sysexits.h

# number of seconds of output silence before runners are considered unused
# and are subject to termination (see stall-monitor)
declare -ri TAMED_STALL_SECONDS="${TAMED_STALL_SECONDS:-1}"

# id of process that indirectly spawned tamed (default $PPID)
declare -ri TAMED_SPAWNER_PID="${TAMED_SPAWNER_PID:-$PPID}"

# options to pass to JVM via dslc
declare -r TAMED_JAVA_OPTS="${TAMED_JAVA_OPTS:-}"
export JAVA_OPTS="$TAMED_JAVA_OPTS"

# set by `main', global for `cleanup' and `runner-report-all'
declare root=

# non-empty if in TUI (terminal UI) mode (use `in-tui-mode')
declare -r TAMED_TUI="${TAMED_TUI:-}"
declare tui_mode=

# file into which aggregate runner report will be placed (none if empty)
declare -r TAMED_RUNTAB_OUT="${TAMED_RUNTAB_OUT:-}"


# Create FIFOs for runner
#
# The FIFOs are intended to be attached to stderr and stdout
# of the runner and will be created relative to the given
# root path ROOT.
#
# If a FIFO cannot be created, exit with EX_CANTCREAT.
mkfifos()
{
  local -r root="${1?Missing root path}"

  mkdir -p "$root"

  # note that there's no stderr; see `add-runner'
  for n in 0 1; do
    rm -f "$root-$n"

    mkfifo -m 0600 "$root/$n" || {
      log "fatal: failed to create FIFO at $root/n" >&2
      exit $EX_CANTCREAT
    }
  done

  # keep FIFOs open so we don't get EOF from writers
  tail -f >"$root/0" &
}


# Output a line, clearing the remainder of the line if in TUI mode
log()
{
  if in-tui-mode; then
    echo -en "\e[2K"
  fi

  echo "$@"
}


# Spawn a new runner using the next available runner id
#
# See `spawn-runner' for more information.
spawn-next-runner()
{
  local -r root="${1?Missing root path}"

  # get the next available id
  local -ri id=$( < "$root/maxid" )

  spawn-runner "$(( id + 1 ))" "$root"
}


# Spawn a runner
#
# A new runner is created by spawning dslc and attaching
# new FIFOs under the given id ID relative to the given
# run path ROOT.  The PID of the runner will be stored
# alongside the FIFOs in a pidfile `pid'.
spawn-runner()
{
  local -ri id="${1?Missing id}"
  local -r root="${2?Missing root run path}"

  local -r base="$root/$id"

  mkfifos "$base"

  # flag as available (the client will manipulate these)
  echo 0 > "$base/busy"

  # runtab is used for reporting, which we will optionally aggregate
  > "$base/runtab"
  monitor-runner-runtab "$root" "$base/runtab" &

  # monitor runner usage and kill when inactive
  stall-monitor "$base" &


  # loop to restart runner in case of crash
  (
    declare -i job=0
    trap force-job-reload HUP

    force-job-reload() {
      kill -9 $job
    }

    while true; do
      # if this runner is busy, then it must have terminated while
      # processing (otherwise the client `tame` would have marked it as
      # available); let's act on its behalf so that the client sees that we
      # failed (which we'll represent with error code 2).
      declare -i busy=$(< "$base/busy")
      if runner-is-busy "$base" && ! runner-is-reloading; then
        inject-runner-unexpected-exit "$base" "$id"
      fi

      # store the time that the runner was started so that we can later
      # determine if it should be restarted to forcefully reclaim memory
      date +%s > "$base/created-ts"

      "$mypath/dslc" < "$base/0" &> "$base/1" & job=$!
      runner-done-reloading

      declare -i status=0
      wait "$job" 2>/dev/null || status=$?

      # 129 = signal (128) + HUP (1), which is an explicit reload request
      # that we need not report
      if [ "$status" -ne 129 ]; then
        echo "warning: runner $id exited with code $status (pid $job); restarting" >&2
      fi
    done
  ) &

  echo "$!" > "$base/pid"

  # we assume that this is the new largest runner id
  echo "$id" > "$root/maxid"

  log "runner $id ($!): $base"
}


# Whether the runner at the provided base is busy
runner-is-busy() {
  local -r base="$root/$id"

  declare -i busy=$(< "$base/busy")
  test "$busy" -eq 1
}


# Whether the runner at the provided base is flagged as having a reload
# request
runner-is-reloading() {
  local -r base="$root/$id"

  test -f "$base/reloading"
}


# Clear the runner's `reloading` flag, if any.
#
# This flag is set by the `tame` client before requesting a
# reload.  Clearing this flag allows the client to observe that reloading
# the runner is complete and requests may be issued.
runner-done-reloading() {
  local -r base="$root/$id"

  rm -f "$base/reloading"
}


# Inject an exit code into the runner's output stream indicating an
# unexpected exit
#
# The string `DONE n` is normally output at the end of a runner's
# compilation (via `dslc`), where `n` is the exit code.  But if the runner
# terminates before compilation completes (e.g. is OOM-killed), then it will
# never have the chance to do so, leaving the client waiting for a
# response.  If the client is not checking for stalls (due to
# configuration), it may hang indefinitely.
#
# This function will inject a message into the output stream of the runner
# as if `dslc` itself replied so that the `tame` client can observe a
# failure and react accordingly.  This uses the `tame` `EX_UNEXPECTED` exit
# code.
#
# This also outputs a warning to stderr.
inject-runner-unexpected-exit() {
  local -r base="${1?Missing base}"
  local -ri id="${2?Missing id}"

  echo "warning: runner $id exited unexpectedly" >&2

  # TODO: Worth a shared file with `tame`?
  local -ri EX_UNEXPECTED=7
  echo "DONE $EX_UNEXPECTED" > "$base/1"
}


# Monitor the given runner runtab and append to the aggregate runtab
#
# The aggregate runtab is append-only and has a row-level lock to support
# concurrent writes without having to rely on kernel buffering.
monitor-runner-runtab()
{
  local -r root="${1?Missing root run path}"
  local -r runtab="${2?Missing runtab path}"

  # no use in aggregating if it was not requested
  test -n "$TAMED_RUNTAB_OUT" || return 0

  while ! spawner-dead; do
    # this is a shared file, and while buffering _should_ be sufficient, we
    # may as well avoid potential headaches entirely by locking during the
    # operation
    tail -f "$runtab" | while read -r row; do
        # we want to lock _per row write_, since output will be interleaved
        # between all the runners
        (
          local -ri timeout=3

          flock -w $timeout 7 || {
              echo "error: failed to acquire lock on aggregate runtab" >&2
              exit $EX_RUNTAB_LOCK
          }

          echo "$row" >&7
        ) 7>> "$TAMED_RUNTAB_OUT"
    done
  done
}


# Check that we can write to the provided runtab, and clear it
runtab-check-and-clear()
{
  test -n "$TAMED_RUNTAB_OUT" || return 0

  # clear the runtab, and see if we can write to it
  >"$TAMED_RUNTAB_OUT" || {
    echo "error: unable to write to '$TAMED_RUNTAB_OUT' (TAMED_RUNTAB_OUT)"
    exit $EX_RUNTAB_OUT
  }

  echo "tamed: aggregating runner runtabs into '$TAMED_RUNTAB_OUT'"
}


# Kill runner at BASE when it becomes inactive for TAMED_STALL_SECONDS
# seconds
#
# This monitors the modification time on the stdout FIFO.  stdin does not
# need to be monitored since dslc immediately echoes back commands it
# receives.
#
# dslc is pretty chatty at the time of writing this, so TAMED_STALL_SECONDS
# can easily be <=30s even for large packages.  This may need to change in
# the future if it becomes too much less chatty.  Increase that environment
# variable if runners stall unexpectedly in the middle of builds.
#
# If the id of the spawning process has been provided then we will never
# consider ourselves to be stalled if that process is still running.  This
# prevents, for example, tamed from killing itself while a parent make
# process is still running.
stall-monitor()
{
  local -r base="${1?Missing base}"

  # monitor output FIFO modification time
  while true; do
    local -i since last
    since=$( date +%s )
    sleep "$TAMED_STALL_SECONDS"
    last=$( stat -c%Y "$base/1" )

    # keep waiting if there has been activity since $since
    test "$last" -le "$since" || continue

    spawner-dead || continue

    # no activity; kill
    local -r pid=$( cat "$base/pid" )
    kill "$pid"
    wait "$pid" 2>/dev/null

    # this stall subprocess is no longer needed
    break
  done
}


# Check to see if the spawning process has died
#
# If no spawning process was provided, then this always returns a zero
# status.  Otherwise, it returns whether the given pid is _not_ running.
spawner-dead()
{
  test "$TAMED_SPAWNER_PID" -gt 0 || return 0

  ! ps "$TAMED_SPAWNER_PID" &>/dev/null
}


# Exit if tamed is already running at path ROOT
#
# If tamed is already running at ROOT, exit with status
# EX_RUNNING; otherwise, do nothing except output a warning
# if a stale pid file exists.
abort-if-running()
{
  local -r root="${1?Missing root rundir}"

  local -ri pid=$( cat "$root/pid" 2>/dev/null )

  test "$pid" -gt 0 || return 0

  ! ps "$pid" &>/dev/null || {
    log "fatal: tamed is already running at $root (pid $pid)!" >&2
    exit $EX_RUNNING
  }

  test -z "$pid" || {
    log "warning: clearing stale tamed (pid $pid)" >&2
  }
}


# Exit with EX_NOTRUNNING if tamed is not running at path ROOT
#
# ROOT must both exist and contain a `pid` file of a running process.
abort-if-not-running()
{
  local -r root="${1?Missing root rundir}"

  test -d "$root" || {
    log "tamed is not running at $root: path does not exist" >&2
    exit $EX_NOTRUNNING
  }

  local -ri pid=$( cat "$root/pid" 2>/dev/null )

  # this should not happen unless bash crashed
  ps "$pid" &>/dev/null || {
    log "tamed is not running at $root: process $pid has terminated" >&2
    exit $EX_NOTRUNNING
  }
}


# Kill running tamed at path ROOT
#
# If no pidfile is found at ROOT, do nothing.  This sends a
# signal only to the parent tamed process, _not_ individual
# runners; the target tamed is expected to clean up itself.
# Consequently, if a tamed terminated abnormally without
# cleaning up, this will not solve that problem.
#
# Note that this is also called by tame to clean up an old tamed
# before spawning a new one.
kill-running()
{
  local -r root="${1?Missing root}"

  test -d "$root" || return 0
  local -r pid=$( < "$root"/pid 2>/dev/null )

  test -n "$pid" || return 0

  log "killing tamed at $root ($pid)..."
  kill "$pid"
}


runner-report-all()
{
  local -r root="${1?Missing root}"

  abort-if-not-running "$root"
  for-each-runner "$root" runner-report
}


for-each-runner()
{
  local -r root="${1?Missing root}"
  local -r cmd="${2?Missing command}"
  shift 2

  local -ri maxid=$(cat "$root/maxid")

  echo "tamed is running at $root with $((maxid+1)) runner(s)"

  for runner in $(seq 0 "$maxid"); do
    echo
    "$cmd" "$root" "$@" "$runner"
  done
}


# Report on the status and current operation of each runner
#
# This report is generated by tamed rather than delegating to the runners
# themselves to avoid the complexity of mitigating output races.
runner-report()
{
  local -r root="${1?Missing root}"
  local -ri id="${2?Missing runner id}"

  local -r path="$root/$id"
  test -f "$path/cmdline" || return 0

  local cmdline=$(< "$path/cmdline" )
  local -a cmdstart cmdstart_fmt

  cmdstart=( $(< "$path/cmdstart" ) )
  cmdstart_fmt=$(date --date=@"${cmdstart[0]}" +%Y-%m-%dT%H:%M:%S)

  local -i now=$(date +%s)

  cat <<EOF
runner:  $id
command: $cmdline
start:   ${cmdstart[0]}.${cmdstart[1]} ($cmdstart_fmt)
elapsed: $((now - cmdstart)) seconds
EOF
}


elide-paths()
{
  local -r cols="${1?Missing columns}"
  local -r buffer="${2?Missing buffer}"

  # first, keep the first letter and last three of each dir, if doing so
  # would remove three or more characters; for example:
  #   "suppliers/foobarbaz/quux/quuux.xmlo" => "s…ers/f…baz/quux/quuux.xmlo"
  result=$(
    echo "$buffer" \
      | sed 's|\([a-zA-Z0-9_-]\)[a-zA-Z0-9_-]\{3,\}\([a-zA-Z9-9_-]\{3\}\)/|\1…\2/|g'
  )

  [ "${#result}" -gt $cols ] || {
    echo -n "$result"
    return
  }

  # more aggressive: remove all but the first letter if it would save at
  # least three characters, as in:
  #   "suppliers/foobarbaz/quux/quuux.xmlo" => "s…/f…/quux/quuux.xmlo"
  result=$(
    echo "$buffer" | sed 's|\([a-zA-Z0-9_-]\)[^ /]\{3,\}/|\1…/|g'
  )

  [ "${#result}" -gt $cols ] || {
    echo -n "$result"
    return
  }

  # even more aggressive: elide all but the filename, as in:
  #   "suppliers/foobarbaz/quux/quuux.xmlo" => "…/quuux.xmlo"
  result=$(
    echo "$buffer" | sed 's|[a-zA-Z0-9_-/]*/|…/|g'
  )

  [ "${#result}" -gt $cols ] || {
    echo -n "$result"
    return
  }

  # at this point, it's better to provide _some_ useful information for
  # _some_ runners, so just truncate the previous result (we probably have
  # too many runners for the current terminal width)
  echo -n "${result::$((cols-1))}…"
}


# Report of all runners' status on a single line
#
# Idle runners are not output for now, since that increases the likelihood
# that we will not output something when runners are done doing their jobs
# (including overwriting the PS1).
runner-report-line() {
  local -r root="${1?Missing root}"

  # buffer output so that our report does not get mixed with normal
  # runner output
  local buffer=$( runner-report-all "$root" | awk '
    /^command: idle/,/^$/ { next }      # skip idle
    /^command:/ { printf "[%s ", $NF }  # e.g. "[foo/bar.xmlo "
    /^elapsed:/ { printf "%ds] ", $2 }  # e.g. "2s] "
  ' )

  # ensure proper empty output without formatting if there is no line
  test -n "$buffer" || return 0

  # bash has checkwinsize, but that runs after every command; try to use
  # tput, defaulting to 80.  Note that we have to check this every time, in
  # case the terminal has been resized.
  local -ri cols=$(tput cols || echo 80)

  # rather than worrying about line wrapping, fit to one line
  if [[ "${#buffer}" -gt $cols ]]; then
    buffer=$(elide-paths $cols "$buffer")
  fi

  # output in bold, overwrite our line that may already be present here, and
  # place cursor at beginning of the line so any runner output will
  # overwrite
  echo -en "\e[1m$buffer\e[0m\r"
}


# Clean up child processes before exit
#
# This should be called before exit (perhaps by a trap).  Kills
# the entire process group.
#
# Do not attach this to a SIGTERM trap or it will infinitely
# recurse.
cleanup()
{
  rm -rf "$root"
  kill 0
}


# Output usage information and exit
usage()
{
  cat <<EOF
Usage: $0 [--kill] [runpath]
Start tamed and runners.  Do not fork into background process.

The default value of RUNPATH is \`/run/user/$UID/tamed'.

Only one runner is currently supported.  tamed exits once all
runners have terminated.  Runners will be killed once they are
inactive for at least TAMED_STALL_SECONDS (default 1), unless
the process identified by TAMED_SPAWNER_PID is still running.
For example, a build script may wish to set TAMED_SPAWNER_PID
to the process id of make itself.  It defaults to the actual
parent process id (PPID), so tamed will not kill itself if
run manually on a shell (unless the shell exits first).

TAMED_RUNTAB_OUT can specify a file in which to write job
start times (as seconds from the Unix epoch); durations
(in milliseconds); and commands from each of the runners.
The table is tab-delimited.  Here are some useful examples:

  # format nicely into columns and view in pager
  $ column runtab | less

  # sort by runtime descending (second column)
  $ sort -rnk2 runtab

  # take the runtime and command columns
  $ cut -2,3 runtab

  # convert milliseconds into minutes (!) and sort desc
  $ awk '{ \$2 = \$2 / 1000 / 60; print }' runtab | sort -nrk2

  # convert to CSV (assuming no quoting is needed)
  $ tr '\t' , < runtab > runtab.csv

Options:
  --help    show this message
  --kill    kill a runing tamed at path RUNPATH
  --report  display runner report (this is subject to change
              in later versions)

Environment Variables:
  TAMED_STALL_SECONDS   number of seconds of runner inactivity before
                          runner is automatically killed (default 1)
  TAMED_SPAWNER_PID     inhibit stalling while this process is running
                          (default PPID)
  TAMED_JAVA_OPTS       opts to pass to dslc, and in turn, the JVM
  TAMED_TUI             run in TUI mode (provide UI features like a
                          dynamic runner status line)
  TAMED_RUNTAB_OUT      file into which aggregate runner report will
                          be written (otherwise reports are only
                          available per-runner while tamed is running)
EOF

  exit $EX_USAGE
}


# Determine whether to enable TUI mode
#
# TUI (terminal UI) mode will augment the output with features that only
# make sense when running on a user's terminal, such as the runner status
# line.
tui-check()
{
  test "$TAMED_TUI" == 1 || return 0
  tui_mode=1
  log "tamed is running in TUI mode (TAMED_TUI=0 to disable)"
}


# Whether we're running in TUI mode
in-tui-mode()
{
  test -n "$tui_mode"
}


# If in TUI mode, continuously update the last line of output with runner
# status
#
# This is not an easy undertaking with how our build process currently
# works.  Make is responsible, currently, for echoing lines, and so we must
# frequently re-echo our status line in an attempt to redisplay the line
# after it is overwritten.
#
# Further, most output is unaware that the entire line needs to be
# overwritten; if output is not properly transformed in the Makefile, then
# portions of the status line may remain in the history, partly overwritten
# by build output.
#
# Another concern is that we do not want to keep outputting after the
# process is finished, which would overwrite the PS1.  To try to avoid this,
# we omit idle runner output and only clear the line _once_ when the status
# line is empty, in the hope that all runners will be idle for long enough
# before the build completes, make exists, exits, and the PS1 is output.
#
# If not in TUI mode, this does nothing.
tui-runner-status-line()
{
  in-tui-mode || return 0

  local cache= cleared=

  while ! spawner-dead; do
    # this will fail if no runners have been created yet, so just ignore
    # it; if we fail to output the status line, the build will still work
    cache=$(runner-report-line "$root" 2>/dev/null)

    # if the line is empty, clear the output _once_ (to get rid of
    # whatever was there before), but do not do it again, otherwise we
    # risk overwriting lines post-build (like the PS1 or late-stage make
    # targets).
    if [ -z "$cache" -a -z "$cleared" ]; then
      log -n ""
      cleared=1

      sleep 1
      continue
    fi

    cleared=

    # output the cache frequently to try to overcome build output
    for i in {0..9}; do
      log -n "$cache"
      sleep 0.1
    done
  done
}


# Run tamed
main()
{
  local kill= report=
  case "${1:-}" in
    --kill) kill=1; shift;;
    --report) report=1; shift;;
    --help) usage;;
  esac

  root="${1:-/run/user/$UID/tamed}"

  # report requested
  test -z "$report" || {
    runner-report-all "$root"
    exit
  }

  # kill if requested
  test -z "$kill" || {
    kill-running "$root"
    exit
  }

  abort-if-running "$root"
  tui-check
  runtab-check-and-clear

  # clean up background processes before we exit
  trap exit TERM
  trap cleanup EXIT

  # start fresh
  rm -rf "$root"; mkdir -p "$root"
  local -i pid=$$
  echo $pid > "$root/pid"

  # start with a single runner; we'll spawn more if requested
  spawn-runner 0 "$root"
  trap "spawn-next-runner '$root'" USR1

  # status line reporting on runners for TUI mode
  tui-runner-status-line &

  # wait for runners to complete or for a signal to be received by this
  # process that terminates `wait'
  while true; do
    wait -n || {
      status=$?

      # ignore USR{1,2}
      if [ $status -ne 138 -a $status -ne 140 ]; then
        exit $status
      fi
    }
  done
}

main "$@"