From 3d1f9d22a7622f5e4517d2ffd57d5d18f017d8c0 Mon Sep 17 00:00:00 2001 From: Mike Gerwitz Date: Fri, 9 May 2014 00:46:18 -0400 Subject: [PATCH] Initial concept and working tests for shspec My original prototype was more feature-rich than this, but this formalizes it and provides self-tests. It is indeed odd seeing shell code that does not look like shell. --- bin/shspec | 33 ++++++ src/cli | 24 +++++ src/common | 262 ++++++++++++++++++++++++++++++++++++++++++++++++ src/expect-core | 28 ++++++ src/run-spec | 29 ++++++ src/runner | 26 +++++ src/spec | 224 +++++++++++++++++++++++++++++++++++++++++ src/specstack | 164 ++++++++++++++++++++++++++++++ test/test-spec | 198 ++++++++++++++++++++++++++++++++++++ 9 files changed, 988 insertions(+) create mode 100755 bin/shspec create mode 100755 src/cli create mode 100644 src/common create mode 100644 src/expect-core create mode 100755 src/run-spec create mode 100755 src/runner create mode 100644 src/spec create mode 100644 src/specstack create mode 100644 test/test-spec diff --git a/bin/shspec b/bin/shspec new file mode 100755 index 0000000..0e032c3 --- /dev/null +++ b/bin/shspec @@ -0,0 +1,33 @@ +#!/bin/bash +# shspec BDD CLI +# +# Copyright (C) 2014 Mike Gewitz +# +# This file is part of shspec. +# +# shspec is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +## + +# all of our arguments (at the moment) are paths; if they're relative, then +# that'll be a problem, since we're changing directories, so convert them to +# absolute paths +declare -a paths=() +for arg in "$@"; do + paths+=( "$( readlink -f "$arg" )" ) +done + +# place ourselves in a known state and kick off the CLI +cd "$(dirname $0)"/../src \ + && exec ./cli "${paths[@]}" + diff --git a/src/cli b/src/cli new file mode 100755 index 0000000..0acda37 --- /dev/null +++ b/src/cli @@ -0,0 +1,24 @@ +#!/bin/bash +# shspec CLI +# +# Copyright (C) 2014 Mike Gerwitz +# +# This file is part of shspec. +# +# shspec is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +## + +# we do not have much of a CLI atm; delegate to runner +exec ./runner "$@" + diff --git a/src/common b/src/common new file mode 100644 index 0000000..f721136 --- /dev/null +++ b/src/common @@ -0,0 +1,262 @@ +#!/bin/bash +# Common test case file +# +# Copyright (C) 2014 LoVullo Associates, Inc. +# +# This file is part of trello-sh. +# +# trello-sh is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +## + +declare -a __desc_stack=() +declare -i __desc_stackp=0 + +# current test csae +__desc_case= + +# stderr file +__desc_errpath="$(mktemp)" + +# most recent expect result and its exit code +__desc_result= +__desc_rexit=0 + +# most recent caller for assertions +__desc_caller= + + +__desc-push() +{ + __desc_stack[$__desc_stackp]="$*" + ((__desc_stackp++)) +} + +__desc-pop() +{ + [ "$__desc_stackp" -gt 0 ] || return 1 + + # notice that we unset before we decrease the pointer; this allows for a + # single level of un-popping + unset __desc_stack[$__desc_stackp] + ((__desc_stackp--)) +} + +__desc-unpop() +{ + ((__desc_stackp++)) +} + +__desc-head() +{ + local headi=$((__desc_stackp - 1)) + echo "${__desc_stack[ $headi ]}" +} + +__desc-headtype() +{ + local parts=( $(__desc-head) ) + echo "${parts[0]}" +} + +__desc-assert-within() +{ + local in="$1" + local chk="$2" + local line="$3" + local file="$4" + local phrase="${5:-be contained within}" + local head="$(__desc-headtype)" + local printin="$(__desc-type-clean "$in")" + + [ "$head" == "$in" ] \ + || bail "\`$chk' must $phrase \`$printin'; found \`$head' at $file:$line" +} + +__desc-type-clean() +{ + # a colon prefix is semantic; strip for display + echo "${1#:}" +} + +__desc-assert-follow() +{ + __desc-assert-within "$@" follow +} + + +begin-case() +{ + __desc_case="$1" +} + +tests() +{ + source ../src/"$1" +} + +describe() +{ + __desc-push "describe $(caller) $@" +} + +it() +{ + __desc-push "it $(caller) $@" +} + +end() +{ + local head="$(__desc-headtype)" + local cleanhead="$(__desc-type-clean "$head")" + + # some statements are implicitly terminated; explicitly doing so is + # indicitive of a syntax issue + [ "${head:0:1}" != : ] \ + || bail "unexpected \`end': still processing \`$cleanhead'" $(caller) + + __desc-pop || bail "unmatched \`end'" +} + +end-case() +{ + local tcase="$__desc_case" + __desc_case= + + # nothing to do if our stack is clean + [ "$__desc_stackp" -gt 0 ] || return 0 + + # output an error message for each item in the stack + while ((__desc_stackp--)); do + read type line file __ <<< "$(__desc-head)" + echo "error: $tcase: unterminated \`$type' at $file:$line" + done + + exit 1 +} + +bail() +{ + local msg="$1" + local line="$2" + local file="$3" + + echo -n "error: $1" >&2 + [ $# -gt 1 ] && echo -n " at $file:$line" >&2 + + echo + exit 1 +} + + +expect() +{ + __desc-assert-within it expect $(caller) + __desc_result="$($@ 2>"$__desc_errpath")" + __desc_rexit=$? + __desc-push ":expect $(caller) $@" +} + +to() +{ + __desc_caller=${__desc_caller:-$(caller)} + + [ $# -gt 0 ] || bail "missing assertion string for \`to'" $__desc_caller + + local type="$1" + shift + + __desc-assert-follow :expect to $(caller) + __desc-pop + + local assert="_desca-$type" + [ "$( type -t "$assert" )" == function ] \ + || bail "unknown \`to' assertion: \`$type'" $__desc_caller + + # invoke partially-applied assertion with remaining arguments + $assert "$@" || fail + __desc_caller= +} + +and() +{ + __desc_caller="$(caller)" + __desc-unpop + "$@" +} + + +# +# Assertions +# +_desca-succeed() +{ + __desca-exit -eq 0 +} + +_desca-fail() +{ + __desca-exit -ne 0 +} + +_desca-exit() +{ + __desca--argsep with 2 __desca-exit -eq +} + + +__desca-exit() +{ + __desca--argn 2 "$#" + [ "$__desc_rexit" "$@" ] +} + +_desca-output() +{ + local cmp="$1" + + diff="$( diff <( echo "$__desc_result" ) <( echo "$cmp" ) )" \ + && return 0 + + echo "$diff" >&2 + return 1 +} + +__desca--argsep() +{ + local delim="$1" + local cmdlen="$2" + local partial= + shift 2 + + # bash doesn't support multi-dimensional arrays, so here's to awkward command + # reconstruction with argument counts + while ((cmdlen--)); do + partial="$partial $1" + shift + done + + local next="$1" + shift + + [ "$next" == "$delim" ] \ + || bail "expected \`$delim' but found \`$next'" $__desc_caller + + $partial "$@" +} + +__desca--argn() +{ + [ $2 -eq "$1" ] || bail "invalid number of arguments" $__desc_caller +} + diff --git a/src/expect-core b/src/expect-core new file mode 100644 index 0000000..c52af18 --- /dev/null +++ b/src/expect-core @@ -0,0 +1,28 @@ +#!/bin/bash +# Core expectations +# +# Copyright (C) 2014 Mike Gewitz +# +# This file is part of shspec. +# +# shspec is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +## + +[ -z $__INC_EXPECT_CORE ] || return +__INC_EXPECT_CORE=1 + + +_expect--succeed() { test "$(_spec-last-exit)" -eq 0; } +_expect--fail() { test "$(_spec-last-exit)" -ne 0; } + diff --git a/src/run-spec b/src/run-spec new file mode 100755 index 0000000..87a9c70 --- /dev/null +++ b/src/run-spec @@ -0,0 +1,29 @@ +#!/bin/bash +# shspec CLI +# +# Copyright (C) 2014 Mike Gerwitz +# +# This file is part of shspec. +# +# shspec is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +## + +source spec + +declare -r tcase="${1?Missing test case path}" + +_begin-spec + source "$tcase" || exit $? +_end-spec + diff --git a/src/runner b/src/runner new file mode 100755 index 0000000..07c76e5 --- /dev/null +++ b/src/runner @@ -0,0 +1,26 @@ +#!/bin/bash +# shspec CLI +# +# Copyright (C) 2014 Mike Gerwitz +# +# This file is part of shspec. +# +# shspec is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +## + +# each test is run in a separate process to ensure environment isolation +for tcase in "$@"; do + env -i ./run-spec "$tcase" +done + diff --git a/src/spec b/src/spec new file mode 100644 index 0000000..c78ba21 --- /dev/null +++ b/src/spec @@ -0,0 +1,224 @@ +#!/bin/bash +# Specification language +# +# Copyright (C) 2014 Mike Gewitz +# +# This file is part of shspec. +# +# shspec is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Upon examining this code, you will likely notice that it is not +# re-entrant. This is okay, because the intended use is to invoke a new +# process *per specification*, not to run multiple specifications +# concurrently within the same process. +# +# You've been warned. +## + +[ -z $__INC_SPEC ] || return +__INC_SPEC=1 + +source specstack +source expect-core + + +## +# Attempts to make tempoary path in /dev/shm, falling back to default +mktemp-shm() +{ + local -r shm=/dev/shm + [ -d $shm -a -r $shm ] && mktemp -p$shm || mktemp +} + + +# stderr file +declare -r __spec_errpath="$(mktemp-shm)" + +# most recent expect result and its exit code +declare __spec_result= +declare -i __spec_rexit=0 + +# most recent caller for expectations +declare __spec_caller= + + +## +# Begin a new specification definition +# +# A specification is any shell script using the specification commands +# defined herein. +_begin-spec() +{ + : placeholder +} + + +## +# Mark the end of a specification +# +# If the specification was improperly nested, this will output a list of +# nesting errors and return a non-zero error. Otherwise, nothing is done. +_end-spec() +{ + # if the stack is empty then everything is in order + _sstack-empty && return 0 + + # otherwise, output an error message for each item in the stack + until _sstack-empty; do + _sstack-read type line file _ < <(_sstack-head) + _sstack-pop + echo "error: unterminated \`$type' at $file:$line" + done + + return 1 +} + + +## +# Begin describing a SUT +# +# All arguments are used as the description of the SUT (but remember that, +# even though the DSL makes it look like plain english, shell quoting and +# escaping rules still apply!). +describe() +{ + local -r desc="$*" + _sstack-push "describe" $(caller) "$desc" +} + + +## +# Declare a fact +# +# Like `describe', the entire argument list is used as the fact description. +it() +{ + local -r desc="$*" + _sstack-push "it" $(caller) "$desc" +} + + +## +# End the nearest declaration +# +# Note that some declarations (such as `expect') are implicitly closed and +# should not use this command. +end() +{ + local -r head="$(_sstack-head-type)" + local -r cleanhead="$head" + + # some statements are implicitly terminated; explicitly doing so is + # indicitive of a syntax issue + [ "${head:0:1}" != : ] \ + || _bail "unexpected \`end': still processing \`$cleanhead'" $(caller) + + _sstack-pop >/dev/null || _bail "unmatched \`end'" +} + + +## +# Declare the premise of an expectation +# +# All arguments are interpreted to be the command line to execute for the +# test. The actual expectations that assert upon this declaration are +# defined by `to'. +# +# That is, this declares "given this command, I can expect that..." +expect() +{ + _sstack-assert-within it expect $(caller) + __spec_result="$("$@" 2>"$__spec_errpath")" + __spec_rexit=$? + _sstack-push :expect $(caller) "$@" +} + + +## +# Declare expectations +# +# This declares an expectation on the immediately preceding expectation +# premise. +to() +{ + __spec_caller=${__spec_caller:-$(caller)} + + [ $# -gt 0 ] || _bail "missing assertion string for \`to'" $__spec_caller + + local -r expect_full="$*" + local -r type="$1" + shift + + _sstack-assert-follow :expect to $(caller) + _sstack-pop + + local -r assert="_expect--$type" + type "$assert" &>/dev/null \ + || _bail "unknown expectation: \`$type'" $__spec_caller + + # invoke partially-applied assertion with remaining arguments + $assert "$@" || fail "$expect_full" + __spec_caller= +} + + +## +# Outputs failure details and exits +# +# TODO: This is a temporary implementation; we would like to list all +# failures, and the actual display shall be handled by a reporter, not by +# us. This function will ultimately, therefore, simply collect data for +# later processing. +fail() +{ + echo "expected $*" >&2 + + echo ' stdout:' + sed 's/^/ /g' <<< "$__spec_result" + echo + echo ' stderr:' + sed 's/^/ /g' "$__spec_errpath" + echo + echo " exit code: $__spec_rexit" + + exit 1 +} + + +## +# Something went wrong; immediately abort processing with an error +# +# This should only be used in the case of a specification parsing error or +# other fatal errors; it should not be used for failing expectations. +_bail() +{ + local msg="$1" + local line="$2" + local file="$3" + + echo -n "error: $1" >&2 + [ $# -gt 1 ] && echo -n " at $file:$line" >&2 + + echo + exit 1 +} + + +## +# Retrieve the exit code of the most recently executed command +_spec-last-exit() +{ + echo "$__spec_rexit" +} + diff --git a/src/specstack b/src/specstack new file mode 100644 index 0000000..91cbc96 --- /dev/null +++ b/src/specstack @@ -0,0 +1,164 @@ +#!/bin/bash +# Parser stack +# +# Copyright (C) 2014 Mike Gewitz +# +# This file is part of shspec. +# +# shspec is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +## + +[ -z $__INC_SPECSTACK ] || return +__INC_SPECSTACK=1 + +declare -a __sstack=() +declare -i __sstackp=0 + + +## +# Push a frame onto the stack +_sstack-push() +{ + local -r type="$1" + local -r srcline="$2" + local -r srcfile="$3" + shift 3 + + __sstack[$__sstackp]="$type|$srcline|$srcfile|$*" + ((__sstackp++)) +} + + +## +# Pop a frame from the stack +# +# It is possible to recover the most recently popped frame. +_sstack-pop() +{ + [ "$__sstackp" -gt 0 ] || return 1 + + # notice that we unset before we decrease the pointer; this allows for a + # single level of un-popping + unset __sstack[$__sstackp] + ((__sstackp--)) +} + + +## +# Recover the most recently popped frame +# +# Note that this should never be called more than once in an attempt to +# recover additional frames; it will not work, and you will make bad things +# happen, and people will hate you. +_sstack-unpop() +{ + ((__sstackp++)) +} + + +## +# Return with a non-zero status only if the stack is non-empty +_sstack-empty() +{ + test "$__sstackp" -eq 0 +} + + +## +# Output the current size of the stack +_sstack-size() +{ + echo "$__sstackp" +} + + +## +# Output the current stack frame +_sstack-head() +{ + local -ri headi=$((__sstackp-1)) + echo "${__sstack[$headi]}" +} + + +## +# Output the type of the current stack frame +_sstack-head-type() +{ + __sstack-headn 0 +} + + +## +# Output the Nth datum of the current stack frame +__sstack-headn() +{ + local -ri i="$1" + local parts + + _sstack-read -a parts <<< "$(_sstack-head)" + echo "${parts[$i]}" +} + + +## +# Deconstruct stack frame from stdin in a `read`-like manner +_sstack-read() +{ + IFS=\| read "$@" +} + + +## +# Deconstruct current stack frame in a `read`-like manner +# +# Return immediately with a non-zero status if there are no frames on the +# stack. +_sstack-read-pop() +{ + local -r head="$(_sstack-pop)" || return 1 + _sstack-read "$@" <<< "$head" +} + + +## +# Assert that the immediately preceding frame is of the given type +# +# Conceptually, this allows determining if the parent node in a tree-like +# structure is of a certain type. +_sstack-assert-within() +{ + local -r in="$1" + local -r chk="$2" + local -ri line="$3" + local -r file="$4" + local -r phrase="${5:-be contained within}" + + local -r head="$(_sstack-head-type)" + + [ "$head" == "$in" ] \ + || _bail "\`$chk' must $phrase \`$head'; found \`$head' at $file:$line" +} + + +## +# Alias for _sstack-assert-within with altered error message +# +# This is intended to convey a different perspective: that a given node is a +# sibling, not a child, in a tree-like structure. +_sstack-assert-follow() +{ + _sstack-assert-within "$@" follow +} + diff --git a/test/test-spec b/test/test-spec new file mode 100644 index 0000000..5d84e6f --- /dev/null +++ b/test/test-spec @@ -0,0 +1,198 @@ +#!/bin/bash +# Specification DSL test +# +# Copyright (C) 2014 Mike Gewitz +# +# This file is part of shspec. +# +# shspec is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# TODO: This is incompletely tested because the tests must be written as the +# system itself is written! Therefore, it is not presently capable of +# running all necessary assertions on itself. +# +# Note: this test case will run pretty slowly because it forks a new process +# for each expectation. +## + + +## +# It is necessary to test syntax in a separate process so we do not +# interfere with the existing stack +# +# This will take an arbitrary script from stdin and execute it +test-run() +{ + ../src/run-spec <( cat ) +} + + +describe _begin-spec + it is a placeholder that exits successfully + expect _begin-spec + to succeed + end +end + + +describe describe + it fails if not paired with an "'end'" + expect test-run <<< ' + describe foo + # no end + '; to fail + end + + it succeeds if paired with matching "'end'" + expect test-run <<< ' + describe foo + end + '; to succeed + end +end + + +describe it + it fails within "'describe'" if not paired with an "'end'" + expect test-run <<< ' + describe it test + it will fail + # no end + end + '; to fail + end + + it succeeds within "'describe'" if paired with an "'end'" + expect test-run <<< ' + describe it test + it will succeed + end + end + '; to succeed + end +end + + +describe expect + it fails when not within "'describe'" + expect test-run <<< ' + describe will fail + # not within it + expect true; to succeed + end + '; to fail + end + + it succeeds with "'to'" clause when when within "'describe'" + expect test-run <<< ' + describe foo + it will succeed + expect true; to succeed + end + end + '; to succeed + end + + it fails when missing "'to'" clause of expectation + expect test-run <<< ' + describe foo + it will fail + # no "to" clause + expect true + end + end + '; to fail + end + + it cannot be "'end'd" before "'to'" clause + expect test-run <<< ' + describe foo + it will fail + expect true + # expectation is unfinished + end + end + end + '; to fail + end + + it properly executes quoted command lines + expect test-run <<< ' + chk() { test $# -eq 1; } + describe foo + it handles whitespace + expect chk "foo bar" + to succeed + end + end + '; to succeed + end + + + describe to + it fails when missing assertion string + expect test-run <<< ' + describe foo + it will fail + # missing assertion sting + expect true; to + end + end + '; to fail + end + + it can only follow an "'expect'" clause + expect test-run <<< ' + describe foo + it will fail + # does not follow expect clause + to succeed + end + '; to fail + end + + it will fail on unknwon expectation + expect test-run <<< ' + describe foo + it will fail + expect true + to ___some-invalid-expectation___ + end + end + '; to fail + end + + it will fail when expectation fails + expect test-run <<< ' + describe foo + it will fail + # true is not a failure + expect true; to fail + end + end + '; to fail + end + + it will succeed when expectation succeeds + expect test-run <<< ' + describe foo + it will succeed + expect false; to fail + end + end + '; to succeed + end + end +end +