Initial concept and working tests for shspec

My original prototype was more feature-rich than this, but this formalizes
it and provides self-tests.

It is indeed odd seeing shell code that does not look like shell.
stderr
Mike Gerwitz 2014-05-09 00:46:18 -04:00
commit 3d1f9d22a7
No known key found for this signature in database
GPG Key ID: F22BB8158EE30EAB
9 changed files with 988 additions and 0 deletions

33
bin/shspec 100755
View File

@ -0,0 +1,33 @@
#!/bin/bash
# shspec BDD CLI
#
# Copyright (C) 2014 Mike Gewitz
#
# This file is part of shspec.
#
# shspec is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
##
# all of our arguments (at the moment) are paths; if they're relative, then
# that'll be a problem, since we're changing directories, so convert them to
# absolute paths
declare -a paths=()
for arg in "$@"; do
paths+=( "$( readlink -f "$arg" )" )
done
# place ourselves in a known state and kick off the CLI
cd "$(dirname $0)"/../src \
&& exec ./cli "${paths[@]}"

24
src/cli 100755
View File

@ -0,0 +1,24 @@
#!/bin/bash
# shspec CLI
#
# Copyright (C) 2014 Mike Gerwitz
#
# This file is part of shspec.
#
# shspec is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
##
# we do not have much of a CLI atm; delegate to runner
exec ./runner "$@"

262
src/common 100644
View File

@ -0,0 +1,262 @@
#!/bin/bash
# Common test case file
#
# Copyright (C) 2014 LoVullo Associates, Inc.
#
# This file is part of trello-sh.
#
# trello-sh is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
##
declare -a __desc_stack=()
declare -i __desc_stackp=0
# current test csae
__desc_case=
# stderr file
__desc_errpath="$(mktemp)"
# most recent expect result and its exit code
__desc_result=
__desc_rexit=0
# most recent caller for assertions
__desc_caller=
__desc-push()
{
__desc_stack[$__desc_stackp]="$*"
((__desc_stackp++))
}
__desc-pop()
{
[ "$__desc_stackp" -gt 0 ] || return 1
# notice that we unset before we decrease the pointer; this allows for a
# single level of un-popping
unset __desc_stack[$__desc_stackp]
((__desc_stackp--))
}
__desc-unpop()
{
((__desc_stackp++))
}
__desc-head()
{
local headi=$((__desc_stackp - 1))
echo "${__desc_stack[ $headi ]}"
}
__desc-headtype()
{
local parts=( $(__desc-head) )
echo "${parts[0]}"
}
__desc-assert-within()
{
local in="$1"
local chk="$2"
local line="$3"
local file="$4"
local phrase="${5:-be contained within}"
local head="$(__desc-headtype)"
local printin="$(__desc-type-clean "$in")"
[ "$head" == "$in" ] \
|| bail "\`$chk' must $phrase \`$printin'; found \`$head' at $file:$line"
}
__desc-type-clean()
{
# a colon prefix is semantic; strip for display
echo "${1#:}"
}
__desc-assert-follow()
{
__desc-assert-within "$@" follow
}
begin-case()
{
__desc_case="$1"
}
tests()
{
source ../src/"$1"
}
describe()
{
__desc-push "describe $(caller) $@"
}
it()
{
__desc-push "it $(caller) $@"
}
end()
{
local head="$(__desc-headtype)"
local cleanhead="$(__desc-type-clean "$head")"
# some statements are implicitly terminated; explicitly doing so is
# indicitive of a syntax issue
[ "${head:0:1}" != : ] \
|| bail "unexpected \`end': still processing \`$cleanhead'" $(caller)
__desc-pop || bail "unmatched \`end'"
}
end-case()
{
local tcase="$__desc_case"
__desc_case=
# nothing to do if our stack is clean
[ "$__desc_stackp" -gt 0 ] || return 0
# output an error message for each item in the stack
while ((__desc_stackp--)); do
read type line file __ <<< "$(__desc-head)"
echo "error: $tcase: unterminated \`$type' at $file:$line"
done
exit 1
}
bail()
{
local msg="$1"
local line="$2"
local file="$3"
echo -n "error: $1" >&2
[ $# -gt 1 ] && echo -n " at $file:$line" >&2
echo
exit 1
}
expect()
{
__desc-assert-within it expect $(caller)
__desc_result="$($@ 2>"$__desc_errpath")"
__desc_rexit=$?
__desc-push ":expect $(caller) $@"
}
to()
{
__desc_caller=${__desc_caller:-$(caller)}
[ $# -gt 0 ] || bail "missing assertion string for \`to'" $__desc_caller
local type="$1"
shift
__desc-assert-follow :expect to $(caller)
__desc-pop
local assert="_desca-$type"
[ "$( type -t "$assert" )" == function ] \
|| bail "unknown \`to' assertion: \`$type'" $__desc_caller
# invoke partially-applied assertion with remaining arguments
$assert "$@" || fail
__desc_caller=
}
and()
{
__desc_caller="$(caller)"
__desc-unpop
"$@"
}
#
# Assertions
#
_desca-succeed()
{
__desca-exit -eq 0
}
_desca-fail()
{
__desca-exit -ne 0
}
_desca-exit()
{
__desca--argsep with 2 __desca-exit -eq
}
__desca-exit()
{
__desca--argn 2 "$#"
[ "$__desc_rexit" "$@" ]
}
_desca-output()
{
local cmp="$1"
diff="$( diff <( echo "$__desc_result" ) <( echo "$cmp" ) )" \
&& return 0
echo "$diff" >&2
return 1
}
__desca--argsep()
{
local delim="$1"
local cmdlen="$2"
local partial=
shift 2
# bash doesn't support multi-dimensional arrays, so here's to awkward command
# reconstruction with argument counts
while ((cmdlen--)); do
partial="$partial $1"
shift
done
local next="$1"
shift
[ "$next" == "$delim" ] \
|| bail "expected \`$delim' but found \`$next'" $__desc_caller
$partial "$@"
}
__desca--argn()
{
[ $2 -eq "$1" ] || bail "invalid number of arguments" $__desc_caller
}

28
src/expect-core 100644
View File

@ -0,0 +1,28 @@
#!/bin/bash
# Core expectations
#
# Copyright (C) 2014 Mike Gewitz
#
# This file is part of shspec.
#
# shspec is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
##
[ -z $__INC_EXPECT_CORE ] || return
__INC_EXPECT_CORE=1
_expect--succeed() { test "$(_spec-last-exit)" -eq 0; }
_expect--fail() { test "$(_spec-last-exit)" -ne 0; }

29
src/run-spec 100755
View File

@ -0,0 +1,29 @@
#!/bin/bash
# shspec CLI
#
# Copyright (C) 2014 Mike Gerwitz
#
# This file is part of shspec.
#
# shspec is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
##
source spec
declare -r tcase="${1?Missing test case path}"
_begin-spec
source "$tcase" || exit $?
_end-spec

26
src/runner 100755
View File

@ -0,0 +1,26 @@
#!/bin/bash
# shspec CLI
#
# Copyright (C) 2014 Mike Gerwitz
#
# This file is part of shspec.
#
# shspec is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
##
# each test is run in a separate process to ensure environment isolation
for tcase in "$@"; do
env -i ./run-spec "$tcase"
done

224
src/spec 100644
View File

@ -0,0 +1,224 @@
#!/bin/bash
# Specification language
#
# Copyright (C) 2014 Mike Gewitz
#
# This file is part of shspec.
#
# shspec is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Upon examining this code, you will likely notice that it is not
# re-entrant. This is okay, because the intended use is to invoke a new
# process *per specification*, not to run multiple specifications
# concurrently within the same process.
#
# You've been warned.
##
[ -z $__INC_SPEC ] || return
__INC_SPEC=1
source specstack
source expect-core
##
# Attempts to make tempoary path in /dev/shm, falling back to default
mktemp-shm()
{
local -r shm=/dev/shm
[ -d $shm -a -r $shm ] && mktemp -p$shm || mktemp
}
# stderr file
declare -r __spec_errpath="$(mktemp-shm)"
# most recent expect result and its exit code
declare __spec_result=
declare -i __spec_rexit=0
# most recent caller for expectations
declare __spec_caller=
##
# Begin a new specification definition
#
# A specification is any shell script using the specification commands
# defined herein.
_begin-spec()
{
: placeholder
}
##
# Mark the end of a specification
#
# If the specification was improperly nested, this will output a list of
# nesting errors and return a non-zero error. Otherwise, nothing is done.
_end-spec()
{
# if the stack is empty then everything is in order
_sstack-empty && return 0
# otherwise, output an error message for each item in the stack
until _sstack-empty; do
_sstack-read type line file _ < <(_sstack-head)
_sstack-pop
echo "error: unterminated \`$type' at $file:$line"
done
return 1
}
##
# Begin describing a SUT
#
# All arguments are used as the description of the SUT (but remember that,
# even though the DSL makes it look like plain english, shell quoting and
# escaping rules still apply!).
describe()
{
local -r desc="$*"
_sstack-push "describe" $(caller) "$desc"
}
##
# Declare a fact
#
# Like `describe', the entire argument list is used as the fact description.
it()
{
local -r desc="$*"
_sstack-push "it" $(caller) "$desc"
}
##
# End the nearest declaration
#
# Note that some declarations (such as `expect') are implicitly closed and
# should not use this command.
end()
{
local -r head="$(_sstack-head-type)"
local -r cleanhead="$head"
# some statements are implicitly terminated; explicitly doing so is
# indicitive of a syntax issue
[ "${head:0:1}" != : ] \
|| _bail "unexpected \`end': still processing \`$cleanhead'" $(caller)
_sstack-pop >/dev/null || _bail "unmatched \`end'"
}
##
# Declare the premise of an expectation
#
# All arguments are interpreted to be the command line to execute for the
# test. The actual expectations that assert upon this declaration are
# defined by `to'.
#
# That is, this declares "given this command, I can expect that..."
expect()
{
_sstack-assert-within it expect $(caller)
__spec_result="$("$@" 2>"$__spec_errpath")"
__spec_rexit=$?
_sstack-push :expect $(caller) "$@"
}
##
# Declare expectations
#
# This declares an expectation on the immediately preceding expectation
# premise.
to()
{
__spec_caller=${__spec_caller:-$(caller)}
[ $# -gt 0 ] || _bail "missing assertion string for \`to'" $__spec_caller
local -r expect_full="$*"
local -r type="$1"
shift
_sstack-assert-follow :expect to $(caller)
_sstack-pop
local -r assert="_expect--$type"
type "$assert" &>/dev/null \
|| _bail "unknown expectation: \`$type'" $__spec_caller
# invoke partially-applied assertion with remaining arguments
$assert "$@" || fail "$expect_full"
__spec_caller=
}
##
# Outputs failure details and exits
#
# TODO: This is a temporary implementation; we would like to list all
# failures, and the actual display shall be handled by a reporter, not by
# us. This function will ultimately, therefore, simply collect data for
# later processing.
fail()
{
echo "expected $*" >&2
echo ' stdout:'
sed 's/^/ /g' <<< "$__spec_result"
echo
echo ' stderr:'
sed 's/^/ /g' "$__spec_errpath"
echo
echo " exit code: $__spec_rexit"
exit 1
}
##
# Something went wrong; immediately abort processing with an error
#
# This should only be used in the case of a specification parsing error or
# other fatal errors; it should not be used for failing expectations.
_bail()
{
local msg="$1"
local line="$2"
local file="$3"
echo -n "error: $1" >&2
[ $# -gt 1 ] && echo -n " at $file:$line" >&2
echo
exit 1
}
##
# Retrieve the exit code of the most recently executed command
_spec-last-exit()
{
echo "$__spec_rexit"
}

164
src/specstack 100644
View File

@ -0,0 +1,164 @@
#!/bin/bash
# Parser stack
#
# Copyright (C) 2014 Mike Gewitz
#
# This file is part of shspec.
#
# shspec is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
##
[ -z $__INC_SPECSTACK ] || return
__INC_SPECSTACK=1
declare -a __sstack=()
declare -i __sstackp=0
##
# Push a frame onto the stack
_sstack-push()
{
local -r type="$1"
local -r srcline="$2"
local -r srcfile="$3"
shift 3
__sstack[$__sstackp]="$type|$srcline|$srcfile|$*"
((__sstackp++))
}
##
# Pop a frame from the stack
#
# It is possible to recover the most recently popped frame.
_sstack-pop()
{
[ "$__sstackp" -gt 0 ] || return 1
# notice that we unset before we decrease the pointer; this allows for a
# single level of un-popping
unset __sstack[$__sstackp]
((__sstackp--))
}
##
# Recover the most recently popped frame
#
# Note that this should never be called more than once in an attempt to
# recover additional frames; it will not work, and you will make bad things
# happen, and people will hate you.
_sstack-unpop()
{
((__sstackp++))
}
##
# Return with a non-zero status only if the stack is non-empty
_sstack-empty()
{
test "$__sstackp" -eq 0
}
##
# Output the current size of the stack
_sstack-size()
{
echo "$__sstackp"
}
##
# Output the current stack frame
_sstack-head()
{
local -ri headi=$((__sstackp-1))
echo "${__sstack[$headi]}"
}
##
# Output the type of the current stack frame
_sstack-head-type()
{
__sstack-headn 0
}
##
# Output the Nth datum of the current stack frame
__sstack-headn()
{
local -ri i="$1"
local parts
_sstack-read -a parts <<< "$(_sstack-head)"
echo "${parts[$i]}"
}
##
# Deconstruct stack frame from stdin in a `read`-like manner
_sstack-read()
{
IFS=\| read "$@"
}
##
# Deconstruct current stack frame in a `read`-like manner
#
# Return immediately with a non-zero status if there are no frames on the
# stack.
_sstack-read-pop()
{
local -r head="$(_sstack-pop)" || return 1
_sstack-read "$@" <<< "$head"
}
##
# Assert that the immediately preceding frame is of the given type
#
# Conceptually, this allows determining if the parent node in a tree-like
# structure is of a certain type.
_sstack-assert-within()
{
local -r in="$1"
local -r chk="$2"
local -ri line="$3"
local -r file="$4"
local -r phrase="${5:-be contained within}"
local -r head="$(_sstack-head-type)"
[ "$head" == "$in" ] \
|| _bail "\`$chk' must $phrase \`$head'; found \`$head' at $file:$line"
}
##
# Alias for _sstack-assert-within with altered error message
#
# This is intended to convey a different perspective: that a given node is a
# sibling, not a child, in a tree-like structure.
_sstack-assert-follow()
{
_sstack-assert-within "$@" follow
}

198
test/test-spec 100644
View File

@ -0,0 +1,198 @@
#!/bin/bash
# Specification DSL test
#
# Copyright (C) 2014 Mike Gewitz
#
# This file is part of shspec.
#
# shspec is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# TODO: This is incompletely tested because the tests must be written as the
# system itself is written! Therefore, it is not presently capable of
# running all necessary assertions on itself.
#
# Note: this test case will run pretty slowly because it forks a new process
# for each expectation.
##
##
# It is necessary to test syntax in a separate process so we do not
# interfere with the existing stack
#
# This will take an arbitrary script from stdin and execute it
test-run()
{
../src/run-spec <( cat )
}
describe _begin-spec
it is a placeholder that exits successfully
expect _begin-spec
to succeed
end
end
describe describe
it fails if not paired with an "'end'"
expect test-run <<< '
describe foo
# no end
'; to fail
end
it succeeds if paired with matching "'end'"
expect test-run <<< '
describe foo
end
'; to succeed
end
end
describe it
it fails within "'describe'" if not paired with an "'end'"
expect test-run <<< '
describe it test
it will fail
# no end
end
'; to fail
end
it succeeds within "'describe'" if paired with an "'end'"
expect test-run <<< '
describe it test
it will succeed
end
end
'; to succeed
end
end
describe expect
it fails when not within "'describe'"
expect test-run <<< '
describe will fail
# not within it
expect true; to succeed
end
'; to fail
end
it succeeds with "'to'" clause when when within "'describe'"
expect test-run <<< '
describe foo
it will succeed
expect true; to succeed
end
end
'; to succeed
end
it fails when missing "'to'" clause of expectation
expect test-run <<< '
describe foo
it will fail
# no "to" clause
expect true
end
end
'; to fail
end
it cannot be "'end'd" before "'to'" clause
expect test-run <<< '
describe foo
it will fail
expect true
# expectation is unfinished
end
end
end
'; to fail
end
it properly executes quoted command lines
expect test-run <<< '
chk() { test $# -eq 1; }
describe foo
it handles whitespace
expect chk "foo bar"
to succeed
end
end
'; to succeed
end
describe to
it fails when missing assertion string
expect test-run <<< '
describe foo
it will fail
# missing assertion sting
expect true; to
end
end
'; to fail
end
it can only follow an "'expect'" clause
expect test-run <<< '
describe foo
it will fail
# does not follow expect clause
to succeed
end
'; to fail
end
it will fail on unknwon expectation
expect test-run <<< '
describe foo
it will fail
expect true
to ___some-invalid-expectation___
end
end
'; to fail
end
it will fail when expectation fails
expect test-run <<< '
describe foo
it will fail
# true is not a failure
expect true; to fail
end
end
'; to fail
end
it will succeed when expectation succeeds
expect test-run <<< '
describe foo
it will succeed
expect false; to fail
end
end
'; to succeed
end
end
end