Switch normal regression testing to TAP output.
This commit is contained in:
Eric S. Raymond
parent
129136b51e
commit
d235313e97
3 changed files with 223 additions and 17 deletions
|
|
@ -19,7 +19,7 @@ TESTLOADS := $(shell ls -1 *.log | sed '/.log/s///' | sort)
|
|||
.PHONY: check coverage clean testlist listcheck savegames buildregress
|
||||
.PHONY: savecheck regress
|
||||
|
||||
check: savecheck regress
|
||||
check: savecheck tapcheck
|
||||
@echo "=== No diff output is good news."
|
||||
@-advent -x 2>/dev/null # Get usage message into coverage tests
|
||||
@-advent -l /dev/null <pitfall.log >/dev/null
|
||||
|
|
@ -87,22 +87,6 @@ savecheck: savegames
|
|||
@advent -r thousand_saves.adv < pitfall.log > /tmp/coverage_advent_readfail 2>&1 || exit 1
|
||||
@rm -f /tmp/coverage*
|
||||
|
||||
# General regression testing of commands and output; look at the *.log and
|
||||
# corresponding *.chk files to see which tests this runs.
|
||||
regress:
|
||||
@for file in $(TESTLOADS); do \
|
||||
$(ECHO) -n " $${file} "; grep '##' $${file}.log || echo ' ## (no description)'; \
|
||||
OPTS=`sed -n /#options:/s///p <$${file}.log`; \
|
||||
if $(advent) $$OPTS < $${file}.log >/tmp/regress$$$$ 2>&1; \
|
||||
then diff --text -u $${file}.chk /tmp/regress$$$$ || exit 1; \
|
||||
else echo "*** Nonzero return status on $${file}!"; exit 1; fi \
|
||||
done; \
|
||||
$(ECHO) " multifile ## Test multiple file arguments"
|
||||
@(echo "inven" | advent isofoo.log /dev/stdin | diff --text -u multifile.chk -) || exit 1
|
||||
rm -f scratch.tmp /tmp/regress$$$$
|
||||
|
||||
# Steps towards TAP. savecheck is not yet integrated.
|
||||
|
||||
# The TAP filter. Only affects presentation of the test suite messages
|
||||
TAPCONSUMER=tapview
|
||||
|
||||
|
|
|
|||
31
tests/tapdiffer
Executable file
31
tests/tapdiffer
Executable file
|
|
@ -0,0 +1,31 @@
|
|||
#! /bin/sh
|
||||
#
|
||||
# tapdiffer - Render diff between input and checkfile as a TAP report
|
||||
#
|
||||
# Usage: tapdiffer LEGEND CHECKFILE
|
||||
#
|
||||
# Output is a TAP report, ok if the diff is empty and not ok otherwise.
|
||||
# A nonempty diff is shipped as a TAP YAML block following "not ok"
|
||||
# unless QUIET=1 in the environment.
|
||||
#
|
||||
legend=$1
|
||||
checkfile=$2
|
||||
|
||||
trap 'rm /tmp/tapdiff$$' EXIT HUP INT QUIT TERM
|
||||
|
||||
if diff --text -u ${checkfile} - >/tmp/tapdiff$$
|
||||
then
|
||||
echo "ok - ${legend}"
|
||||
exit 0
|
||||
else
|
||||
echo "not ok - ${checkfile}: ${legend}"
|
||||
if [ ! "${QUIET}" = 1 ]
|
||||
then
|
||||
echo " --- |"
|
||||
sed </tmp/tapdiff$$ -e 's/^/ /'
|
||||
echo " ..."
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# end
|
||||
191
tests/tapview
Executable file
191
tests/tapview
Executable file
|
|
@ -0,0 +1,191 @@
|
|||
#! /bin/sh
|
||||
# tapview - a TAP (Test Anything Protocol) viewer in pure POSIX shell
|
||||
#
|
||||
# Copyright by Eric S. Raymond
|
||||
#
|
||||
# This code is intended to be embedded in your project. The author
|
||||
# grants permission for it to be distributed under the prevailing
|
||||
# license of your project if you choose, provided that license is
|
||||
# OSD-compliant; otherwise the following SPDX tag applies.
|
||||
#
|
||||
# SPDX-License-Identifier: BSD-2-Clause
|
||||
#
|
||||
# This is version 1.0
|
||||
# A newer version may be available at https://gitlab.com/esr/tapview
|
||||
#
|
||||
# POSIX allows but does not mandate that -n suppresses emission of a
|
||||
# trailing newline in echo. Thus, some shell builtin echos don't do
|
||||
# that. Cope gracefully.
|
||||
# shellcheck disable=SC2039
|
||||
if [ "$(echo -n "a"; echo "b")" != "ab" ]
|
||||
then
|
||||
ECHO="echo"
|
||||
elif [ "$(/bin/echo -n "a"; /bin/echo "b")" = "ab" ]
|
||||
then
|
||||
ECHO="/bin/echo"
|
||||
else
|
||||
echo "tapview: bailing out, your echo lacks -n support."
|
||||
exit 3
|
||||
fi
|
||||
|
||||
OK="."
|
||||
FAIL="F"
|
||||
SKIP="s"
|
||||
TODO_NOT_OK="x"
|
||||
TODO_OK="u"
|
||||
|
||||
ship_char() {
|
||||
# shellcheck disable=SC2039
|
||||
"${ECHO}" -n "$1"
|
||||
}
|
||||
|
||||
ship_line() {
|
||||
report="${report}${1}\n"
|
||||
}
|
||||
|
||||
testcount=0
|
||||
failcount=0
|
||||
skipcount=0
|
||||
todocount=0
|
||||
test_before_plan=no
|
||||
test_after_plan=no
|
||||
expect=""
|
||||
status=0
|
||||
|
||||
report=""
|
||||
IFS=""
|
||||
state=start
|
||||
while read -r line
|
||||
do
|
||||
if expr "$line" : "Bail out!" >/dev/null
|
||||
then
|
||||
ship_line "$line"
|
||||
status=2
|
||||
break
|
||||
fi
|
||||
if expr "$line" : '1\.\.[0-9][0-9]*' >/dev/null >/dev/null
|
||||
then
|
||||
if [ "$expect" != "" ]
|
||||
then
|
||||
if [ "${testcount}" -gt 0 ]
|
||||
then
|
||||
echo ""
|
||||
fi
|
||||
ship_line "Cannot have more than one plan line."
|
||||
echo "${report}"
|
||||
exit 1
|
||||
fi
|
||||
if expr "$line" : ".* *SKIP" >/dev/null || expr "$line" : ".* *skip" >/dev/null
|
||||
then
|
||||
ship_line "$line"
|
||||
echo "${report}"
|
||||
exit 1 # Not specified in the standard whether this should exit 1 or 0
|
||||
fi
|
||||
expect=$(expr "$line" : '1\.\.\([0-9][0-9]*\)')
|
||||
continue
|
||||
fi
|
||||
if expr "$line" : "ok" >/dev/null
|
||||
then
|
||||
testcount=$((testcount + 1))
|
||||
if [ "$expect" = "" ]
|
||||
then
|
||||
test_before_plan=yes
|
||||
else
|
||||
test_after_plan=yes
|
||||
fi
|
||||
if expr "$line" : ".*# *TODO" >/dev/null || expr "$line" : ".*# *todo" >/dev/null
|
||||
then
|
||||
ship_char ${TODO_OK}
|
||||
ship_line "$line"
|
||||
todocount=$((todocount + 1))
|
||||
elif expr "$line" : ".*# *SKIP" >/dev/null || expr "$line" : ".*# *skip" >/dev/null
|
||||
then
|
||||
ship_char ${SKIP}
|
||||
ship_line "$line"
|
||||
skipcount=$((skipcount + 1))
|
||||
else
|
||||
ship_char ${OK}
|
||||
fi
|
||||
state=ok
|
||||
continue
|
||||
fi
|
||||
if expr "$line" : "not ok" >/dev/null
|
||||
then
|
||||
testcount=$((testcount + 1))
|
||||
if [ "$expect" = "" ]
|
||||
then
|
||||
test_before_plan=yes
|
||||
else
|
||||
test_after_plan=yes
|
||||
fi
|
||||
if expr "$line" : ".*# *SKIP" >/dev/null || expr "$line" : ".*# *skip" >/dev/null
|
||||
then
|
||||
ship_char "${SKIP}"
|
||||
state=ok
|
||||
skipcount=$((skipcount + 1))
|
||||
continue
|
||||
fi
|
||||
if expr "$line" : ".*# *TODO" >/dev/null || expr "$line" : ".*# *todo" >/dev/null
|
||||
then
|
||||
ship_char ${TODO_NOT_OK}
|
||||
state=ok
|
||||
todocount=$((todocount + 1))
|
||||
continue
|
||||
fi
|
||||
ship_char "${FAIL}"
|
||||
ship_line "$line"
|
||||
state=not_ok
|
||||
failcount=$((failcount + 1))
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
# shellcheck disable=SC2166
|
||||
if [ "${state}" = "yaml" ]
|
||||
then
|
||||
ship_line "$line"
|
||||
if expr "$line" : '[ ]*\.\.\.' >/dev/null
|
||||
then
|
||||
state=ok
|
||||
fi
|
||||
elif expr "$line" : "[ ]*---" >/dev/null
|
||||
then
|
||||
ship_line "$line"
|
||||
state=yaml
|
||||
fi
|
||||
done
|
||||
|
||||
/bin/echo ""
|
||||
|
||||
if [ -z "$expect" ]
|
||||
then
|
||||
ship_line "Missing a plan."
|
||||
status=1
|
||||
elif [ "$test_before_plan" = "yes" ] && [ "$test_after_plan" = "yes" ]
|
||||
then
|
||||
ship_line "A plan line may only be placed before or after all tests."
|
||||
status=1
|
||||
elif [ "${expect}" -gt "${testcount}" ]
|
||||
then
|
||||
ship_line "Expected ${expect} tests but only ${testcount} ran."
|
||||
status=1
|
||||
elif [ "${expect}" -lt "${testcount}" ]
|
||||
then
|
||||
ship_line "Expected ${expect} tests but ${testcount} ran."
|
||||
status=1
|
||||
fi
|
||||
|
||||
report="${report}${testcount} tests, ${failcount} failures"
|
||||
if [ "$todocount" != 0 ]
|
||||
then
|
||||
report="${report}, ${todocount} TODOs"
|
||||
fi
|
||||
if [ "$skipcount" != 0 ]
|
||||
then
|
||||
report="${report}, ${skipcount} SKIPs"
|
||||
fi
|
||||
|
||||
echo "${report}."
|
||||
|
||||
exit "${status}"
|
||||
|
||||
# end
|
||||
Loading…
Add table
Add a link
Reference in a new issue