Re: Lots of test failures when --disable-multibyte

[Peter Stephenson <p.w.stephenson@xxxxxxxxxxxx>]

On Mon, 2022-04-04 at 15:00 -0700, Bart Schaefer wrote:
> ** We might want to consider a ZTST_ mode where we attempt all tests
> in a file even if one fails.  I confirmed these by marking them "0f"
> one by one, but that could become tedious.

This might help?

pws

diff --git a/Test/README b/Test/README
index 726d68e72..670434ac3 100644
--- a/Test/README
+++ b/Test/README
@@ -20,6 +20,13 @@ more information about the tests being performed with
   ZTST_verbose=1 make check
 (`test' is equivalent to `check') or change 1 to 2 for even more detail.
 
+A test file is usually aborted on the first error.  To continue to the
+end, run with
+  ZTST_continue=1 make check
+This can usefully be combined with ZTST_verbose.  The test is always
+aborted on a syntax error as in that case it is not obvoius how to
+continue.
+
 Individual or groups of tests can be performed with
   make TESTNUM=C02 check
 or
diff --git a/Test/ztst.zsh b/Test/ztst.zsh
index 89fe69b5b..a5e118550 100755
--- a/Test/ztst.zsh
+++ b/Test/ztst.zsh
@@ -143,6 +143,8 @@ ZTST_testfailed() {
 $ZTST_failmsg"
   fi
   ZTST_testfailed=1
+  ret=1
+  (( ++failures ))
   return 1
 }
 ZTST_testxpassed() {
@@ -373,12 +375,12 @@ ZTST_diff() {
 
   return "$diff_ret"
 }
-    
+
 ZTST_test() {
   local last match mbegin mend found substlines
   local diff_out diff_err
   local ZTST_skip
-  integer expected_to_fail
+  integer expected_to_fail ret failures
 
   while true; do
     rm -f $ZTST_in $ZTST_out $ZTST_err
@@ -492,7 +494,7 @@ $ZTST_curline"
 $ZTST_code${$(<$ZTST_terr):+
 Error output:
 $(<$ZTST_terr)}"
-	return 1
+	if [[ -z $ZTST_continue ]]; then return 1; else continue; fi
       fi
 
       ZTST_verbose 2 "ZTST_test: test produced standard output:
@@ -515,7 +517,7 @@ $(<$ZTST_terr)"
 $ZTST_code${$(<$ZTST_terr):+
 Error output:
 $(<$ZTST_terr)}"
-	return 1
+	if [[ -z $ZTST_continue ]]; then return 1; else continue; fi
       fi
       if [[ $ZTST_flags = *q* && -s $ZTST_err ]]; then
 	substlines="$(<$ZTST_err)"
@@ -529,21 +531,27 @@ $(<$ZTST_terr)}"
         fi
 	ZTST_testfailed "error output differs from expected as shown above for:
 $ZTST_code"
-	return 1
+	if [[ -z $ZTST_continue ]]; then return 1; else continue; fi
       fi
       if (( expected_to_fail )); then
         ZTST_testxpassed
-        return 1
+	if [[ -z $ZTST_continue ]]; then return 1; else continue; fi
       fi
     fi
     ZTST_verbose 1 "Test successful."
     [[ -n $last ]] && break
   done
 
-  ZTST_verbose 2 "ZTST_test: all tests successful"
+  if (( failures )); then
+    ZTST_verbose 1 "ZTST_test: $failures tests failed"
+  else
+    ZTST_verbose 2 "ZTST_test: all tests successful"
+  fi
 
   # reset message to keep ZTST_testfailed output correct
   ZTST_message=''
+
+  return ret
 }