Re: [PATCH] find RLIM_NLIMITS correctly on Cygwin

[Jun T <takimoto-j@xxxxxxxxxxxxxxxxx>]

> 2020/02/28 23:19, Daniel Shahaf <d.s@xxxxxxxxxxxxxxxxxx> wrote:
> 
> Jun T wrote on Fri, 28 Feb 2020 17:42 +0900:
>> 
>> Currently each test chunk can have flags 'd' 'D' 'q' and 'f'.
>> We can add a new flag, say 'n', to make the failure not fatal.
> 
> If we add a new letter flag, we won't be able to make the non-fatalness
> specific to only one platform, though.  We a new parameter, we'd be
> able to do «[[ $OSTYPE == cygwin* ]] && ZTST_failure_is_fatal=false».

I feel just letting the test fail may be enough,  but if we are going
to add a new parameter/flag to ztst.zsh then it would be something like
the patch below.

BTW, in B01cd.ztst, line 73:

#  . d  Don't diff stdout against the expected stdout.

What does this '.' mean?  Is it just a typo?



 Test/B01cd.ztst    |  7 +++++--
 Test/B12limit.ztst | 10 ++++++++++
 Test/ztst.zsh      | 32 ++++++++++++++++++++++----------
 3 files changed, 37 insertions(+), 12 deletions(-)


diff --git a/Test/B01cd.ztst b/Test/B01cd.ztst
index d903b7462..a91fdd95d 100644
--- a/Test/B01cd.ztst
+++ b/Test/B01cd.ztst
@@ -96,8 +96,9 @@
 # itself.  (The example below isn't particularly useful as errors with
 # `cd' are unusual.)
 #
-# A couple of features aren't used in this file, but are usefuil in cases
-# where features may not be available so should not be tested.  They boh
+# A couple of features aren't used in this file, but are useful in cases
+# where features may not be available so should not be tested, or a
+# failure of a test need not be considered as a fatal error.  They all
 # take the form of variables.  Note that to keep the test framework simple
 # there is no magic in setting the variables: the chunk of code being
 # executed needs to avoid executing any test code by appropriate structure
@@ -107,6 +108,8 @@
 #   is to be skipped.
 # ZTST_skip: Set this in any test case if that single test case is to be
 #   skipped.  Testing resumes at the next test case in the same file.
+# ZTST_not_fatal: Set this in any test case if its failure is not fatal.
+#   The failure is reported but the test case is considered as passed.
 #
 # Syntax highlighting for Vim is available, see Util/ztst-*.vim.
  cd cdtst.tmp/sub/fake &&
diff --git a/Test/B12limit.ztst b/Test/B12limit.ztst
new file mode 100644
index 000000000..2a072da08
--- /dev/null
+++ b/Test/B12limit.ztst
@@ -0,0 +1,10 @@
+# check if there is unknown resouce(s)
+
+%test
+
+ ZTST_not_fatal=1
+ limit | grep UNKNOWN || print OK
+0:Check if there is unknown resouce(s) in the system
+>OK
+F:A failure here just indicates there is a resource in your system that is
+F:unknown to zsh developers. Please report this to zsh-workers mailing list.
diff --git a/Test/ztst.zsh b/Test/ztst.zsh
index 375efd16c..496a5c1bd 100755
--- a/Test/ztst.zsh
+++ b/Test/ztst.zsh
@@ -133,17 +133,22 @@ rm -rf dummy.tmp *.tmp
 
 # Report failure.  Note that all output regarding the tests goes to stdout.
 # That saves an unpleasant mixture of stdout and stderr to sort out.
+# If $2 is given and equal to 1, this faiure is not fatal.
 ZTST_testfailed() {
   print -r "Test $ZTST_testname failed: $1"
   if [[ -n $ZTST_message ]]; then
     print -r "Was testing: $ZTST_message"
   fi
-  print -r "$ZTST_testname: test failed."
+  if (( $2 == 1 )); then
+    print -r "$ZTST_testname: Non-fatal failure."
+  else
+    print -r "$ZTST_testname: test failed."
+    ZTST_testfailed=1
+  fi
   if [[ -n $ZTST_failmsg ]]; then
     print -r "The following may (or may not) help identifying the cause:
 $ZTST_failmsg"
   fi
-  ZTST_testfailed=1
   return 1
 }
 
@@ -365,8 +370,8 @@ ZTST_diff() {
 ZTST_test() {
   local last match mbegin mend found substlines
   local diff_out diff_err
-  local ZTST_skip
-  integer expected_to_fail
+  local ZTST_skip ZTST_not_fatal
+  integer expected_to_fail not_fatal
 
   while true; do
     rm -f $ZTST_in $ZTST_out $ZTST_err
@@ -461,6 +466,13 @@ $ZTST_curline"
 	fi
       fi
 
+      if [[ -n $ZTST_not_fatal ]]; then
+	not_fatal=1
+	ZTST_not_fatal=
+      else
+	not_fatal=0
+      fi
+
       if [[ $ZTST_flags = *f* ]]; then
         expected_to_fail=1
         ZTST_xfail_diff() { ZTST_diff "$@" > /dev/null }
@@ -479,7 +491,7 @@ $ZTST_curline"
 	ZTST_testfailed "bad status $ZTST_status, expected $ZTST_xstatus from:
 $ZTST_code${$(<$ZTST_terr):+
 Error output:
-$(<$ZTST_terr)}"
+$(<$ZTST_terr)}" $not_fatal
 	return 1
       fi
 
@@ -502,8 +514,8 @@ $(<$ZTST_terr)"
 	ZTST_testfailed "output differs from expected as shown above for:
 $ZTST_code${$(<$ZTST_terr):+
 Error output:
-$(<$ZTST_terr)}"
-	return 1
+$(<$ZTST_terr)}" $not_fatal
+	(( not_fatal )) && continue || return 1
       fi
       if [[ $ZTST_flags = *q* && -s $ZTST_err ]]; then
 	substlines="$(<$ZTST_err)"
@@ -516,12 +528,12 @@ $(<$ZTST_terr)}"
           continue
         fi
 	ZTST_testfailed "error output differs from expected as shown above for:
-$ZTST_code"
-	return 1
+$ZTST_code" $not_fatal
+	(( not_fatal )) && continue || return 1
       fi
       if (( expected_to_fail )); then
         ZTST_testfailed "test was expected to fail, but passed."
-        return 1
+	(( not_fatal )) && continue || return 1
       fi
     fi
     ZTST_verbose 1 "Test successful."