Re: Baffling array substitution behavior

[Peter Stephenson <pws@xxxxxxx>]

On Tue, 07 Oct 2008 22:35:53 -0700
Bart Schaefer <schaefer@xxxxxxxxxxxxxxxx> wrote:
> You may have seen the the pids4kill function that I posted to zsh-users
> a little while ago.  When I was first trying it out, I mistook opt_args
> for a plain array rather than an associative array, so I wrote:
> 
>    local u=$opt_args[(I)-u]
> 
> Strangely, the entire assignment vanished on parameter substitution, so
> what was executed was
> 
>    local
> 
> which naturally was not what I wanted.  Turns out this is a side-effect
> of "setopt rcexpandparam", but is it really an intended side-effect?

Probably not, but it's an effect of this code:

	if (PM_TYPE(pm->node.flags) & (PM_ARRAY|PM_HASHED)) {
	    /* Overload v->isarr as the flag bits for hashed arrays. */
	    v->isarr = flags | (isvarat ? SCANPM_ISVAR_AT : 0);
	    /* If no flags were passed, we need something to represent *
	     * `true' yet differ from an explicit WANTVALS.  This is a *
	     * bit of a hack, but makes some sense:  When no subscript *
	     * is provided, all values are substituted.                */
	    if (!v->isarr)
		v->isarr = SCANPM_MATCHMANY;
	}

There are no flags passed in at this point because you didn't ask
explicitly for keys or values, and nothing else is set at this point (we
haven't got down far enough to examine the subscript).  So it's being told
to match many and does.

We can use a more explicit flag, although it may have side effects.  The
test system just picked up one: the test for assigning to slices of an
associative array was depending explicitly on SCANPM_MATCHMANY.

Index: Doc/Zsh/expn.yo
===================================================================
RCS file: /cvsroot/zsh/zsh/Doc/Zsh/expn.yo,v
retrieving revision 1.94
diff -u -r1.94 expn.yo
--- Doc/Zsh/expn.yo	27 Sep 2008 20:58:26 -0000	1.94
+++ Doc/Zsh/expn.yo	9 Oct 2008 13:34:11 -0000
@@ -611,7 +611,8 @@
 where the parameter var(xx)
 is set to tt(LPAR())var(a b c)tt(RPAR()), are substituted with
 `var(fooabar foobbar foocbar)' instead of the default
-`var(fooa b cbar)'.
+`var(fooa b cbar)'.  Note that an empty array will therefore cause
+all arguments to be removed.
 
 Internally, each such expansion is converted into the
 equivalent list for brace expansion.  E.g., tt(${^var}) becomes
Index: Doc/Zsh/options.yo
===================================================================
RCS file: /cvsroot/zsh/zsh/Doc/Zsh/options.yo,v
retrieving revision 1.64
diff -u -r1.64 options.yo
--- Doc/Zsh/options.yo	11 Sep 2008 12:49:19 -0000	1.64
+++ Doc/Zsh/options.yo	9 Oct 2008 13:34:11 -0000
@@ -485,7 +485,8 @@
 `var(foo)tt(${)var(xx)tt(})var(bar)', where the parameter
 var(xx) is set to tt(LPAR())var(a b c)tt(RPAR()), are substituted with
 `var(fooabar foobbar foocbar)' instead of the default
-`var(fooa b cbar)'.
+`var(fooa b cbar)'.  Note that an empty array will therefore cause
+all arguments to be removed.
 )
 pindex(REMATCH_PCRE)
 cindex(regexp, PCRE)
Index: Src/params.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/params.c,v
retrieving revision 1.148
diff -u -r1.148 params.c
--- Src/params.c	29 Sep 2008 20:56:24 -0000	1.148
+++ Src/params.c	9 Oct 2008 13:34:11 -0000
@@ -1810,11 +1810,10 @@
 	    /* Overload v->isarr as the flag bits for hashed arrays. */
 	    v->isarr = flags | (isvarat ? SCANPM_ISVAR_AT : 0);
 	    /* If no flags were passed, we need something to represent *
-	     * `true' yet differ from an explicit WANTVALS.  This is a *
-	     * bit of a hack, but makes some sense:  When no subscript *
-	     * is provided, all values are substituted.                */
+	     * `true' yet differ from an explicit WANTVALS.  Use a     *
+	     * special flag for this case.                             */
 	    if (!v->isarr)
-		v->isarr = SCANPM_MATCHMANY;
+		v->isarr = SCANPM_ARRONLY;
 	}
 	v->pm = pm;
 	v->flags = 0;
@@ -2188,7 +2187,8 @@
 	zsfree(val);
 	return;
     }
-    if ((v->pm->node.flags & PM_HASHED) && (v->isarr & SCANPM_MATCHMANY)) {
+    if ((v->pm->node.flags & PM_HASHED) &&
+	(v->isarr & (SCANPM_MATCHMANY|SCANPM_ARRONLY))) {
 	zerr("%s: attempt to set slice of associative array", v->pm->node.nam);
 	zsfree(val);
 	return;
Index: Src/zsh.h
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/zsh.h,v
retrieving revision 1.148
diff -u -r1.148 zsh.h
--- Src/zsh.h	29 Sep 2008 21:46:58 -0000	1.148
+++ Src/zsh.h	9 Oct 2008 13:34:11 -0000
@@ -1532,16 +1532,25 @@
 #define TYPESET_OPTNUM "LRZiEF"
 
 /* Flags for extracting elements of arrays and associative arrays */
-#define SCANPM_WANTVALS   (1<<0)
-#define SCANPM_WANTKEYS   (1<<1)
-#define SCANPM_WANTINDEX  (1<<2)
-#define SCANPM_MATCHKEY   (1<<3)
-#define SCANPM_MATCHVAL   (1<<4)
-#define SCANPM_MATCHMANY  (1<<5)
-#define SCANPM_ASSIGNING  (1<<6)
-#define SCANPM_KEYMATCH   (1<<7)
-#define SCANPM_DQUOTED    (1<<8)
-#define SCANPM_ISVAR_AT   ((-1)<<15)	/* Only sign bit is significant */
+#define SCANPM_WANTVALS   (1<<0) /* Return value includes hash values */
+#define SCANPM_WANTKEYS   (1<<1) /* Return value includes hash keys */
+#define SCANPM_WANTINDEX  (1<<2) /* Return value includes array index */
+#define SCANPM_MATCHKEY   (1<<3) /* Subscript matched against key */
+#define SCANPM_MATCHVAL   (1<<4) /* Subscript matched against value */
+#define SCANPM_MATCHMANY  (1<<5) /* Subscript matched repeatedly, return all */
+#define SCANPM_ASSIGNING  (1<<6) /* Assigning whole array/hash */
+#define SCANPM_KEYMATCH   (1<<7) /* keys of hash treated as patterns */
+#define SCANPM_DQUOTED    (1<<8) /* substitution was double-quoted
+				  * (only used for testing early end of
+				  * subscript)
+				  */
+#define SCANPM_ARRONLY    (1<<9) /* value is array but we don't
+				  * necessarily want to match multiple
+				  * elements
+				  */
+#define SCANPM_ISVAR_AT   ((-1)<<15)	/* "$foo[@]"-style substitution
+					 * Only sign bit is significant
+					 */
 
 /*
  * Flags for doing matches inside parameter substitutions, i.e.
Index: Test/D04parameter.ztst
===================================================================
RCS file: /cvsroot/zsh/zsh/Test/D04parameter.ztst,v
retrieving revision 1.35
diff -u -r1.35 D04parameter.ztst
--- Test/D04parameter.ztst	27 Sep 2008 20:58:28 -0000	1.35
+++ Test/D04parameter.ztst	9 Oct 2008 13:34:11 -0000
@@ -1086,3 +1086,37 @@
 >no
 >no
 >no
+
+  rcexbug() {
+    emulate -L zsh
+    setopt rcexpandparam
+    local -A hash
+    local -a full empty
+    full=(X x)
+    hash=(X x)
+    print ORDINARY ARRAYS
+    : The following behaves as documented in zshoptions
+    print FULL expand=$full
+    : Empty arrays remove the adjacent argument
+    print EMPTY expand=$empty
+    print ASSOCIATIVE ARRAY
+    print Subscript flags returning many values
+    print FOUND key=$hash[(I)X] val=$hash[(R)x]
+    : This should behave like $empty, and does
+    print LOST key=$hash[(I)y] val=$hash[(R)Y]
+    print Subscript flags returning single values
+    : Doc says "substitutes ... empty string"
+    : so must not behave like an empty array
+    print STRING key=$hash[(i)y] val=$hash[(r)Y]
+  }
+  rcexbug
+0:Lookup failures on elements of arrays with RC_EXPAND_PARAM
+>ORDINARY ARRAYS
+>FULL expand=X expand=x
+>EMPTY
+>ASSOCIATIVE ARRAY
+>Subscript flags returning many values
+>FOUND key=X val=x
+>LOST
+>Subscript flags returning single values
+>STRING key= val=

-- 
Peter Stephenson <pws@xxxxxxx>                  Software Engineer
CSR PLC, Churchill House, Cambridge Business Park, Cowley Road
Cambridge, CB4 0WZ, UK                          Tel: +44 (0)1223 692070