Re: Global alias mangles input in 5.0.8

[Peter Stephenson <p.stephenson@xxxxxxxxxxx>]

On Wed, 1 Jul 2015 21:26:12 +0100
Peter Stephenson <p.w.stephenson@xxxxxxxxxxxx> wrote:
> On Wed, 01 Jul 2015 10:23:57 +0100
> Peter Stephenson <p.stephenson@xxxxxxxxxxx> wrote:
> > On Wed, 1 Jul 2015 00:48:23 -0400
> > fcook <fcook@xxxxxxxxxxxxxx> wrote:
> > > I'm running on x86-64 Arch Linux, and noticing what looks like a bug doing the following:
> > > 
> > > > alias -g '$'='4<<('
> > > 
> > > > cat /proc/self/fd/4 $ ls )
> > > 
> > > But under zsh 5.0.8, I get an error: "zsh: no such file or directory: s )".
> 
> I'm not sure why they don't happen with $(, but the reason is that we
> have started to parse the complete (...) as a string so we can match
> the correct closing parenthesis, and then we encounter the end of the
> alias.  We backtrack because alias expansion occurs after the lexer,
> while we're tracking the string at the level of input --- however, in
> this case the alias was already being expanded when we found the "(",
> so we backtrack too far.  The fix is not to do that.

OK, I've found the mistake.  One of the mistakes.

In this case we shouldn't actually backtrack at all, not only not beyond
the start, because our text is coming from the entire remainder of the
alias.  The previous fix failed in the case shown in the new test.

This one is slightly simpler anyway (single-line changes in three
places, not counting removing the old patch).

pws


diff --git a/Src/input.c b/Src/input.c
index 4a5bf89..613f89a 100644
--- a/Src/input.c
+++ b/Src/input.c
@@ -591,7 +591,7 @@ inpoptop(void)
 	     * history is before, but they're both pushed onto
 	     * the input stack.
 	     */
-	    if ((inbufflags & (INP_ALIAS|INP_HIST)) == INP_ALIAS)
+	    if ((inbufflags & (INP_ALIAS|INP_HIST|INP_RAW_KEEP)) == INP_ALIAS)
 		zshlex_raw_back();
 	}
     }
diff --git a/Src/lex.c b/Src/lex.c
index 910773c..b0cd963 100644
--- a/Src/lex.c
+++ b/Src/lex.c
@@ -162,7 +162,7 @@ static int lex_add_raw;
 
 /* variables associated with the above */
 
-static char *tokstr_raw, *lexbuf_ptr_start;
+static char *tokstr_raw;
 static struct lexbufstate lexbuf_raw;
 
 /* text of punctuation tokens */
@@ -229,13 +229,12 @@ lex_context_save(struct lex_stack *ls, int toplevel)
     ls->lex_add_raw = lex_add_raw;
     ls->tokstr_raw = tokstr_raw;
     ls->lexbuf_raw = lexbuf_raw;
-    ls->lexbuf_ptr_start = lexbuf_ptr_start;
     ls->lexstop = lexstop;
     ls->toklineno = toklineno;
 
     tokstr = zshlextext = lexbuf.ptr = NULL;
     lexbuf.siz = 256;
-    tokstr_raw = lexbuf_raw.ptr = lexbuf_ptr_start = NULL;
+    tokstr_raw = lexbuf_raw.ptr = NULL;
     lexbuf_raw.siz = lexbuf_raw.len = lex_add_raw = 0;
 }
 
@@ -258,7 +257,6 @@ lex_context_restore(const struct lex_stack *ls, int toplevel)
     lex_add_raw = ls->lex_add_raw;
     tokstr_raw = ls->tokstr_raw;
     lexbuf_raw = ls->lexbuf_raw;
-    lexbuf_ptr_start = ls->lexbuf_ptr_start;
     lexstop = ls->lexstop;
     toklineno = ls->toklineno;
 }
@@ -1884,7 +1882,7 @@ zshlex_raw_add(int c)
 void
 zshlex_raw_back(void)
 {
-    if (!lex_add_raw || lexbuf_raw.ptr == lexbuf_ptr_start)
+    if (!lex_add_raw)
 	return;
     lexbuf_raw.ptr--;
     lexbuf_raw.len--;
@@ -1995,7 +1993,7 @@ skipcomm(void)
     cmdpop();
     return lexstop;
 #else
-    char *new_tokstr, *new_lexbuf_ptr_start;
+    char *new_tokstr;
     int new_lexstop, new_lex_add_raw;
     struct lexbufstate new_lexbuf;
 
@@ -2023,6 +2021,18 @@ skipcomm(void)
 	new_tokstr = tokstr;
 	new_lexbuf = lexbuf;
 
+	/*
+	 * If we're expanding an alias at this point, we need the whole
+	 * remaining text as part of the string for the command in
+	 * parentheses, so don't backtrack.  This is different from the
+	 * usual case where the alias is fully within the command, where
+	 * we want the unexpanded text so that it will be expanded
+	 * again when the command in the parentheses is executed.
+	 *
+	 * I never wanted to be a software engineer, you know.
+	 */
+	if (inbufflags & INP_ALIAS)
+	    inbufflags |= INP_RAW_KEEP;
 	zcontext_save_partial(ZCONTEXT_LEX|ZCONTEXT_PARSE);
 	hist_in_word(1);
     } else {
@@ -2042,7 +2052,6 @@ skipcomm(void)
     }
     tokstr_raw = new_tokstr;
     lexbuf_raw = new_lexbuf;
-    lexbuf_ptr_start = lexbuf_raw.ptr;
     lex_add_raw = new_lex_add_raw;
     /*
      * Don't do any ZLE specials down here: they're only needed
@@ -2067,7 +2076,6 @@ skipcomm(void)
      */
     new_tokstr = tokstr_raw;
     new_lexbuf = lexbuf_raw;
-    new_lexbuf_ptr_start = lexbuf_ptr_start;
     /*
      * We're also going to propagate the lexical state:
      * if we couldn't parse the command substitution we
@@ -2083,7 +2091,6 @@ skipcomm(void)
 	 */
 	tokstr_raw = new_tokstr;
 	lexbuf_raw = new_lexbuf;
-	lexbuf_ptr_start = new_lexbuf_ptr_start;
     } else {
 	if (!new_lexstop) {
 	    /* Ignore the ')' added on input */
@@ -2098,7 +2105,6 @@ skipcomm(void)
 	tokstr = new_tokstr;
 	lexbuf = new_lexbuf;
 	lexstop = new_lexstop;
-	lexbuf_ptr_start = (char *)NULL;
 	hist_in_word(0);
     }
 
diff --git a/Src/zsh.h b/Src/zsh.h
index d11d4fe..69fef33 100644
--- a/Src/zsh.h
+++ b/Src/zsh.h
@@ -425,6 +425,7 @@ enum {
 #define INP_HISTCONT  (1<<5)	/* stack is continued from history expn.   */
 #define INP_LINENO    (1<<6)    /* update line number                      */
 #define INP_APPEND    (1<<7)    /* Append new lines to allow backup        */
+#define INP_RAW_KEEP  (1<<8)    /* Input needed in raw mode even if alias  */
 
 /* Flags for metafy */
 #define META_REALLOC	0
@@ -2801,7 +2802,6 @@ struct lex_stack {
     int lex_add_raw;
     char *tokstr_raw;
     struct lexbufstate lexbuf_raw;
-    char *lexbuf_ptr_start;
     int lexstop;
     zlong toklineno;
 };
diff --git a/Test/D03procsubst.ztst b/Test/D03procsubst.ztst
index 07ec639..7b87589 100644
--- a/Test/D03procsubst.ztst
+++ b/Test/D03procsubst.ztst
@@ -121,3 +121,8 @@
   eval 'foo echo this is bound to work)'
 0:backtacking within command string parsing with alias still pending
 >this is bound to work
+
+  alias foo='cat <( print'
+  eval 'foo here is some output)'
+0:full alias expanded when substitution starts in alias
+>here is some output