Re: PATCH: fix 4, was Re: unpatch: metafying zle line

[Peter Stephenson <pws@xxxxxxx>]

Peter Stephenson <pws@xxxxxxx> wrote:
> In fact the first test that fails is the third from last, and it's not
> catastrophic; a stray space appears at the end of the line in
> Y03arguments.ztst test "words array in reset arguments".  There ought to
> be a prize for anyone who tracks that down.  But there isn't.

I win! (Surprised?)  This was a bug in handling of automatic removing of
suffixes.

This turns on ZLE_UNICODE_SUPPORT by default when the environment supports
it.  For now we require full ISO 10646 support (__STDC_ISO_10646__).  I'm
happy to relax that, but it will need a certain amount---probably a
lot---of probing for features in configure to make sure they are there.
Recent Linux distributions definitely work as things stand, and I would
guess therefore BSD-based ones.  It would be good to make a list if anyone
has more information.  (I don't know whether Solaris 9 is currently
supported, for example; I know 8 isn't yet.)

The problems Andrej's been seeing are likely to be mostly specific to real
wide characters: since the completion code uses multibyte characters, with
normal ASCII it sees exactly what it would have with ZLE_UNICODE_SUPPORT
turned off.  So we will be relying a lot on people who need characters not
in ASCII.

Other fixes in this patch: insert-last-word couldn't be repeated because
the line wasn't handled consistently with the stored previous line.  That's
because the last word comes from parsing the command line and hence is a
metafied character string.  Using metafy_line() here is therefore the
simplest fix.

Also, I've added a warning when "bindkey -m" is used, since this will
completely trash use of non-ASCII characters.  Actually, it always did (you
needed ^V which should still work) but I think it's now worth pointing out
to users that bindkey -m doesn't really fit into the brave new world.

Also, any widget that removes completion suffixes is allowed to manipulate
the line directly so should always run with the line unmetafied.

Index: Src/system.h
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/system.h,v
retrieving revision 1.32
diff -u -r1.32 system.h
--- Src/system.h	1 Aug 2005 09:55:00 -0000	1.32
+++ Src/system.h	15 Aug 2005 09:34:48 -0000
@@ -705,7 +705,7 @@
  * between wide characters and multibyte strings.
  */
 #if defined(HAVE_MBRTOWC) && defined(HAVE_WCRTOMB)
-/*#define ZLE_UNICODE_SUPPORT	1*/
+#define ZLE_UNICODE_SUPPORT	1
 #endif
 #else
 # ifdef HAVE_LANGINFO_H
Index: Src/Zle/zle_hist.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle_hist.c,v
retrieving revision 1.26
diff -u -r1.26 zle_hist.c
--- Src/Zle/zle_hist.c	11 Aug 2005 18:40:53 -0000	1.26
+++ Src/Zle/zle_hist.c	15 Aug 2005 09:34:49 -0000
@@ -531,10 +531,11 @@
 	}
     }
 
+    metafy_line();
     if (lastinsert && lastlen &&
-	lastpos <= zlecs &&
-	lastlen == zlecs - lastpos &&
-	memcmp(lastinsert, (char *)&zleline[lastpos], lastlen) == 0)
+	lastpos <= zlemetacs &&
+	lastlen == zlemetacs - lastpos &&
+	memcmp(lastinsert, (char *)&zlemetaline[lastpos], lastlen) == 0)
 	deleteword = 1;
     else
 	lasthist = curhist;
@@ -548,9 +549,9 @@
 	 * confusion.
 	 */
 	if (deleteword) {
-	    int pos = zlecs;
-	    zlecs = lastpos;
-	    foredel(pos - zlecs);
+	    int pos = zlemetacs;
+	    zlemetacs = lastpos;
+	    foredel(pos - zlemetacs);
 	    /*
 	     * Mark that this has been deleted.
 	     * For consistency with history lines, we really ought to
@@ -604,9 +605,9 @@
      * successfully found a new one to insert.
      */
     if (deleteword > 0) {
-	int pos = zlecs;
-	zlecs = lastpos;
-	foredel(pos - zlecs);
+	int pos = zlemetacs;
+	zlemetacs = lastpos;
+	foredel(pos - zlemetacs);
     }
     if (lastinsert) {
 	zfree(lastinsert, lastlen);
@@ -625,13 +626,15 @@
     save = *t;
     *t = '\0';			/* ignore trailing whitespace */
     lasthist = evhist;
-    lastpos = zlecs;
+    lastpos = zlemetacs;
     lastlen = t - s;
     lastinsert = zalloc(t - s);
     memcpy(lastinsert, s, lastlen);
     n = zmult;
     zmult = 1;
 
+    unmetafy_line();
+
     zs = stringaszleline((unsigned char *)s, 0, &len, NULL, NULL);
     doinsert(zs, len);
     free(zs);
Index: Src/Zle/zle_keymap.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle_keymap.c,v
retrieving revision 1.16
diff -u -r1.16 zle_keymap.c
--- Src/Zle/zle_keymap.c	18 Feb 2005 13:57:28 -0000	1.16
+++ Src/Zle/zle_keymap.c	15 Aug 2005 09:34:49 -0000
@@ -818,6 +818,10 @@
 	zwarnnam(name, "keymap `%s' is protected", kmname, 0);
 	return 1;
     }
+#ifdef ZLE_UNICODE_SUPPORT
+    zwarnnam(name, "warning: `bindkey -m' disables multibyte support",
+	     NULL, 0);
+#endif
     for(i = 128; i < 256; i++)
 	if(metabind[i - 128] != z_undefinedkey) {
 	    m[0] = i;
Index: Src/Zle/zle_misc.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle_misc.c,v
retrieving revision 1.24
diff -u -r1.24 zle_misc.c
--- Src/Zle/zle_misc.c	10 Aug 2005 10:56:41 -0000	1.24
+++ Src/Zle/zle_misc.c	15 Aug 2005 09:34:49 -0000
@@ -30,7 +30,7 @@
 #include "zle.mdh"
 #include "zle_misc.pro"
 
-/* insert a metafied string, with repetition and suffix removal */
+/* insert a zle string, with repetition and suffix removal */
 
 /**/
 void
@@ -1107,6 +1107,18 @@
 	    LinkList args = newlinklist();
 	    char buf[20];
 	    int osc = sfcontext;
+	    int wasmeta = (zlemetaline != 0);
+
+	    if (wasmeta) {
+		/*
+		 * The suffix removal function runs as a normal
+		 * ZLE function, not a completion function, so
+		 * the line should be unmetafied if this was
+		 * called from completion.  (It may not be since
+		 * we may decide to remove the suffix later.)
+		 */
+		umetafy_line();
+	    }
 
 	    sprintf(buf, "%d", suffixlen[0]);
 	    addlinknode(args, suffixfunc);
@@ -1118,13 +1130,16 @@
 	    doshfunc(suffixfunc, prog, args, 0, 1);
 	    sfcontext = osc;
 	    endparamscope();
+
+	    if (wasmeta)
+		metafy_line();
 	}
 	zsfree(suffixfunc);
 	suffixfunc = NULL;
     } else {
 #ifdef ZLE_UNICODE_SUPPORT
 	/* TODO: best I can think of for now... */
-	int sl = (unsigned int)c < 256 ? suffixlen[c] : 0;
+	int sl = (unsigned int)c <= 256 ? suffixlen[c] : 0;
 #else
 	int sl = suffixlen[c];
 #endif


-- 
Peter Stephenson <pws@xxxxxxx>                  Software Engineer
CSR PLC, Churchill House, Cambridge Business Park, Cowley Road
Cambridge, CB4 0WZ, UK                          Tel: +44 (0)1223 692070


**********************************************************************
This email and any files transmitted with it are confidential and
intended solely for the use of the individual or entity to whom they
are addressed. If you have received this email in error please notify
the system manager.

**********************************************************************