Re: zsh and memory

[Sven Wischnowsky <wischnow@xxxxxxxxxxxxxxxxxxxxxxx>]

[ Also noticed by Mike... ]

Timothy Writer wrote:

> Sven Wischnowsky <wischnow@xxxxxxxxxxxxxxxxxxxxxxx> writes:
> 
> > I wrote:
> > 
> > > I'm really tempted to allocate heaps using mmap()
> > > (anonymous) to get them out of the way of the zalloc() allocator. I
> > > small test showed that with this I only get 39KB of free memory after
> > > the completion test, which is really not too bad. I don't have a patch 
> > > for that yet, though.
> > 
> > ... and now I don't think I'll ever write one: neither Solaris nor
> > Linux seem to have MAP_ANONYMOUS. Sniff.
> 
> Hmmm ... how are you calling mmap exactly?  I've definitely used
> MAP_ANONYMOUS on Linux (as far back as kernel 1.3) and I think I've used it
> on Solaris too.  Linux doesn't have shared anonymous mappings, i.e. you have
> to use MAP_PRIVATE with MAP_ANONYMOUS.

I had only looked at the manual and there MAP_ANONYMOUS doesn't appear 
in the list -- only in a short comment near the end.

For Solaris, however: the 5.6 box I have access to here really doesn't 
have a MAP_ANONYMOUS in the header file.

Anyway, below is the patch to make zsh heaps allocated using
mmap(). At least for me this keeps the amount of unused space on the
real heap nicely small. I don't see any disadvantages with this.


BUT, of course, the real problem is the way we store shell code, which 
eats a lot of space. After the next official release I'd like to try
to write a byte-code machine for zsh; that should also make things
faster... I don't know if I'll find enough free time, though.


Bye
 Sven

diff -ru ../z.old/Src/mem.c Src/mem.c
--- ../z.old/Src/mem.c	Fri Dec 17 11:40:35 1999
+++ Src/mem.c	Fri Dec 17 11:43:04 1999
@@ -64,15 +64,29 @@
 	to duplicate a structure in order to preserve it (i.e. a function
 	definition), call permalloc(), then dupstruct().
 
-	If we use zsh's own allocator we use a simple trick to avoid that
-	the (*real*) heap fills up with empty zsh-heaps: we allocate a
-	large block of memory before allocating a heap pool, this memory
-	is freed again immediately after the pool is allocated. If there
-	are only small blocks on the free list this guarantees that the
-	memory for the pool is at the end of the memory which means that
-	we can give it back to the system when the pool is freed.
+	If possible, the heaps are allocated using mmap() so that the
+	(*real*) heap isn't filled up with empty zsh heaps. If mmap()
+	is not available and zsh's own allocator we use a simple trick
+	to avoid that: we allocate a large block of memory before allocating
+	a heap pool, this memory is freed again immediately after the pool
+	is allocated. If there are only small blocks on the free list this
+	guarantees that the memory for the pool is at the end of the memory
+	which means that we can give it back to the system when the pool is
+	freed.
 */
 
+#if defined(HAVE_SYS_MMAN_H) && defined(HAVE_MMAP) && defined(HAVE_MUNMAP)
+
+#include <sys/mman.h>
+
+#if defined(MAP_ANONYMOUS) && defined(MAP_PRIVATE)
+
+#define USE_MMAP 1
+#define MMAP_FLAGS (MAP_ANONYMOUS | MAP_PRIVATE)
+
+#endif
+#endif
+
 /* != 0 if we are allocating in the heaplist */
  
 /**/
@@ -232,8 +246,13 @@
 	    if (!fheap && h->used < HEAP_ARENA_SIZE)
 		fheap = h;
 	    hl = h;
-	} else
+	} else {
+#ifdef USE_MMAP
+	    munmap((void *) h, h->size);
+#else
 	    zfree(h, HEAPSIZE);
+#endif
+	}
     }
     if (hl)
 	hl->next = NULL;
@@ -268,8 +287,13 @@
 	    zfree(hs, sizeof(*hs));
 
 	    hl = h;
-	} else
+	} else {
+#ifdef USE_MMAP
+	    munmap((void *) h, h->size);
+#else
 	    zfree(h, HEAPSIZE);
+#endif
+	}
     }
     if (hl)
 	hl->next = NULL;
@@ -303,7 +327,7 @@
     {
 	Heap hp;
         /* not found, allocate new heap */
-#ifdef ZSH_MEM
+#if defined(ZSH_MEM) && !defined(USE_MMAP)
 	static int called = 0;
 	void *foo = called ? (void *)malloc(HEAPFREE) : NULL;
             /* tricky, see above */
@@ -313,9 +337,34 @@
 	n = HEAP_ARENA_SIZE > size ? HEAPSIZE : size + sizeof(*h);
 	for (hp = NULL, h = heaps; h; hp = h, h = h->next);
 
+#ifdef USE_MMAP
+	{
+	    static size_t pgsz = 0;
+
+	    if (!pgsz) {
+
+#ifdef _SC_PAGESIZE
+		pgsz = sysconf(_SC_PAGESIZE);     /* SVR4 */
+#else
+# ifdef _SC_PAGE_SIZE
+		pgsz = sysconf(_SC_PAGE_SIZE);    /* HPUX */
+# else
+		pgsz = getpagesize();
+# endif
+#endif
+
+		pgsz--;
+	    }
+	    n = (n + pgsz) & ~pgsz;
+	    h = (Heap) mmap(NULL, n, PROT_READ | PROT_WRITE,
+			    MMAP_FLAGS, -1, 0);
+	    h->size = n;
+	}
+#else
 	h = (Heap) zalloc(n);
+#endif
 
-#ifdef ZSH_MEM
+#if defined(ZSH_MEM) && !defined(USE_MMAP)
 	if (called)
 	    zfree(foo, HEAPFREE);
 	called = 1;
@@ -381,7 +430,11 @@
 	    else
 		heaps = h->next;
 	    fheap = NULL;
+#ifdef USE_MMAP
+	    munmap((void *) h, h->size);
+#else
 	    zfree(h, HEAPSIZE);
+#endif
 	    return NULL;
 	}
 	if (old > HEAP_ARENA_SIZE || new > HEAP_ARENA_SIZE) {
@@ -696,7 +749,9 @@
 {
     struct m_hdr *m, *mp, *mt;
     long n, s, os = 0;
+#ifndef USE_MMAP
     struct heap *h, *hp, *hf = NULL, *hfp = NULL;
+#endif
 
     /* some systems want malloc to return the highest valid address plus one
        if it is called with an argument of zero */
@@ -769,12 +824,14 @@
     } else
 	s = 0;
 
-/* search the free list for an block of at least the requested size */
+    /* search the free list for an block of at least the requested size */
     for (mp = NULL, m = m_free; m && m->len < size; mp = m, m = m->next);
 
- /* if there is an empty zsh heap at a lower address we steal it and take
-    the memory from it, putting the rest on the free list (remember
-    that the blocks on the free list are ordered) */
+#ifndef USE_MMAP
+
+    /* if there is an empty zsh heap at a lower address we steal it and take
+       the memory from it, putting the rest on the free list (remember
+       that the blocks on the free list are ordered) */
 
     for (hp = NULL, h = heaps; h; hp = h, h = h->next)
 	if (!h->used &&
@@ -801,6 +858,7 @@
 
 	for (mp = NULL, m = m_free; m && m->len < size; mp = m, m = m->next);
     }
+#endif
     if (!m) {
 	long nal;
 	/* no matching free block was found, we have to request new
diff -ru ../z.old/Src/zsh.h Src/zsh.h
--- ../z.old/Src/zsh.h	Fri Dec 17 11:40:37 1999
+++ Src/zsh.h	Fri Dec 17 11:43:04 1999
@@ -1556,6 +1556,7 @@
 
 struct heap {
     struct heap *next;		/* next one                                  */
+    size_t size;		/* size of heap                              */
     size_t used;		/* bytes used from the heap                  */
     struct heapstack *sp;	/* used by pushheap() to save the value used */
 #ifdef PAD_64_BIT

--
Sven Wischnowsky                         wischnow@xxxxxxxxxxxxxxxxxxxxxxx