Index: stdlib/malloc.3 =================================================================== --- stdlib/malloc.3 (revision 0) +++ stdlib/malloc.3 (working copy) @@ -0,0 +1,507 @@ +.\" Copyright (c) 1980, 1991, 1993 +.\" The Regents of the University of California. All rights reserved. +.\" +.\" This code is derived from software contributed to Berkeley by +.\" the American National Standards Committee X3, on Information +.\" Processing Systems. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" 3. All advertising materials mentioning features or use of this software +.\" must display the following acknowledgement: +.\" This product includes software developed by the University of +.\" California, Berkeley and its contributors. +.\" 4. Neither the name of the University nor the names of its contributors +.\" may be used to endorse or promote products derived from this software +.\" without specific prior written permission. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" @(#)malloc.3 8.1 (Berkeley) 6/4/93 +.\" $FreeBSD: head/lib/libc/stdlib/malloc.3 152734 2005-11-23 20:34:37Z ru $ +.\" +.Dd August 19, 2004 +.Dt MALLOC 3 +.Os +.Sh NAME +.Nm malloc , calloc , realloc , free , reallocf +.Nd general purpose memory allocation functions +.Sh LIBRARY +.Lb libc +.Sh SYNOPSIS +.In stdlib.h +.Ft void * +.Fn malloc "size_t size" +.Ft void * +.Fn calloc "size_t number" "size_t size" +.Ft void * +.Fn realloc "void *ptr" "size_t size" +.Ft void * +.Fn reallocf "void *ptr" "size_t size" +.Ft void +.Fn free "void *ptr" +.Ft const char * +.Va _malloc_options ; +.Ft void +.Fo \*(lp*_malloc_message\*(rp +.Fa "const char *p1" "const char *p2" "const char *p3" "const char *p4" +.Fc +.Sh DESCRIPTION +The +.Fn malloc +function allocates +.Fa size +bytes of memory. +The allocated space is suitably aligned (after possible pointer coercion) +for storage of any type of object. +If the space is at least +.Va pagesize +bytes in length (see +.Xr getpagesize 3 ) , +the returned memory will be page boundary aligned as well. +If +.Fn malloc +fails, a +.Dv NULL +pointer is returned. +.Pp +Note that +.Fn malloc +does +.Em NOT +normally initialize the returned memory to zero bytes. +.Pp +The +.Fn calloc +function allocates space for +.Fa number +objects, +each +.Fa size +bytes in length. +The result is identical to calling +.Fn malloc +with an argument of +.Dq "number * size" , +with the exception that the allocated memory is explicitly initialized +to zero bytes. +.Pp +The +.Fn realloc +function changes the size of the previously allocated memory referenced by +.Fa ptr +to +.Fa size +bytes. +The contents of the memory are unchanged up to the lesser of the new and +old sizes. +If the new size is larger, +the value of the newly allocated portion of the memory is undefined. +If the requested memory cannot be allocated, +.Dv NULL +is returned and +the memory referenced by +.Fa ptr +is valid and unchanged. +If memory can be allocated, the memory referenced by +.Fa ptr +is freed and a pointer to the newly allocated memory is returned. +Note that +.Fn realloc +and +.Fn reallocf +may move the memory allocation resulting in a different return value than +.Fa ptr . +If +.Fa ptr +is +.Dv NULL , +the +.Fn realloc +function behaves identically to +.Fn malloc +for the specified size. +.Pp +The +.Fn reallocf +function is identical to the +.Fn realloc +function, except that it +will free the passed pointer when the requested memory cannot be allocated. +This is a +.Fx +specific API designed to ease the problems with traditional coding styles +for realloc causing memory leaks in libraries. +.Pp +The +.Fn free +function causes the allocated memory referenced by +.Fa ptr +to be made available for future allocations. +If +.Fa ptr +is +.Dv NULL , +no action occurs. +.Sh TUNING +Once, when the first call is made to one of these memory allocation +routines, various flags will be set or reset, which affect the +workings of this allocation implementation. +.Pp +The ``name'' of the file referenced by the symbolic link named +.Pa /etc/malloc.conf , +the value of the environment variable +.Ev MALLOC_OPTIONS , +and the string pointed to by the global variable +.Va _malloc_options +will be interpreted, in that order, character by character as flags. +.Pp +Most flags are single letters, +where uppercase indicates that the behavior is set, or on, +and lowercase means that the behavior is not set, or off. +.Bl -tag -width indent +.It A +All warnings (except for the warning about unknown +flags being set) become fatal. +The process will call +.Xr abort 3 +in these cases. +.It J +Each byte of new memory allocated by +.Fn malloc , +.Fn realloc +or +.Fn reallocf +as well as all memory returned by +.Fn free , +.Fn realloc +or +.Fn reallocf +will be initialized to 0xd0. +This options also sets the +.Dq R +option. +This is intended for debugging and will impact performance negatively. +.It H +Pass a hint to the kernel about pages unused by the allocation functions. +This will help performance if the system is paging excessively. +This option is off by default. +.It R +Causes the +.Fn realloc +and +.Fn reallocf +functions to always reallocate memory even if the initial allocation was +sufficiently large. +This can substantially aid in compacting memory. +.It U +Generate +.Dq utrace +entries for +.Xr ktrace 1 , +for all operations. +Consult the source for details on this option. +.It V +Attempting to allocate zero bytes will return a +.Dv NULL +pointer instead of +a valid pointer. +(The default behavior is to make a minimal allocation and return a +pointer to it.) +This option is provided for System V compatibility. +This option is incompatible with the +.Dq X +option. +.It X +Rather than return failure for any allocation function, +display a diagnostic message on +.Dv stderr +and cause the program to drop +core (using +.Xr abort 3 ) . +This option should be set at compile time by including the following in +the source code: +.Bd -literal -offset indent +_malloc_options = "X"; +.Ed +.It Z +This option implicitly sets the +.Dq J +and +.Dq R +options, and then zeros out the bytes that were requested. +This is intended for debugging and will impact performance negatively. +.It < +Reduce the size of the cache by a factor of two. +The default cache size is 16 pages. +This option can be specified multiple times. +.It > +Double the size of the cache by a factor of two. +The default cache size is 16 pages. +This option can be specified multiple times. +.El +.Pp +The +.Dq J +and +.Dq Z +options are intended for testing and debugging. +An application which changes its behavior when these options are used +is flawed. +.Sh RETURN VALUES +The +.Fn malloc +and +.Fn calloc +functions return a pointer to the allocated memory if successful; otherwise +a +.Dv NULL +pointer is returned and +.Va errno +is set to +.Er ENOMEM . +.Pp +The +.Fn realloc +and +.Fn reallocf +functions return a pointer, possibly identical to +.Fa ptr , +to the allocated memory +if successful; otherwise a +.Dv NULL +pointer is returned, and +.Va errno +is set to +.Er ENOMEM +if the error was the result of an allocation failure. +The +.Fn realloc +function always leaves the original buffer intact +when an error occurs, whereas +.Fn reallocf +deallocates it in this case. +.Pp +The +.Fn free +function returns no value. +.Sh DEBUGGING MALLOC PROBLEMS +The major difference between this implementation and other allocation +implementations is that the free pages are not accessed unless allocated, +and are aggressively returned to the kernel for reuse. +.Bd -ragged -offset indent +Most allocation implementations will store a data structure containing a +linked list in the free chunks of memory, +used to tie all the free memory together. +That can be suboptimal, +as every time the free-list is traversed, +the otherwise unused, and likely paged out, +pages are faulted into primary memory. +On systems which are paging, +this can result in a factor of five increase in the number of page-faults +done by a process. +.Ed +.Pp +A side effect of this architecture is that many minor transgressions on +the interface which would traditionally not be detected are in fact +detected. +As a result, programs that have been running happily for +years may suddenly start to complain loudly, when linked with this +allocation implementation. +.Pp +The first and most important thing to do is to set the +.Dq A +option. +This option forces a coredump (if possible) at the first sign of trouble, +rather than the normal policy of trying to continue if at all possible. +.Pp +It is probably also a good idea to recompile the program with suitable +options and symbols for debugger support. +.Pp +If the program starts to give unusual results, coredump or generally behave +differently without emitting any of the messages listed in the next +section, it is likely because it depends on the storage being filled with +zero bytes. +Try running it with +.Dq Z +option set; +if that improves the situation, this diagnosis has been confirmed. +If the program still misbehaves, +the likely problem is accessing memory outside the allocated area, +more likely after than before the allocated area. +.Pp +Alternatively, if the symptoms are not easy to reproduce, setting the +.Dq J +option may help provoke the problem. +.Pp +In truly difficult cases, the +.Dq U +option, if supported by the kernel, can provide a detailed trace of +all calls made to these functions. +.Pp +Unfortunately this implementation does not provide much detail about +the problems it detects, the performance impact for storing such information +would be prohibitive. +There are a number of allocation implementations available on the 'Net +which focus on detecting and pinpointing problems by trading performance +for extra sanity checks and detailed diagnostics. +.Sh DIAGNOSTIC MESSAGES +If +.Fn malloc , +.Fn calloc , +.Fn realloc +or +.Fn free +detect an error or warning condition, +a message will be printed to file descriptor STDERR_FILENO. +Errors will result in the process dumping core. +If the +.Dq A +option is set, all warnings are treated as errors. +.Pp +The +.Va _malloc_message +variable allows the programmer to override the function which emits +the text strings forming the errors and warnings if for some reason +the +.Dv stderr +file descriptor is not suitable for this. +Please note that doing anything which tries to allocate memory in +this function will assure death of the process. +.Pp +The following is a brief description of possible error messages and +their meanings: +.Pp +.Bl -diag +.It "(ES): mumble mumble mumble" +The allocation functions were compiled with +.Dq EXTRA_SANITY +defined, and an error was found during the additional error checking. +Consult the source code for further information. +.It "mmap(2) failed, check limits" +This most likely means that the system is dangerously overloaded or that +the process' limits are incorrectly specified. +.It "freelist is destroyed" +The internal free-list has been corrupted. +.It "out of memory" +The +.Dq X +option was specified and an allocation of memory failed. +.El +.Pp +The following is a brief description of possible warning messages and +their meanings: +.Bl -diag +.It "chunk/page is already free" +The process attempted to +.Fn free +memory which had already been freed. +.It "junk pointer, ..." +A pointer specified to one of the allocation functions points outside the +bounds of the memory of which they are aware. +.It "malloc() has never been called" +No memory has been allocated, +yet something is being freed or +realloc'ed. +.It "modified (chunk-/page-) pointer" +The pointer passed to +.Fn free +or +.Fn realloc +has been modified. +.It "pointer to wrong page" +The pointer that +.Fn free , +.Fn realloc , +or +.Fn reallocf +is trying to free does not reference a possible page. +.It "recursive call" +A process has attempted to call an allocation function recursively. +This is not permitted. +In particular, signal handlers should not +attempt to allocate memory. +.It "unknown char in MALLOC_OPTIONS" +An unknown option was specified. +Even with the +.Dq A +option set, this warning is still only a warning. +.El +.Sh ENVIRONMENT +The following environment variables affect the execution of the allocation +functions: +.Bl -tag -width ".Ev MALLOC_OPTIONS" +.It Ev MALLOC_OPTIONS +If the environment variable +.Ev MALLOC_OPTIONS +is set, the characters it contains will be interpreted as flags to the +allocation functions. +.El +.Sh EXAMPLES +To set a systemwide reduction of cache size, and to dump core whenever +a problem occurs: +.Pp +.Bd -literal -offset indent +ln -s 'A<' /etc/malloc.conf +.Ed +.Pp +To specify in the source that a program does no return value checking +on calls to these functions: +.Bd -literal -offset indent +_malloc_options = "X"; +.Ed +.Sh SEE ALSO +.Xr brk 2 , +.Xr mmap 2 , +.Xr alloca 3 , +.Xr getpagesize 3 , +.Xr memory 3 +.Pa /usr/share/doc/papers/malloc.ascii.gz +.Sh STANDARDS +The +.Fn malloc , +.Fn calloc , +.Fn realloc +and +.Fn free +functions conform to +.St -isoC . +.Sh HISTORY +The present allocation implementation started out as a file system for a +drum attached to a 20bit binary challenged computer which was built +with discrete germanium transistors. +It has since graduated to +handle primary storage rather than secondary. +It first appeared in its new shape and ability in +.Fx 2.2 . +.Pp +The +.Fn reallocf +function first appeared in +.Fx 3.0 . +.Sh AUTHORS +.An Poul-Henning Kamp Aq phk@FreeBSD.org +.Sh BUGS +The messages printed in case of problems provide no detail about the +actual values. +.Pp +It can be argued that returning a +.Dv NULL +pointer when asked to +allocate zero bytes is a silly response to a silly question. Index: stdlib/Makefile.inc =================================================================== --- stdlib/Makefile.inc (revision 244355) +++ stdlib/Makefile.inc (working copy) @@ -47,3 +47,10 @@ MLINKS+=strtol.3 strtoll.3 strtol.3 strtoq.3 strtol.3 strtoimax.3 MLINKS+=strtoul.3 strtoull.3 strtoul.3 strtouq.3 strtoul.3 strtoumax.3 MLINKS+=tsearch.3 tdelete.3 tsearch.3 tfind.3 tsearch.3 twalk.3 + +WITH_PHK_MALLOC=yes +.if defined(WITH_PHK_MALLOC) +MISRCS+=malloc.c +SYM_MAPS+=${.CURDIR}/stdlib/jemalloc/Symbol.map +MAN+=malloc.3 +.endif Index: stdlib/malloc.c =================================================================== --- stdlib/malloc.c (revision 0) +++ stdlib/malloc.c (working copy) @@ -0,0 +1,1236 @@ +/* + * ---------------------------------------------------------------------------- + * "THE BEER-WARE LICENSE" (Revision 42): + * wrote this file. As long as you retain this notice you + * can do whatever you want with this stuff. If we meet some day, and you think + * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp + * ---------------------------------------------------------------------------- + * + */ + +#include +__FBSDID("$FreeBSD: head/lib/libc/stdlib/malloc.c 154248 2006-01-12 07:28:21Z jasone $"); + +/* + * Defining MALLOC_EXTRA_SANITY will enable extra checks which are related + * to internal conditions and consistency in malloc.c. This has a + * noticeable runtime performance hit, and generally will not do you + * any good unless you fiddle with the internals of malloc or want + * to catch random pointer corruption as early as possible. + */ +#undef MALLOC_EXTRA_SANITY + +/* + * What to use for Junk. This is the byte value we use to fill with + * when the 'J' option is enabled. + */ +#define SOME_JUNK 0xd0 /* as in "Duh" :-) */ + +/* + * The basic parameters you can tweak. + * + * malloc_pageshift pagesize = 1 << malloc_pageshift + * It's probably best if this is the native + * page size, but it doesn't have to be. + * + * malloc_minsize minimum size of an allocation in bytes. + * If this is too small it's too much work + * to manage them. This is also the smallest + * unit of alignment used for the storage + * returned by malloc/realloc. + * + */ + +#include "namespace.h" +#if defined(__FreeBSD__) +# if defined(__i386__) +# define malloc_pageshift 12U +# define malloc_minsize 16U +# endif +# if defined(__ia64__) +# define malloc_pageshift 13U +# define malloc_minsize 16U +# endif +# if defined(__alpha__) +# define malloc_pageshift 13U +# define malloc_minsize 16U +# endif +# if defined(__sparc64__) +# define malloc_pageshift 13U +# define malloc_minsize 16U +# endif +# if defined(__amd64__) +# define malloc_pageshift 12U +# define malloc_minsize 16U +# endif +# if defined(__arm__) +# define malloc_pageshift 12U +# define malloc_minsize 16U +# endif +# define HAS_UTRACE + /* + * Make malloc/free/realloc thread-safe in libc for use with + * kernel threads. + */ +# include "libc_private.h" +# include "spinlock.h" + static spinlock_t thread_lock = _SPINLOCK_INITIALIZER; + spinlock_t *__malloc_lock = &thread_lock; +# define _MALLOC_LOCK() if (__isthreaded) _SPINLOCK(&thread_lock); +# define _MALLOC_UNLOCK() if (__isthreaded) _SPINUNLOCK(&thread_lock); +#endif /* __FreeBSD__ */ + +#if defined(__sparc__) && defined(sun) +# define malloc_pageshift 12U +# define malloc_minsize 16U +# define MAP_ANON (0) + static int fdzero; +# define MMAP_FD fdzero +# define INIT_MMAP() \ + { if ((fdzero = _open(_PATH_DEVZERO, O_RDWR, 0000)) == -1) \ + wrterror("open of /dev/zero"); } +# define MADV_FREE MADV_DONTNEED +#endif /* __sparc__ */ + +/* Insert your combination here... */ +#if defined(__FOOCPU__) && defined(__BAROS__) +# define malloc_pageshift 12U +# define malloc_minsize 16U +#endif /* __FOOCPU__ && __BAROS__ */ + +#ifndef ZEROSIZEPTR +#define ZEROSIZEPTR ((void *)(uintptr_t)(1 << (malloc_pageshift - 1))) +#endif + +/* + * No user serviceable parts behind this point. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "un-namespace.h" + +/* + * This structure describes a page worth of chunks. + */ + +struct pginfo { + struct pginfo *next; /* next on the free list */ + void *page; /* Pointer to the page */ + u_short size; /* size of this page's chunks */ + u_short shift; /* How far to shift for this size chunks */ + u_short free; /* How many free chunks */ + u_short total; /* How many chunk */ + u_int bits[1]; /* Which chunks are free */ +}; + +/* + * This structure describes a number of free pages. + */ + +struct pgfree { + struct pgfree *next; /* next run of free pages */ + struct pgfree *prev; /* prev run of free pages */ + void *page; /* pointer to free pages */ + void *end; /* pointer to end of free pages */ + size_t size; /* number of bytes free */ +}; + +/* + * How many bits per u_int in the bitmap. + * Change only if not 8 bits/byte + */ +#define MALLOC_BITS (8*sizeof(u_int)) + +/* + * Magic values to put in the page_directory + */ +#define MALLOC_NOT_MINE ((struct pginfo*) 0) +#define MALLOC_FREE ((struct pginfo*) 1) +#define MALLOC_FIRST ((struct pginfo*) 2) +#define MALLOC_FOLLOW ((struct pginfo*) 3) +#define MALLOC_MAGIC ((struct pginfo*) 4) + +#ifndef malloc_pageshift +#define malloc_pageshift 12U +#endif + +#ifndef malloc_minsize +#define malloc_minsize 16U +#endif + +#if !defined(malloc_pagesize) +#define malloc_pagesize (1UL<>1) +#endif + +/* A mask for the offset inside a page. */ +#define malloc_pagemask ((malloc_pagesize)-1) + +#define pageround(foo) (((foo) + (malloc_pagemask))&(~(malloc_pagemask))) +#define ptr2index(foo) (((u_long)(foo) >> malloc_pageshift)-malloc_origo) + +#ifndef _MALLOC_LOCK +#define _MALLOC_LOCK() +#endif + +#ifndef _MALLOC_UNLOCK +#define _MALLOC_UNLOCK() +#endif + +#ifndef MMAP_FD +#define MMAP_FD (-1) +#endif + +#ifndef INIT_MMAP +#define INIT_MMAP() +#endif + +/* Number of free pages we cache */ +static unsigned malloc_cache = 16; + +/* The offset from pagenumber to index into the page directory */ +static u_long malloc_origo; + +/* The last index in the page directory we care about */ +static u_long last_index; + +/* Pointer to page directory. Allocated "as if with" malloc */ +static struct pginfo **page_dir; + +/* How many slots in the page directory */ +static unsigned malloc_ninfo; + +/* Free pages line up here */ +static struct pgfree free_list; + +/* Abort(), user doesn't handle problems. */ +static int malloc_abort = 1; + +/* Are we trying to die ? */ +static int suicide; + +/* always realloc ? */ +static int malloc_realloc; + +#if defined(MADV_FREE) +/* pass the kernel a hint on free pages ? */ +static int malloc_hint = 0; +#endif + +/* xmalloc behaviour ? */ +static int malloc_xmalloc; + +/* sysv behaviour for malloc(0) ? */ +static int malloc_sysv; + +/* zero fill ? */ +static int malloc_zero; + +/* junk fill ? */ +static int malloc_junk = 1; + +#ifdef HAS_UTRACE + +/* utrace ? */ +static int malloc_utrace; + +struct ut { void *p; size_t s; void *r; }; + +void utrace(struct ut *, int); + +#define UTRACE(a, b, c) \ + if (malloc_utrace) \ + {struct ut u; u.p=a; u.s = b; u.r=c; utrace(&u, sizeof u);} +#else /* !HAS_UTRACE */ +#define UTRACE(a,b,c) +#endif /* HAS_UTRACE */ + +/* my last break. */ +static void *malloc_brk; + +/* one location cache for free-list holders */ +static struct pgfree *px; + +/* compile-time options */ +const char *_malloc_options; + +/* Name of the current public function */ +static const char *malloc_func; + +/* Macro for mmap */ +#define MMAP(size) \ + mmap(NULL, (size), PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, \ + MMAP_FD, (off_t)0); + +/* + * Necessary function declarations + */ +static int extend_pgdir(u_long index); +static void *imalloc(size_t size); +static void ifree(void *ptr); +static void *irealloc(void *ptr, size_t size); + +static void +wrtmessage(const char *p1, const char *p2, const char *p3, const char *p4) +{ + + _write(STDERR_FILENO, p1, strlen(p1)); + _write(STDERR_FILENO, p2, strlen(p2)); + _write(STDERR_FILENO, p3, strlen(p3)); + _write(STDERR_FILENO, p4, strlen(p4)); +} + +void (*_malloc_message)(const char *p1, const char *p2, const char *p3, + const char *p4) = wrtmessage; + +static void +wrterror(char const *p) +{ + + suicide = 1; + _malloc_message(_getprogname(), malloc_func, " error: ", p); + abort(); +} + +static void +wrtwarning(char *p) +{ + + /* + * Sensitive processes, somewhat arbitrarily defined here as setuid, + * setgid, root and wheel cannot afford to have malloc mistakes. + */ + if (malloc_abort || issetugid() || getuid() == 0 || getgid() == 0) + wrterror(p); + _malloc_message(_getprogname(), malloc_func, " warning: ", p); +} + +/* + * Allocate a number of pages from the OS + */ +static void * +map_pages(size_t pages) +{ + caddr_t result, tail; + + result = (caddr_t)pageround((u_long)sbrk(0)); + tail = result + (pages << malloc_pageshift); + if (tail < result) + return (NULL); + + if (brk(tail)) { +#ifdef MALLOC_EXTRA_SANITY + wrterror("(ES): map_pages fails\n"); +#endif /* MALLOC_EXTRA_SANITY */ + return (NULL); + } + + last_index = ptr2index(tail) - 1; + malloc_brk = tail; + + if ((last_index+1) >= malloc_ninfo && !extend_pgdir(last_index)) + return (NULL); + + return (result); +} + +/* + * Extend page directory + */ +static int +extend_pgdir(u_long index) +{ + struct pginfo **new, **old; + u_long i, oldlen; + + /* Make it this many pages */ + i = index * sizeof *page_dir; + i /= malloc_pagesize; + i += 2; + + /* remember the old mapping size */ + oldlen = malloc_ninfo * sizeof *page_dir; + + /* + * NOTE: we allocate new pages and copy the directory rather than tempt + * fate by trying to "grow" the region.. There is nothing to prevent + * us from accidently re-mapping space that's been allocated by our caller + * via dlopen() or other mmap(). + * + * The copy problem is not too bad, as there is 4K of page index per + * 4MB of malloc arena. + * + * We can totally avoid the copy if we open a file descriptor to associate + * the anon mappings with. Then, when we remap the pages at the new + * address, the old pages will be "magically" remapped.. But this means + * keeping open a "secret" file descriptor..... + */ + + /* Get new pages */ + new = (struct pginfo**) MMAP(i * malloc_pagesize); + if (new == MAP_FAILED) + return (0); + + /* Copy the old stuff */ + memcpy(new, page_dir, + malloc_ninfo * sizeof *page_dir); + + /* register the new size */ + malloc_ninfo = i * malloc_pagesize / sizeof *page_dir; + + /* swap the pointers */ + old = page_dir; + page_dir = new; + + /* Now free the old stuff */ + munmap(old, oldlen); + return (1); +} + +/* + * Initialize the world + */ +static void +malloc_init(void) +{ + const char *p; + char b[64]; + int i, j; + int save_errno = errno; + + INIT_MMAP(); + +#ifdef MALLOC_EXTRA_SANITY + malloc_junk = 1; +#endif /* MALLOC_EXTRA_SANITY */ + + for (i = 0; i < 3; i++) { + if (i == 0) { + j = readlink("/etc/malloc.conf", b, sizeof b - 1); + if (j <= 0) + continue; + b[j] = '\0'; + p = b; + } else if (i == 1 && issetugid() == 0) { + p = getenv("MALLOC_OPTIONS"); + } else if (i == 1) { + continue; + } else { + p = _malloc_options; + } + for (; p != NULL && *p != '\0'; p++) { + switch (*p) { + case '>': malloc_cache <<= 1; break; + case '<': malloc_cache >>= 1; break; + case 'a': malloc_abort = 0; break; + case 'A': malloc_abort = 1; break; +#if defined(MADV_FREE) + case 'h': malloc_hint = 0; break; + case 'H': malloc_hint = 1; break; +#endif + case 'r': malloc_realloc = 0; break; + case 'R': malloc_realloc = 1; break; + case 'j': malloc_junk = 0; break; + case 'J': malloc_junk = 1; break; +#ifdef HAS_UTRACE + case 'u': malloc_utrace = 0; break; + case 'U': malloc_utrace = 1; break; +#endif + case 'v': malloc_sysv = 0; break; + case 'V': malloc_sysv = 1; break; + case 'x': malloc_xmalloc = 0; break; + case 'X': malloc_xmalloc = 1; break; + case 'z': malloc_zero = 0; break; + case 'Z': malloc_zero = 1; break; + default: + _malloc_message(_getprogname(), malloc_func, + " warning: ", "unknown char in MALLOC_OPTIONS\n"); + break; + } + } + } + + + UTRACE(0, 0, 0); + + /* + * We want junk in the entire allocation, and zero only in the part + * the user asked for. + */ + if (malloc_zero) + malloc_junk=1; + + /* Allocate one page for the page directory */ + page_dir = (struct pginfo **) MMAP(malloc_pagesize); + + if (page_dir == MAP_FAILED) + wrterror("mmap(2) failed, check limits\n"); + + /* + * We need a maximum of malloc_pageshift buckets, steal these from the + * front of the page_directory; + */ + malloc_origo = ((u_long)pageround((u_long)sbrk(0))) >> malloc_pageshift; + malloc_origo -= malloc_pageshift; + + malloc_ninfo = malloc_pagesize / sizeof *page_dir; + + /* Recalculate the cache size in bytes, and make sure it's nonzero */ + + if (!malloc_cache) + malloc_cache++; + + malloc_cache <<= malloc_pageshift; + + /* + * This is a nice hack from Kaleb Keithly (kaleb@x.org). + * We can sbrk(2) further back when we keep this on a low address. + */ + px = (struct pgfree *) imalloc (sizeof *px); + errno = save_errno; +} + +/* + * Allocate a number of complete pages + */ +static void * +malloc_pages(size_t size) +{ + void *p, *delay_free = NULL; + size_t i; + struct pgfree *pf; + u_long index; + + size = pageround(size); + + p = NULL; + + /* Look for free pages before asking for more */ + for(pf = free_list.next; pf; pf = pf->next) { + +#ifdef MALLOC_EXTRA_SANITY + if (pf->size & malloc_pagemask) + wrterror("(ES): junk length entry on free_list\n"); + if (!pf->size) + wrterror("(ES): zero length entry on free_list\n"); + if (pf->page == pf->end) + wrterror("(ES): zero entry on free_list\n"); + if (pf->page > pf->end) + wrterror("(ES): sick entry on free_list\n"); + if ((void*)pf->page >= (void*)sbrk(0)) + wrterror("(ES): entry on free_list past brk\n"); + if (page_dir[ptr2index(pf->page)] != MALLOC_FREE) + wrterror("(ES): non-free first page on free-list\n"); + if (page_dir[ptr2index(pf->end)-1] != MALLOC_FREE) + wrterror("(ES): non-free last page on free-list\n"); +#endif /* MALLOC_EXTRA_SANITY */ + + if (pf->size < size) + continue; + + if (pf->size == size) { + p = pf->page; + if (pf->next != NULL) + pf->next->prev = pf->prev; + pf->prev->next = pf->next; + delay_free = pf; + break; + } + + p = pf->page; + pf->page = (char *)pf->page + size; + pf->size -= size; + break; + } + +#ifdef MALLOC_EXTRA_SANITY + if (p != NULL && page_dir[ptr2index(p)] != MALLOC_FREE) + wrterror("(ES): allocated non-free page on free-list\n"); +#endif /* MALLOC_EXTRA_SANITY */ + + size >>= malloc_pageshift; + + /* Map new pages */ + if (p == NULL) + p = map_pages(size); + + if (p != NULL) { + + index = ptr2index(p); + page_dir[index] = MALLOC_FIRST; + for (i=1;ibits[0] * + (((malloc_pagesize >> bits)+MALLOC_BITS-1) / MALLOC_BITS); + + /* Don't waste more than two chunks on this */ + if ((1<<(bits)) <= l+l) { + bp = (struct pginfo *)pp; + } else { + bp = (struct pginfo *)imalloc(l); + if (bp == NULL) { + ifree(pp); + return (0); + } + } + + bp->size = (1<shift = bits; + bp->total = bp->free = malloc_pagesize >> bits; + bp->page = pp; + + /* set all valid bits in the bitmap */ + k = bp->total; + i = 0; + + /* Do a bunch at a time */ + for(;k-i >= MALLOC_BITS; i += MALLOC_BITS) + bp->bits[i / MALLOC_BITS] = ~0; + + for(; i < k; i++) + bp->bits[i/MALLOC_BITS] |= 1<<(i%MALLOC_BITS); + + if (bp == bp->page) { + /* Mark the ones we stole for ourselves */ + for(i=0;l > 0;i++) { + bp->bits[i/MALLOC_BITS] &= ~(1<<(i%MALLOC_BITS)); + bp->free--; + bp->total--; + l -= (1 << bits); + } + } + + /* MALLOC_LOCK */ + + page_dir[ptr2index(pp)] = bp; + + bp->next = page_dir[bits]; + page_dir[bits] = bp; + + /* MALLOC_UNLOCK */ + + return (1); +} + +/* + * Allocate a fragment + */ +static void * +malloc_bytes(size_t size) +{ + int i,j; + u_int u; + struct pginfo *bp; + int k; + u_int *lp; + + /* Don't bother with anything less than this */ + if (size < malloc_minsize) + size = malloc_minsize; + + /* Find the right bucket */ + j = 1; + i = size-1; + while (i >>= 1) + j++; + + /* If it's empty, make a page more of that size chunks */ + if (page_dir[j] == NULL && !malloc_make_chunks(j)) + return (NULL); + + bp = page_dir[j]; + + /* Find first word of bitmap which isn't empty */ + for (lp = bp->bits; !*lp; lp++) + ; + + /* Find that bit, and tweak it */ + u = 1; + k = 0; + while (!(*lp & u)) { + u += u; + k++; + } + *lp ^= u; + + /* If there are no more free, remove from free-list */ + if (!--bp->free) { + page_dir[j] = bp->next; + bp->next = NULL; + } + + /* Adjust to the real offset of that chunk */ + k += (lp-bp->bits)*MALLOC_BITS; + k <<= bp->shift; + + if (malloc_junk) + memset((u_char *)bp->page + k, SOME_JUNK, bp->size); + + return ((u_char *)bp->page + k); +} + +/* + * Allocate a piece of memory + */ +static void * +imalloc(size_t size) +{ + void *result; + + if (suicide) + abort(); + + if ((size + malloc_pagesize) < size) /* Check for overflow */ + result = NULL; + else if ((size + malloc_pagesize) >= (uintptr_t)page_dir) + result = NULL; + else if (size <= malloc_maxsize) + result = malloc_bytes(size); + else + result = malloc_pages(size); + + if (malloc_zero && result != NULL) + memset(result, 0, size); + + return (result); +} + +/* + * Change the size of an allocation. + */ +static void * +irealloc(void *ptr, size_t size) +{ + void *p; + u_long osize, index; + struct pginfo **mp; + int i; + + if (suicide) + abort(); + + index = ptr2index(ptr); + + if (index < malloc_pageshift) { + wrtwarning("junk pointer, too low to make sense\n"); + return (NULL); + } + + if (index > last_index) { + wrtwarning("junk pointer, too high to make sense\n"); + return (NULL); + } + + mp = &page_dir[index]; + + if (*mp == MALLOC_FIRST) { /* Page allocation */ + + /* Check the pointer */ + if ((u_long)ptr & malloc_pagemask) { + wrtwarning("modified (page-) pointer\n"); + return (NULL); + } + + /* Find the size in bytes */ + for (osize = malloc_pagesize; *(++mp) == MALLOC_FOLLOW;) + osize += malloc_pagesize; + + if (!malloc_realloc && /* Unless we have to, */ + size <= osize && /* .. or are too small, */ + size > (osize - malloc_pagesize)) { /* .. or can free a page, */ + if (malloc_junk) + memset((u_char *)ptr + size, SOME_JUNK, osize-size); + return (ptr); /* ..don't do anything else. */ + } + + } else if (*mp >= MALLOC_MAGIC) { /* Chunk allocation */ + + /* Check the pointer for sane values */ + if (((u_long)ptr & ((*mp)->size-1))) { + wrtwarning("modified (chunk-) pointer\n"); + return (NULL); + } + + /* Find the chunk index in the page */ + i = ((u_long)ptr & malloc_pagemask) >> (*mp)->shift; + + /* Verify that it isn't a free chunk already */ + if ((*mp)->bits[i/MALLOC_BITS] & (1<<(i%MALLOC_BITS))) { + wrtwarning("chunk is already free\n"); + return (NULL); + } + + osize = (*mp)->size; + + if (!malloc_realloc && /* Unless we have to, */ + size <= osize && /* ..or are too small, */ + (size > osize/2 || /* ..or could use a smaller size, */ + osize == malloc_minsize)) { /* ..(if there is one) */ + if (malloc_junk) + memset((u_char *)ptr + size, SOME_JUNK, osize-size); + return (ptr); /* ..don't do anything else. */ + } + + } else { + wrtwarning("pointer to wrong page\n"); + return (NULL); + } + + p = imalloc(size); + + if (p != NULL) { + /* copy the lesser of the two sizes, and free the old one */ + if (!size || !osize) + ; + else if (osize < size) + memcpy(p, ptr, osize); + else + memcpy(p, ptr, size); + ifree(ptr); + } + return (p); +} + +/* + * Free a sequence of pages + */ + +static __inline void +free_pages(void *ptr, u_long index, struct pginfo const *info) +{ + u_long i; + struct pgfree *pf, *pt=NULL; + u_long l; + void *tail; + + if (info == MALLOC_FREE) { + wrtwarning("page is already free\n"); + return; + } + + if (info != MALLOC_FIRST) { + wrtwarning("pointer to wrong page\n"); + return; + } + + if ((u_long)ptr & malloc_pagemask) { + wrtwarning("modified (page-) pointer\n"); + return; + } + + /* Count how many pages and mark them free at the same time */ + page_dir[index] = MALLOC_FREE; + for (i = 1; page_dir[index+i] == MALLOC_FOLLOW; i++) + page_dir[index + i] = MALLOC_FREE; + + l = i << malloc_pageshift; + + if (malloc_junk) + memset(ptr, SOME_JUNK, l); + +#if defined(MADV_FREE) + if (malloc_hint) + madvise(ptr, l, MADV_FREE); +#endif + + tail = (char *)ptr+l; + + /* add to free-list */ + if (px == NULL) + px = imalloc(sizeof *px); /* This cannot fail... */ + px->page = ptr; + px->end = tail; + px->size = l; + if (free_list.next == NULL) { + + /* Nothing on free list, put this at head */ + px->next = free_list.next; + px->prev = &free_list; + free_list.next = px; + pf = px; + px = NULL; + + } else { + + /* Find the right spot, leave pf pointing to the modified entry. */ + tail = (char *)ptr+l; + + for(pf = free_list.next; pf->end < ptr && pf->next != NULL; + pf = pf->next) + ; /* Race ahead here */ + + if (pf->page > tail) { + /* Insert before entry */ + px->next = pf; + px->prev = pf->prev; + pf->prev = px; + px->prev->next = px; + pf = px; + px = NULL; + } else if (pf->end == ptr ) { + /* Append to the previous entry */ + pf->end = (char *)pf->end + l; + pf->size += l; + if (pf->next != NULL && pf->end == pf->next->page ) { + /* And collapse the next too. */ + pt = pf->next; + pf->end = pt->end; + pf->size += pt->size; + pf->next = pt->next; + if (pf->next != NULL) + pf->next->prev = pf; + } + } else if (pf->page == tail) { + /* Prepend to entry */ + pf->size += l; + pf->page = ptr; + } else if (pf->next == NULL) { + /* Append at tail of chain */ + px->next = NULL; + px->prev = pf; + pf->next = px; + pf = px; + px = NULL; + } else { + wrterror("freelist is destroyed\n"); + } + } + + /* Return something to OS ? */ + if (pf->next == NULL && /* If we're the last one, */ + pf->size > malloc_cache && /* ..and the cache is full, */ + pf->end == malloc_brk && /* ..and none behind us, */ + malloc_brk == sbrk(0)) { /* ..and it's OK to do... */ + + /* + * Keep the cache intact. Notice that the '>' above guarantees that + * the pf will always have at least one page afterwards. + */ + pf->end = (char *)pf->page + malloc_cache; + pf->size = malloc_cache; + + brk(pf->end); + malloc_brk = pf->end; + + index = ptr2index(pf->end); + + for(i=index;i <= last_index;) + page_dir[i++] = MALLOC_NOT_MINE; + + last_index = index - 1; + + /* XXX: We could realloc/shrink the pagedir here I guess. */ + } + if (pt != NULL) + ifree(pt); +} + +/* + * Free a chunk, and possibly the page it's on, if the page becomes empty. + */ + +static __inline void +free_bytes(void *ptr, u_long index, struct pginfo *info) +{ + int i; + struct pginfo **mp; + void *vp; + + /* Find the chunk number on the page */ + i = ((u_long)ptr & malloc_pagemask) >> info->shift; + + if (((u_long)ptr & (info->size-1))) { + wrtwarning("modified (chunk-) pointer\n"); + return; + } + + if (info->bits[i/MALLOC_BITS] & (1<<(i%MALLOC_BITS))) { + wrtwarning("chunk is already free\n"); + return; + } + + if (malloc_junk) + memset(ptr, SOME_JUNK, info->size); + + info->bits[i/MALLOC_BITS] |= 1<<(i%MALLOC_BITS); + info->free++; + + mp = page_dir + info->shift; + + if (info->free == 1) { + + /* Page became non-full */ + + mp = page_dir + info->shift; + /* Insert in address order */ + while (*mp && (*mp)->next && (*mp)->next->page < info->page) + mp = &(*mp)->next; + info->next = *mp; + *mp = info; + return; + } + + if (info->free != info->total) + return; + + /* Find & remove this page in the queue */ + while (*mp != info) { + mp = &((*mp)->next); +#ifdef MALLOC_EXTRA_SANITY + if (!*mp) + wrterror("(ES): Not on queue\n"); +#endif /* MALLOC_EXTRA_SANITY */ + } + *mp = info->next; + + /* Free the page & the info structure if need be */ + page_dir[ptr2index(info->page)] = MALLOC_FIRST; + vp = info->page; /* Order is important ! */ + if(vp != (void*)info) + ifree(info); + ifree(vp); +} + +static void +ifree(void *ptr) +{ + struct pginfo *info; + u_long index; + + /* This is legal */ + if (ptr == NULL) + return; + + /* If we're already sinking, don't make matters any worse. */ + if (suicide) + return; + + index = ptr2index(ptr); + + if (index < malloc_pageshift) { + wrtwarning("junk pointer, too low to make sense\n"); + return; + } + + if (index > last_index) { + wrtwarning("junk pointer, too high to make sense\n"); + return; + } + + info = page_dir[index]; + + if (info < MALLOC_MAGIC) + free_pages(ptr, index, info); + else + free_bytes(ptr, index, info); + return; +} + +static void * +pubrealloc(void *ptr, size_t size, const char *func) +{ + void *r; + int err = 0; + static int malloc_active; /* Recusion flag for public interface. */ + static unsigned malloc_started; /* Set when initialization has been done */ + + /* + * If a thread is inside our code with a functional lock held, and then + * catches a signal which calls us again, we would get a deadlock if the + * lock is not of a recursive type. + */ + _MALLOC_LOCK(); + malloc_func = func; + if (malloc_active > 0) { + if (malloc_active == 1) { + wrtwarning("recursive call\n"); + malloc_active = 2; + } + _MALLOC_UNLOCK(); + errno = EDOOFUS; + return (NULL); + } + malloc_active = 1; + + if (!malloc_started) { + if (ptr != NULL) { + wrtwarning("malloc() has never been called\n"); + malloc_active = 0; + _MALLOC_UNLOCK(); + errno = EDOOFUS; + return (NULL); + } + malloc_init(); + malloc_started = 1; + } + + if (ptr == ZEROSIZEPTR) + ptr = NULL; + if (malloc_sysv && !size) { + if (ptr != NULL) + ifree(ptr); + r = NULL; + } else if (!size) { + if (ptr != NULL) + ifree(ptr); + r = ZEROSIZEPTR; + } else if (ptr == NULL) { + r = imalloc(size); + err = (r == NULL); + } else { + r = irealloc(ptr, size); + err = (r == NULL); + } + UTRACE(ptr, size, r); + malloc_active = 0; + _MALLOC_UNLOCK(); + if (malloc_xmalloc && err) + wrterror("out of memory\n"); + if (err) + errno = ENOMEM; + return (r); +} + +/* + * These are the public exported interface routines. + */ + +void * +malloc(size_t size) +{ + + return (pubrealloc(NULL, size, " in malloc():")); +} + +int +posix_memalign(void **memptr, size_t alignment, size_t size) +{ + int err; + void *result; + + /* Make sure that alignment is a large enough power of 2. */ + if (((alignment - 1) & alignment) != 0 || alignment < sizeof(void *)) + return (EINVAL); + + /* + * (size & alignment) is enough to assure the requested alignment, since + * the allocator always allocates power-of-two blocks. + */ + err = errno; /* Protect errno against changes in pubrealloc(). */ + result = pubrealloc(NULL, (size & alignment), " in posix_memalign()"); + errno = err; + + if (result == NULL) + return (ENOMEM); + + *memptr = result; + return (0); +} + +void * +calloc(size_t num, size_t size) +{ + void *ret; + + if (size != 0 && (num * size) / size != num) { + /* size_t overflow. */ + errno = ENOMEM; + return (NULL); + } + + ret = pubrealloc(NULL, num * size, " in calloc():"); + + if (ret != NULL) + memset(ret, 0, num * size); + + return (ret); +} + +void +free(void *ptr) +{ + + pubrealloc(ptr, 0, " in free():"); +} + +void * +realloc(void *ptr, size_t size) +{ + + return (pubrealloc(ptr, size, " in realloc():")); +} + +/* + * Begin library-private functions, used by threading libraries for protection + * of malloc during fork(). These functions are only called if the program is + * running in threaded mode, so there is no need to check whether the program + * is threaded here. + */ + +void +_malloc_prefork(void) +{ + + _spinlock(__malloc_lock); +} + +void +_malloc_postfork(void) +{ + + _spinunlock(__malloc_lock); +} + +void +_malloc_thread_cleanup() +{ + +} Index: gen/tls.c =================================================================== --- gen/tls.c (revision 243068) +++ gen/tls.c (working copy) @@ -39,6 +39,10 @@ #include "libc_private.h" +#define __jemalloc_a0malloc malloc +#define __jemalloc_a0calloc calloc +#define __jemalloc_a0free free + /* Provided by jemalloc to avoid bootstrapping issues. */ void *__jemalloc_a0malloc(size_t size); void *__jemalloc_a0calloc(size_t num, size_t size); Index: Makefile =================================================================== --- Makefile (revision 243068) +++ Makefile (working copy) @@ -85,7 +85,10 @@ .include "${.CURDIR}/resolv/Makefile.inc" .include "${.CURDIR}/stdio/Makefile.inc" .include "${.CURDIR}/stdlib/Makefile.inc" +WITH_PHK_MALLOC=yes +.if !defined(WITH_PHK_MALLOC) .include "${.CURDIR}/stdlib/jemalloc/Makefile.inc" +.endif .include "${.CURDIR}/stdtime/Makefile.inc" .include "${.CURDIR}/string/Makefile.inc" .include "${.CURDIR}/sys/Makefile.inc"