umem_impl.h

Go to the documentation of this file.
00001 /*
00002  * CDDL HEADER START
00003  *
00004  * The contents of this file are subject to the terms of the
00005  * Common Development and Distribution License, Version 1.0 only
00006  * (the "License").  You may not use this file except in compliance
00007  * with the License.
00008  *
00009  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
00010  * or http://www.opensolaris.org/os/licensing.
00011  * See the License for the specific language governing permissions
00012  * and limitations under the License.
00013  *
00014  * When distributing Covered Code, include this CDDL HEADER in each
00015  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
00016  * If applicable, add the following below this CDDL HEADER, with the
00017  * fields enclosed by brackets "[]" replaced with your own identifying
00018  * information: Portions Copyright [yyyy] [name of copyright owner]
00019  *
00020  * CDDL HEADER END
00021  */
00022 /*
00023  * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
00024  * Use is subject to license terms.
00025  */
00026 /*
00027  * Portions Copyright 2006 OmniTI, Inc.
00028  */
00029 
00030 #ifndef _UMEM_IMPL_H
00031 #define _UMEM_IMPL_H
00032 
00033 /* #pragma ident        "@(#)umem_impl.h        1.6     05/06/08 SMI" */
00034 
00035 #include <umem.h>
00036 
00037 #ifdef HAVE_SYS_SYSMACROS_H
00038 #include <sys/sysmacros.h>
00039 #endif
00040 
00041 #if HAVE_SYS_TIME_H
00042 #include <sys/time.h>
00043 #endif
00044 
00045 #include <sys/vmem.h>
00046 #ifdef HAVE_THREAD_H
00047 # include <thread.h>
00048 #else
00049 # include "sol_compat.h"
00050 #endif
00051 
00052 #ifdef  __cplusplus
00053 extern "C" {
00054 #endif
00055 
00056 /*
00057  * umem memory allocator: implementation-private data structures
00058  */
00059 
00060 /*
00061  * Internal flags for umem_cache_create
00062  */
00063 #define UMC_QCACHE      0x00100000
00064 #define UMC_INTERNAL    0x80000000
00065 
00066 /*
00067  * Cache flags
00068  */
00069 #define UMF_AUDIT       0x00000001      /* transaction auditing */
00070 #define UMF_DEADBEEF    0x00000002      /* deadbeef checking */
00071 #define UMF_REDZONE     0x00000004      /* redzone checking */
00072 #define UMF_CONTENTS    0x00000008      /* freed-buffer content logging */
00073 #define UMF_CHECKSIGNAL 0x00000010      /* abort when in signal context */
00074 #define UMF_NOMAGAZINE  0x00000020      /* disable per-cpu magazines */
00075 #define UMF_FIREWALL    0x00000040      /* put all bufs before unmapped pages */
00076 #define UMF_LITE        0x00000100      /* lightweight debugging */
00077 
00078 #define UMF_HASH        0x00000200      /* cache has hash table */
00079 #define UMF_RANDOMIZE   0x00000400      /* randomize other umem_flags */
00080 
00081 #define UMF_BUFTAG      (UMF_DEADBEEF | UMF_REDZONE)
00082 #define UMF_TOUCH       (UMF_BUFTAG | UMF_LITE | UMF_CONTENTS)
00083 #define UMF_RANDOM      (UMF_TOUCH | UMF_AUDIT | UMF_NOMAGAZINE)
00084 #define UMF_DEBUG       (UMF_RANDOM | UMF_FIREWALL)
00085 
00086 #define UMEM_STACK_DEPTH        umem_stack_depth
00087 
00088 #define UMEM_FREE_PATTERN               0xdeadbeefdeadbeefULL
00089 #define UMEM_UNINITIALIZED_PATTERN      0xbaddcafebaddcafeULL
00090 #define UMEM_REDZONE_PATTERN            0xfeedfacefeedfaceULL
00091 #define UMEM_REDZONE_BYTE               0xbb
00092 
00093 #define UMEM_FATAL_FLAGS        (UMEM_NOFAIL)
00094 #define UMEM_SLEEP_FLAGS        (0)
00095 
00096 /*
00097  * Redzone size encodings for umem_alloc() / umem_free().  We encode the
00098  * allocation size, rather than storing it directly, so that umem_free()
00099  * can distinguish frees of the wrong size from redzone violations.
00100  */
00101 #define UMEM_SIZE_ENCODE(x)     (251 * (x) + 1)
00102 #define UMEM_SIZE_DECODE(x)     ((x) / 251)
00103 #define UMEM_SIZE_VALID(x)      ((x) % 251 == 1)
00104 
00105 /*
00106  * The bufctl (buffer control) structure keeps some minimal information
00107  * about each buffer: its address, its slab, and its current linkage,
00108  * which is either on the slab's freelist (if the buffer is free), or
00109  * on the cache's buf-to-bufctl hash table (if the buffer is allocated).
00110  * In the case of non-hashed, or "raw", caches (the common case), only
00111  * the freelist linkage is necessary: the buffer address is at a fixed
00112  * offset from the bufctl address, and the slab is at the end of the page.
00113  *
00114  * NOTE: bc_next must be the first field; raw buffers have linkage only.
00115  */
00116 typedef struct umem_bufctl {
00117         struct umem_bufctl      *bc_next;       /* next bufctl struct */
00118         void                    *bc_addr;       /* address of buffer */
00119         struct umem_slab        *bc_slab;       /* controlling slab */
00120 } umem_bufctl_t;
00121 
00122 /*
00123  * The UMF_AUDIT version of the bufctl structure.  The beginning of this
00124  * structure must be identical to the normal bufctl structure so that
00125  * pointers are interchangeable.
00126  */
00127 
00128 #define UMEM_BUFCTL_AUDIT_SIZE_DEPTH(frames) \
00129         ((size_t)(&((umem_bufctl_audit_t *)0)->bc_stack[frames]))
00130 
00131 /*
00132  * umem_bufctl_audits must be allocated from a UMC_NOHASH cache, so we
00133  * require that 2 of them, plus 2 buftags, plus a umem_slab_t, all fit on
00134  * a single page.
00135  *
00136  * For ILP32, this is about 1000 frames.
00137  * For LP64, this is about 490 frames.
00138  */
00139 
00140 #define UMEM_BUFCTL_AUDIT_ALIGN 32
00141 
00142 #define UMEM_BUFCTL_AUDIT_MAX_SIZE                                      \
00143         (P2ALIGN((PAGESIZE - sizeof (umem_slab_t))/2 -                  \
00144             sizeof (umem_buftag_t), UMEM_BUFCTL_AUDIT_ALIGN))
00145 
00146 #define UMEM_MAX_STACK_DEPTH                                            \
00147         ((UMEM_BUFCTL_AUDIT_MAX_SIZE -                                  \
00148             UMEM_BUFCTL_AUDIT_SIZE_DEPTH(0)) / sizeof (uintptr_t))
00149 
00150 typedef struct umem_bufctl_audit {
00151         struct umem_bufctl      *bc_next;       /* next bufctl struct */
00152         void                    *bc_addr;       /* address of buffer */
00153         struct umem_slab        *bc_slab;       /* controlling slab */
00154         umem_cache_t            *bc_cache;      /* controlling cache */
00155         hrtime_t                bc_timestamp;   /* transaction time */
00156         thread_t                bc_thread;      /* thread doing transaction */
00157         struct umem_bufctl      *bc_lastlog;    /* last log entry */
00158         void                    *bc_contents;   /* contents at last free */
00159         int                     bc_depth;       /* stack depth */
00160         uintptr_t               bc_stack[1];    /* pc stack */
00161 } umem_bufctl_audit_t;
00162 
00163 #define UMEM_LOCAL_BUFCTL_AUDIT(bcpp)                                   \
00164                 *(bcpp) = (umem_bufctl_audit_t *)                       \
00165                     alloca(UMEM_BUFCTL_AUDIT_SIZE)
00166 
00167 #define UMEM_BUFCTL_AUDIT_SIZE                                          \
00168         UMEM_BUFCTL_AUDIT_SIZE_DEPTH(UMEM_STACK_DEPTH)
00169 
00170 /*
00171  * A umem_buftag structure is appended to each buffer whenever any of the
00172  * UMF_BUFTAG flags (UMF_DEADBEEF, UMF_REDZONE, UMF_VERIFY) are set.
00173  */
00174 typedef struct umem_buftag {
00175         uint64_t                bt_redzone;     /* 64-bit redzone pattern */
00176         umem_bufctl_t           *bt_bufctl;     /* bufctl */
00177         intptr_t                bt_bxstat;      /* bufctl ^ (alloc/free) */
00178 } umem_buftag_t;
00179 
00180 #define UMEM_BUFTAG(cp, buf)            \
00181         ((umem_buftag_t *)((char *)(buf) + (cp)->cache_buftag))
00182 
00183 #define UMEM_BUFCTL(cp, buf)            \
00184         ((umem_bufctl_t *)((char *)(buf) + (cp)->cache_bufctl))
00185 
00186 #define UMEM_BUF(cp, bcp)               \
00187         ((void *)((char *)(bcp) - (cp)->cache_bufctl))
00188 
00189 #define UMEM_SLAB(cp, buf)              \
00190         ((umem_slab_t *)P2END((uintptr_t)(buf), (cp)->cache_slabsize) - 1)
00191 
00192 #define UMEM_CPU_CACHE(cp, cpu)         \
00193         (umem_cpu_cache_t *)((char *)cp + cpu->cpu_cache_offset)
00194 
00195 #define UMEM_MAGAZINE_VALID(cp, mp)     \
00196         (((umem_slab_t *)P2END((uintptr_t)(mp), PAGESIZE) - 1)->slab_cache == \
00197             (cp)->cache_magtype->mt_cache)
00198 
00199 #define UMEM_SLAB_MEMBER(sp, buf)       \
00200         ((size_t)(buf) - (size_t)(sp)->slab_base < \
00201             (sp)->slab_cache->cache_slabsize)
00202 
00203 #define UMEM_BUFTAG_ALLOC       0xa110c8edUL
00204 #define UMEM_BUFTAG_FREE        0xf4eef4eeUL
00205 
00206 typedef struct umem_slab {
00207         struct umem_cache       *slab_cache;    /* controlling cache */
00208         void                    *slab_base;     /* base of allocated memory */
00209         struct umem_slab        *slab_next;     /* next slab on freelist */
00210         struct umem_slab        *slab_prev;     /* prev slab on freelist */
00211         struct umem_bufctl      *slab_head;     /* first free buffer */
00212         long                    slab_refcnt;    /* outstanding allocations */
00213         long                    slab_chunks;    /* chunks (bufs) in this slab */
00214 } umem_slab_t;
00215 
00216 #define UMEM_HASH_INITIAL       64
00217 
00218 #define UMEM_HASH(cp, buf)      \
00219         ((cp)->cache_hash_table +       \
00220         (((uintptr_t)(buf) >> (cp)->cache_hash_shift) & (cp)->cache_hash_mask))
00221 
00222 typedef struct umem_magazine {
00223         void    *mag_next;
00224         void    *mag_round[1];          /* one or more rounds */
00225 } umem_magazine_t;
00226 
00227 /*
00228  * The magazine types for fast per-cpu allocation
00229  */
00230 typedef struct umem_magtype {
00231         int             mt_magsize;     /* magazine size (number of rounds) */
00232         int             mt_align;       /* magazine alignment */
00233         size_t          mt_minbuf;      /* all smaller buffers qualify */
00234         size_t          mt_maxbuf;      /* no larger buffers qualify */
00235         umem_cache_t    *mt_cache;      /* magazine cache */
00236 } umem_magtype_t;
00237 
00238 #if (defined(__PTHREAD_MUTEX_SIZE__) && __PTHREAD_MUTEX_SIZE__ >= 24) || defined(UMEM_PTHREAD_MUTEX_TOO_BIG)
00239 #define UMEM_CPU_CACHE_SIZE     128     /* must be power of 2 */
00240 #else
00241 #define UMEM_CPU_CACHE_SIZE     64      /* must be power of 2 */
00242 #endif
00243 #define UMEM_CPU_PAD            (UMEM_CPU_CACHE_SIZE - sizeof (mutex_t) - \
00244         2 * sizeof (uint_t) - 2 * sizeof (void *) - 4 * sizeof (int))
00245 #define UMEM_CACHE_SIZE(ncpus)  \
00246         ((size_t)(&((umem_cache_t *)0)->cache_cpu[ncpus]))
00247 
00248 typedef struct umem_cpu_cache {
00249         mutex_t         cc_lock;        /* protects this cpu's local cache */
00250         uint_t          cc_alloc;       /* allocations from this cpu */
00251         uint_t          cc_free;        /* frees to this cpu */
00252         umem_magazine_t *cc_loaded;     /* the currently loaded magazine */
00253         umem_magazine_t *cc_ploaded;    /* the previously loaded magazine */
00254         int             cc_rounds;      /* number of objects in loaded mag */
00255         int             cc_prounds;     /* number of objects in previous mag */
00256         int             cc_magsize;     /* number of rounds in a full mag */
00257         int             cc_flags;       /* CPU-local copy of cache_flags */
00258 #if (!defined(_LP64) || defined(UMEM_PTHREAD_MUTEX_TOO_BIG)) && !defined(_WIN32)
00259         /* on win32, UMEM_CPU_PAD evaluates to zero, and the MS compiler
00260          * won't allow static initialization of arrays containing structures
00261          * that contain zero size arrays */
00262         char            cc_pad[UMEM_CPU_PAD]; /* for nice alignment (32-bit) */
00263 #endif
00264 } umem_cpu_cache_t;
00265 
00266 /*
00267  * The magazine lists used in the depot.
00268  */
00269 typedef struct umem_maglist {
00270         umem_magazine_t *ml_list;       /* magazine list */
00271         long            ml_total;       /* number of magazines */
00272         long            ml_min;         /* min since last update */
00273         long            ml_reaplimit;   /* max reapable magazines */
00274         uint64_t        ml_alloc;       /* allocations from this list */
00275 } umem_maglist_t;
00276 
00277 #define UMEM_CACHE_NAMELEN      31
00278 
00279 struct umem_cache {
00280         /*
00281          * Statistics
00282          */
00283         uint64_t        cache_slab_create;      /* slab creates */
00284         uint64_t        cache_slab_destroy;     /* slab destroys */
00285         uint64_t        cache_slab_alloc;       /* slab layer allocations */
00286         uint64_t        cache_slab_free;        /* slab layer frees */
00287         uint64_t        cache_alloc_fail;       /* total failed allocations */
00288         uint64_t        cache_buftotal;         /* total buffers */
00289         uint64_t        cache_bufmax;           /* max buffers ever */
00290         uint64_t        cache_rescale;          /* # of hash table rescales */
00291         uint64_t        cache_lookup_depth;     /* hash lookup depth */
00292         uint64_t        cache_depot_contention; /* mutex contention count */
00293         uint64_t        cache_depot_contention_prev; /* previous snapshot */
00294 
00295         /*
00296          * Cache properties
00297          */
00298         char            cache_name[UMEM_CACHE_NAMELEN + 1];
00299         size_t          cache_bufsize;          /* object size */
00300         size_t          cache_align;            /* object alignment */
00301         umem_constructor_t *cache_constructor;
00302         umem_destructor_t *cache_destructor;
00303         umem_reclaim_t  *cache_reclaim;
00304         void            *cache_private;         /* opaque arg to callbacks */
00305         vmem_t          *cache_arena;           /* vmem source for slabs */
00306         int             cache_cflags;           /* cache creation flags */
00307         int             cache_flags;            /* various cache state info */
00308         int             cache_uflags;           /* UMU_* flags */
00309         uint32_t        cache_mtbf;             /* induced alloc failure rate */
00310         umem_cache_t    *cache_next;            /* forward cache linkage */
00311         umem_cache_t    *cache_prev;            /* backward cache linkage */
00312         umem_cache_t    *cache_unext;           /* next in update list */
00313         umem_cache_t    *cache_uprev;           /* prev in update list */
00314         uint32_t        cache_cpu_mask;         /* mask for cpu offset */
00315 
00316         /*
00317          * Slab layer
00318          */
00319         mutex_t         cache_lock;             /* protects slab layer */
00320         size_t          cache_chunksize;        /* buf + alignment [+ debug] */
00321         size_t          cache_slabsize;         /* size of a slab */
00322         size_t          cache_bufctl;           /* buf-to-bufctl distance */
00323         size_t          cache_buftag;           /* buf-to-buftag distance */
00324         size_t          cache_verify;           /* bytes to verify */
00325         size_t          cache_contents;         /* bytes of saved content */
00326         size_t          cache_color;            /* next slab color */
00327         size_t          cache_mincolor;         /* maximum slab color */
00328         size_t          cache_maxcolor;         /* maximum slab color */
00329         size_t          cache_hash_shift;       /* get to interesting bits */
00330         size_t          cache_hash_mask;        /* hash table mask */
00331         umem_slab_t     *cache_freelist;        /* slab free list */
00332         umem_slab_t     cache_nullslab;         /* end of freelist marker */
00333         umem_cache_t    *cache_bufctl_cache;    /* source of bufctls */
00334         umem_bufctl_t   **cache_hash_table;     /* hash table base */
00335         /*
00336          * Depot layer
00337          */
00338         mutex_t         cache_depot_lock;       /* protects depot */
00339         umem_magtype_t  *cache_magtype;         /* magazine type */
00340         umem_maglist_t  cache_full;             /* full magazines */
00341         umem_maglist_t  cache_empty;            /* empty magazines */
00342 
00343         /*
00344          * Per-CPU layer
00345          */
00346         umem_cpu_cache_t cache_cpu[1];          /* cache_cpu_mask + 1 entries */
00347 };
00348 
00349 typedef struct umem_cpu_log_header {
00350         mutex_t         clh_lock;
00351         char            *clh_current;
00352         size_t          clh_avail;
00353         int             clh_chunk;
00354         int             clh_hits;
00355         char            clh_pad[UMEM_CPU_CACHE_SIZE -
00356                                 sizeof (mutex_t) - sizeof (char *) -
00357                                 sizeof (size_t) - 2 * sizeof (int)];
00358 } umem_cpu_log_header_t;
00359 
00360 typedef struct umem_log_header {
00361         mutex_t         lh_lock;
00362         char            *lh_base;
00363         int             *lh_free;
00364         size_t          lh_chunksize;
00365         int             lh_nchunks;
00366         int             lh_head;
00367         int             lh_tail;
00368         int             lh_hits;
00369         umem_cpu_log_header_t lh_cpu[1];        /* actually umem_max_ncpus */
00370 } umem_log_header_t;
00371 
00372 typedef struct umem_cpu {
00373         uint32_t cpu_cache_offset;
00374         uint32_t cpu_number;
00375 } umem_cpu_t;
00376 
00377 #define UMEM_MAXBUF     16384
00378 
00379 #define UMEM_ALIGN              8       /* min guaranteed alignment */
00380 #define UMEM_ALIGN_SHIFT        3       /* log2(UMEM_ALIGN) */
00381 #define UMEM_VOID_FRACTION      8       /* never waste more than 1/8 of slab */
00382 
00383 /*
00384  * For 64 bits, buffers >= 16 bytes must be 16-byte aligned
00385  */
00386 #ifdef _LP64
00387 #define UMEM_SECOND_ALIGN 16
00388 #else
00389 #define UMEM_SECOND_ALIGN UMEM_ALIGN
00390 #endif
00391 
00392 #define MALLOC_MAGIC                    0x3a10c000 /* 8-byte tag */
00393 #define MEMALIGN_MAGIC                  0x3e3a1000
00394 
00395 #ifdef _LP64
00396 #define MALLOC_SECOND_MAGIC             0x16ba7000 /* 8-byte tag, 16-aligned */
00397 #define MALLOC_OVERSIZE_MAGIC           0x06e47000 /* 16-byte tag, _LP64 */
00398 #endif
00399 
00400 #define UMEM_MALLOC_ENCODE(type, sz)    (uint32_t)((type) - (sz))
00401 #define UMEM_MALLOC_DECODE(stat, sz)    (uint32_t)((stat) + (sz))
00402 #define UMEM_FREE_PATTERN_32            (uint32_t)(UMEM_FREE_PATTERN)
00403 
00404 #define UMU_MAGAZINE_RESIZE     0x00000001
00405 #define UMU_HASH_RESCALE        0x00000002
00406 #define UMU_REAP                0x00000004
00407 #define UMU_NOTIFY              0x08000000
00408 #define UMU_ACTIVE              0x80000000
00409 
00410 #define UMEM_READY_INIT_FAILED          -1
00411 #define UMEM_READY_STARTUP              1
00412 #define UMEM_READY_INITING              2
00413 #define UMEM_READY                      3
00414 
00415 #ifdef UMEM_STANDALONE
00416 extern void umem_startup(caddr_t, size_t, size_t, caddr_t, caddr_t);
00417 extern int umem_add(caddr_t, size_t);
00418 #endif
00419 
00420 #ifdef  __cplusplus
00421 }
00422 #endif
00423 
00424 #endif  /* _UMEM_IMPL_H */

Generated on Thu Feb 7 05:40:23 2008 for umem by  doxygen 1.5.4