00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030 #ifndef _UMEM_IMPL_H
00031 #define _UMEM_IMPL_H
00032
00033
00034
00035 #include <umem.h>
00036
00037 #ifdef HAVE_SYS_SYSMACROS_H
00038 #include <sys/sysmacros.h>
00039 #endif
00040
00041 #if HAVE_SYS_TIME_H
00042 #include <sys/time.h>
00043 #endif
00044
00045 #include <sys/vmem.h>
00046 #ifdef HAVE_THREAD_H
00047 # include <thread.h>
00048 #else
00049 # include "sol_compat.h"
00050 #endif
00051
00052 #ifdef __cplusplus
00053 extern "C" {
00054 #endif
00055
00056
00057
00058
00059
00060
00061
00062
00063 #define UMC_QCACHE 0x00100000
00064 #define UMC_INTERNAL 0x80000000
00065
00066
00067
00068
00069 #define UMF_AUDIT 0x00000001
00070 #define UMF_DEADBEEF 0x00000002
00071 #define UMF_REDZONE 0x00000004
00072 #define UMF_CONTENTS 0x00000008
00073 #define UMF_CHECKSIGNAL 0x00000010
00074 #define UMF_NOMAGAZINE 0x00000020
00075 #define UMF_FIREWALL 0x00000040
00076 #define UMF_LITE 0x00000100
00077
00078 #define UMF_HASH 0x00000200
00079 #define UMF_RANDOMIZE 0x00000400
00080
00081 #define UMF_BUFTAG (UMF_DEADBEEF | UMF_REDZONE)
00082 #define UMF_TOUCH (UMF_BUFTAG | UMF_LITE | UMF_CONTENTS)
00083 #define UMF_RANDOM (UMF_TOUCH | UMF_AUDIT | UMF_NOMAGAZINE)
00084 #define UMF_DEBUG (UMF_RANDOM | UMF_FIREWALL)
00085
00086 #define UMEM_STACK_DEPTH umem_stack_depth
00087
00088 #define UMEM_FREE_PATTERN 0xdeadbeefdeadbeefULL
00089 #define UMEM_UNINITIALIZED_PATTERN 0xbaddcafebaddcafeULL
00090 #define UMEM_REDZONE_PATTERN 0xfeedfacefeedfaceULL
00091 #define UMEM_REDZONE_BYTE 0xbb
00092
00093 #define UMEM_FATAL_FLAGS (UMEM_NOFAIL)
00094 #define UMEM_SLEEP_FLAGS (0)
00095
00096
00097
00098
00099
00100
00101 #define UMEM_SIZE_ENCODE(x) (251 * (x) + 1)
00102 #define UMEM_SIZE_DECODE(x) ((x) / 251)
00103 #define UMEM_SIZE_VALID(x) ((x) % 251 == 1)
00104
00105
00106
00107
00108
00109
00110
00111
00112
00113
00114
00115
00116 typedef struct umem_bufctl {
00117 struct umem_bufctl *bc_next;
00118 void *bc_addr;
00119 struct umem_slab *bc_slab;
00120 } umem_bufctl_t;
00121
00122
00123
00124
00125
00126
00127
00128 #define UMEM_BUFCTL_AUDIT_SIZE_DEPTH(frames) \
00129 ((size_t)(&((umem_bufctl_audit_t *)0)->bc_stack[frames]))
00130
00131
00132
00133
00134
00135
00136
00137
00138
00139
00140 #define UMEM_BUFCTL_AUDIT_ALIGN 32
00141
00142 #define UMEM_BUFCTL_AUDIT_MAX_SIZE \
00143 (P2ALIGN((PAGESIZE - sizeof (umem_slab_t))/2 - \
00144 sizeof (umem_buftag_t), UMEM_BUFCTL_AUDIT_ALIGN))
00145
00146 #define UMEM_MAX_STACK_DEPTH \
00147 ((UMEM_BUFCTL_AUDIT_MAX_SIZE - \
00148 UMEM_BUFCTL_AUDIT_SIZE_DEPTH(0)) / sizeof (uintptr_t))
00149
00150 typedef struct umem_bufctl_audit {
00151 struct umem_bufctl *bc_next;
00152 void *bc_addr;
00153 struct umem_slab *bc_slab;
00154 umem_cache_t *bc_cache;
00155 hrtime_t bc_timestamp;
00156 thread_t bc_thread;
00157 struct umem_bufctl *bc_lastlog;
00158 void *bc_contents;
00159 int bc_depth;
00160 uintptr_t bc_stack[1];
00161 } umem_bufctl_audit_t;
00162
00163 #define UMEM_LOCAL_BUFCTL_AUDIT(bcpp) \
00164 *(bcpp) = (umem_bufctl_audit_t *) \
00165 alloca(UMEM_BUFCTL_AUDIT_SIZE)
00166
00167 #define UMEM_BUFCTL_AUDIT_SIZE \
00168 UMEM_BUFCTL_AUDIT_SIZE_DEPTH(UMEM_STACK_DEPTH)
00169
00170
00171
00172
00173
00174 typedef struct umem_buftag {
00175 uint64_t bt_redzone;
00176 umem_bufctl_t *bt_bufctl;
00177 intptr_t bt_bxstat;
00178 } umem_buftag_t;
00179
00180 #define UMEM_BUFTAG(cp, buf) \
00181 ((umem_buftag_t *)((char *)(buf) + (cp)->cache_buftag))
00182
00183 #define UMEM_BUFCTL(cp, buf) \
00184 ((umem_bufctl_t *)((char *)(buf) + (cp)->cache_bufctl))
00185
00186 #define UMEM_BUF(cp, bcp) \
00187 ((void *)((char *)(bcp) - (cp)->cache_bufctl))
00188
00189 #define UMEM_SLAB(cp, buf) \
00190 ((umem_slab_t *)P2END((uintptr_t)(buf), (cp)->cache_slabsize) - 1)
00191
00192 #define UMEM_CPU_CACHE(cp, cpu) \
00193 (umem_cpu_cache_t *)((char *)cp + cpu->cpu_cache_offset)
00194
00195 #define UMEM_MAGAZINE_VALID(cp, mp) \
00196 (((umem_slab_t *)P2END((uintptr_t)(mp), PAGESIZE) - 1)->slab_cache == \
00197 (cp)->cache_magtype->mt_cache)
00198
00199 #define UMEM_SLAB_MEMBER(sp, buf) \
00200 ((size_t)(buf) - (size_t)(sp)->slab_base < \
00201 (sp)->slab_cache->cache_slabsize)
00202
00203 #define UMEM_BUFTAG_ALLOC 0xa110c8edUL
00204 #define UMEM_BUFTAG_FREE 0xf4eef4eeUL
00205
00206 typedef struct umem_slab {
00207 struct umem_cache *slab_cache;
00208 void *slab_base;
00209 struct umem_slab *slab_next;
00210 struct umem_slab *slab_prev;
00211 struct umem_bufctl *slab_head;
00212 long slab_refcnt;
00213 long slab_chunks;
00214 } umem_slab_t;
00215
00216 #define UMEM_HASH_INITIAL 64
00217
00218 #define UMEM_HASH(cp, buf) \
00219 ((cp)->cache_hash_table + \
00220 (((uintptr_t)(buf) >> (cp)->cache_hash_shift) & (cp)->cache_hash_mask))
00221
00222 typedef struct umem_magazine {
00223 void *mag_next;
00224 void *mag_round[1];
00225 } umem_magazine_t;
00226
00227
00228
00229
00230 typedef struct umem_magtype {
00231 int mt_magsize;
00232 int mt_align;
00233 size_t mt_minbuf;
00234 size_t mt_maxbuf;
00235 umem_cache_t *mt_cache;
00236 } umem_magtype_t;
00237
00238 #if (defined(__PTHREAD_MUTEX_SIZE__) && __PTHREAD_MUTEX_SIZE__ >= 24) || defined(UMEM_PTHREAD_MUTEX_TOO_BIG)
00239 #define UMEM_CPU_CACHE_SIZE 128
00240 #else
00241 #define UMEM_CPU_CACHE_SIZE 64
00242 #endif
00243 #define UMEM_CPU_PAD (UMEM_CPU_CACHE_SIZE - sizeof (mutex_t) - \
00244 2 * sizeof (uint_t) - 2 * sizeof (void *) - 4 * sizeof (int))
00245 #define UMEM_CACHE_SIZE(ncpus) \
00246 ((size_t)(&((umem_cache_t *)0)->cache_cpu[ncpus]))
00247
00248 typedef struct umem_cpu_cache {
00249 mutex_t cc_lock;
00250 uint_t cc_alloc;
00251 uint_t cc_free;
00252 umem_magazine_t *cc_loaded;
00253 umem_magazine_t *cc_ploaded;
00254 int cc_rounds;
00255 int cc_prounds;
00256 int cc_magsize;
00257 int cc_flags;
00258 #if (!defined(_LP64) || defined(UMEM_PTHREAD_MUTEX_TOO_BIG)) && !defined(_WIN32)
00259
00260
00261
00262 char cc_pad[UMEM_CPU_PAD];
00263 #endif
00264 } umem_cpu_cache_t;
00265
00266
00267
00268
00269 typedef struct umem_maglist {
00270 umem_magazine_t *ml_list;
00271 long ml_total;
00272 long ml_min;
00273 long ml_reaplimit;
00274 uint64_t ml_alloc;
00275 } umem_maglist_t;
00276
00277 #define UMEM_CACHE_NAMELEN 31
00278
00279 struct umem_cache {
00280
00281
00282
00283 uint64_t cache_slab_create;
00284 uint64_t cache_slab_destroy;
00285 uint64_t cache_slab_alloc;
00286 uint64_t cache_slab_free;
00287 uint64_t cache_alloc_fail;
00288 uint64_t cache_buftotal;
00289 uint64_t cache_bufmax;
00290 uint64_t cache_rescale;
00291 uint64_t cache_lookup_depth;
00292 uint64_t cache_depot_contention;
00293 uint64_t cache_depot_contention_prev;
00294
00295
00296
00297
00298 char cache_name[UMEM_CACHE_NAMELEN + 1];
00299 size_t cache_bufsize;
00300 size_t cache_align;
00301 umem_constructor_t *cache_constructor;
00302 umem_destructor_t *cache_destructor;
00303 umem_reclaim_t *cache_reclaim;
00304 void *cache_private;
00305 vmem_t *cache_arena;
00306 int cache_cflags;
00307 int cache_flags;
00308 int cache_uflags;
00309 uint32_t cache_mtbf;
00310 umem_cache_t *cache_next;
00311 umem_cache_t *cache_prev;
00312 umem_cache_t *cache_unext;
00313 umem_cache_t *cache_uprev;
00314 uint32_t cache_cpu_mask;
00315
00316
00317
00318
00319 mutex_t cache_lock;
00320 size_t cache_chunksize;
00321 size_t cache_slabsize;
00322 size_t cache_bufctl;
00323 size_t cache_buftag;
00324 size_t cache_verify;
00325 size_t cache_contents;
00326 size_t cache_color;
00327 size_t cache_mincolor;
00328 size_t cache_maxcolor;
00329 size_t cache_hash_shift;
00330 size_t cache_hash_mask;
00331 umem_slab_t *cache_freelist;
00332 umem_slab_t cache_nullslab;
00333 umem_cache_t *cache_bufctl_cache;
00334 umem_bufctl_t **cache_hash_table;
00335
00336
00337
00338 mutex_t cache_depot_lock;
00339 umem_magtype_t *cache_magtype;
00340 umem_maglist_t cache_full;
00341 umem_maglist_t cache_empty;
00342
00343
00344
00345
00346 umem_cpu_cache_t cache_cpu[1];
00347 };
00348
00349 typedef struct umem_cpu_log_header {
00350 mutex_t clh_lock;
00351 char *clh_current;
00352 size_t clh_avail;
00353 int clh_chunk;
00354 int clh_hits;
00355 char clh_pad[UMEM_CPU_CACHE_SIZE -
00356 sizeof (mutex_t) - sizeof (char *) -
00357 sizeof (size_t) - 2 * sizeof (int)];
00358 } umem_cpu_log_header_t;
00359
00360 typedef struct umem_log_header {
00361 mutex_t lh_lock;
00362 char *lh_base;
00363 int *lh_free;
00364 size_t lh_chunksize;
00365 int lh_nchunks;
00366 int lh_head;
00367 int lh_tail;
00368 int lh_hits;
00369 umem_cpu_log_header_t lh_cpu[1];
00370 } umem_log_header_t;
00371
00372 typedef struct umem_cpu {
00373 uint32_t cpu_cache_offset;
00374 uint32_t cpu_number;
00375 } umem_cpu_t;
00376
00377 #define UMEM_MAXBUF 16384
00378
00379 #define UMEM_ALIGN 8
00380 #define UMEM_ALIGN_SHIFT 3
00381 #define UMEM_VOID_FRACTION 8
00382
00383
00384
00385
00386 #ifdef _LP64
00387 #define UMEM_SECOND_ALIGN 16
00388 #else
00389 #define UMEM_SECOND_ALIGN UMEM_ALIGN
00390 #endif
00391
00392 #define MALLOC_MAGIC 0x3a10c000
00393 #define MEMALIGN_MAGIC 0x3e3a1000
00394
00395 #ifdef _LP64
00396 #define MALLOC_SECOND_MAGIC 0x16ba7000
00397 #define MALLOC_OVERSIZE_MAGIC 0x06e47000
00398 #endif
00399
00400 #define UMEM_MALLOC_ENCODE(type, sz) (uint32_t)((type) - (sz))
00401 #define UMEM_MALLOC_DECODE(stat, sz) (uint32_t)((stat) + (sz))
00402 #define UMEM_FREE_PATTERN_32 (uint32_t)(UMEM_FREE_PATTERN)
00403
00404 #define UMU_MAGAZINE_RESIZE 0x00000001
00405 #define UMU_HASH_RESCALE 0x00000002
00406 #define UMU_REAP 0x00000004
00407 #define UMU_NOTIFY 0x08000000
00408 #define UMU_ACTIVE 0x80000000
00409
00410 #define UMEM_READY_INIT_FAILED -1
00411 #define UMEM_READY_STARTUP 1
00412 #define UMEM_READY_INITING 2
00413 #define UMEM_READY 3
00414
00415 #ifdef UMEM_STANDALONE
00416 extern void umem_startup(caddr_t, size_t, size_t, caddr_t, caddr_t);
00417 extern int umem_add(caddr_t, size_t);
00418 #endif
00419
00420 #ifdef __cplusplus
00421 }
00422 #endif
00423
00424 #endif