kernel memory leak?
2007-12-25 2:21:00
Sun e6500 (14x CPU, 14GB RAM) as NFS client. Under Solaris 10 b42
free memory drops to about 2GB and most of it is used by kernel (under
Solaris 8 free memory is about 13GB). Memory drops to about 2GB free after
about 1 day working. As you can see it's not cache... Solaris 8 and 10
uses the same system parametes (/etc/system) - so, there must be some
changes to VM or a bug?
kmem_va_8192 1GB
kmem_alloc_8192 4GB
kmem_va (kmem_default) 9GB
Where is all memory?
Some info from system:
bash-2.05# cat /etc/system|egrep -v "^\*|^$"
set hme:hme_adv_100fdx_cap=1
set hme:hme_adv_100hdx_cap=0
set hme:hme_adv_100T4_cap=0
set hme:hme_adv_10fdx_cap=0
set hme:hme_adv_10hdx_cap=0
set hme:hme_adv_autoneg_cap=0
set ecache_scrub_enable=1
set ecache_scan_rate=1000
set ecache_calls_a_sec=100
set tcp:tcp_conn_hash_size=65536
set nfs:nfs3_max_threads=16
set nfs:nrnode=1048576
set nfs:nfs3_async_clusters=32
set nfs:nfs3_dynamic=1
set nstrpush=90
set nfs:nfs_allow_preepoch_time=1
rootdev:/pseudo/md at 0:0,0,blk
bash-2.05#
bash-2.05# vmstat 1
kthr memory page disk faults cpu
r b w swap free re mf pi po fr de sr m0 m1 m2 m5 in sy cs us sy id
3 0 0 10917408 2778408 1018 4674 4104 1 1 0 0 1 1 1 0 9122 25401 17542 9 45 46
0 0 0 10434240 2296208 1547 8171 6549 0 0 0 0 0 0 0 0 14426 47964 29133 15 66 18
bash-2.05# mdb -k
Loading modules: [ unix krtld genunix sd isp ufs_log md ip nfs random ptm ]
> ::memstat
Page Summary Pages MB %Tot
------------ ---------------- ---------------- ----
Kernel 1255856 9811 71%
Anon 25831 201 1%
Exec and libs 1447 11 0%
Page cache 193309 1510 11%
Free (cachelist) 279274 2181 16%
Free (freelist) 10200 79 1%
Total 1765917 13796
> ::kmastat
cache buf buf buf memory alloc alloc
name size in use total in use succeed fail
------------------------- ------ ------ ------ --------- --------- -----
kmem_magazine_1 16 3416 6604 106496 7937 0
kmem_magazine_3 32 6763 7366 237568 12575 0
kmem_magazine_7 64 10851 40386 2605056 51615 0
kmem_magazine_15 128 48252 49077 6381568 64624 0
kmem_magazine_31 256 3781 6138 1622016 11023 0
kmem_magazine_47 384 14568 14574 5685248 15543 0
kmem_magazine_63 512 1060 1260 688128 1631 0
kmem_magazine_95 768 0 500 409600 828 0
kmem_magazine_143 1152 547 553 647168 547 0
kmem_slab_cache 56 1092534 1092575 61726720 1098448 0
kmem_bufctl_cache 24 2310008 2310285 55828480 2323677 0
kmem_bufctl_audit_cache 128 0 0 0 0 0
kmem_va_8192 8192 1189541 1189568 1155006464 1199745 0
kmem_va_16384 16384 352 352 5767168 352 0
kmem_va_24576 24576 109 150 3932160 351 0
kmem_va_32768 32768 0 0 0 0 0
kmem_va_40960 40960 13 18 786432 13 0
kmem_va_49152 49152 0 0 0 0 0
kmem_va_57344 57344 14 16 1048576 14 0
kmem_va_65536 65536 0 0 0 0 0
kmem_alloc_8 8 12876 23391 188416 2890810584 0
kmem_alloc_16 16 8599 17272 278528 2367770336 0
kmem_alloc_24 24 9729 25086 606208 1678876282 0
kmem_alloc_32 32 7677 17272 557056 924888445 0
kmem_alloc_40 40 3007 5684 229376 245744750 0
kmem_alloc_48 48 80112 109850 5324800 1358712396 0
kmem_alloc_56 56 19968 30015 1695744 662355496 0
kmem_alloc_64 64 4075 8636 557056 1758107074 0
kmem_alloc_80 80 43165 50298 4079616 2079494166 0
kmem_alloc_96 96 717073 1041180 101539840 778127035 0
kmem_alloc_112 112 60 216 24576 2109265 0
kmem_alloc_128 128 2236 6489 843776 1339267766 0
kmem_alloc_160 160 1179 2450 401408 149665025 0
kmem_alloc_192 192 1005 2436 475136 576666329 0
kmem_alloc_224 224 1346 2772 630784 581271394 0
kmem_alloc_256 256 846 2325 614400 522588977 0
kmem_alloc_320 320 1539 1850 606208 47658388 0
kmem_alloc_384 384 31 105 40960 3379694 0
kmem_alloc_448 448 2204 2268 1032192 50858764 0
kmem_alloc_512 512 917 2775 1515520 61136536 0
kmem_alloc_640 640 17 84 57344 46272 0
kmem_alloc_768 768 14 100 81920 59796 0
kmem_alloc_896 896 6 72 65536 67333 0
kmem_alloc_1152 1152 233 462 540672 2998888840 0
kmem_alloc_1344 1344 11 78 106496 87278 0
kmem_alloc_1600 1600 6 70 114688 762246 0
kmem_alloc_2048 2048 722 1788 3661824 49010591 0
kmem_alloc_2688 2688 66 318 868352 267748969 0
kmem_alloc_4096 4096 47 76 311296 5106821 0
kmem_alloc_8192 8192 718523 1041702 4238655488 320245252 0
kmem_alloc_12288 12288 20 40 491520 885 0
kmem_alloc_16384 16384 142 165 2703360 28707 0
streams_mblk 64 93826 97409 6283264 1688505285 0
streams_dblk_16 128 867 2394 311296 1304587681 0
streams_dblk_80 192 4405 18984 3702784 1471631664 0
streams_dblk_144 256 108 1767 466944 1441139044 0
streams_dblk_208 320 269 3575 1171456 2732449793 0
streams_dblk_272 384 16 252 98304 157406004 0
streams_dblk_336 448 0 324 147456 155881688 0
streams_dblk_528 640 2 132 90112 77780440 0
streams_dblk_1040 1152 8279 50344 58916864 78015942 0
streams_dblk_1488 1600 157 680 1114112 138739493 0
streams_dblk_1936 2048 4132 10028 20537344 2550095429 0
streams_dblk_2576 2688 1951 3951 10788864 1651812065 0
streams_dblk_3920 4032 1 28 114688 810088 0
streams_dblk_8192 112 0 189 24576 13335974 0
streams_dblk_12112 12224 1 178 2187264 164828753 0
streams_dblk_16384 112 0 189 24576 287649 0
streams_dblk_20304 20416 0 26 532480 65089 0
streams_dblk_24576 112 1 189 24576 929946 0
streams_dblk_28496 28608 0 28 802816 726125 0
streams_dblk_32768 112 0 189 24576 286719 0
streams_dblk_36688 36800 0 26 958464 8627 0
streams_dblk_40960 112 0 189 24576 509792 0
streams_dblk_44880 44992 0 24 1081344 4740 0
streams_dblk_49152 112 0 126 16384 8084 0
streams_dblk_53072 53184 0 24 1277952 5410 0
streams_dblk_57344 112 0 126 16384 11818 0
streams_dblk_61264 61376 0 26 1597440 13296 0
streams_dblk_65536 112 0 126 16384 8131 0
streams_dblk_69456 69568 0 24 1671168 3045 0
streams_dblk_73728 112 0 126 16384 4274 0
streams_dblk_esb 112 0 189 24576 78027 0
streams_fthdr 264 0 0 0 0 0
streams_ftblk 232 0 0 0 0 0
multidata 120 0 0 0 0 0
multidata_pdslab 1472 0 0 0 0 0
multidata_pattbl 32 0 0 0 0 0
taskq_ent_cache 56 997 1305 73728 10466 0
taskq_cache 192 43 84 16384 47 0
id32_cache 32 254 254 8192 258 0
bp_map_16384 16384 4 320 5242880 324276052 0
bp_map_32768 32768 5 256 8388608 124492492 0
bp_map_49152 49152 0 30 1572864 10652 0
bp_map_65536 65536 0 8 524288 5 0
bp_map_81920 81920 0 6 524288 1 0
bp_map_98304 98304 0 5 524288 4 0
bp_map_114688 114688 0 4 524288 1 0
bp_map_131072 131072 0 0 0 0 0
fmodsw_cache 56 16 145 8192 20 0
mod_hash_entries 24 111 339 8192 8232 0
ipp_mod 304 0 0 0 0 0
ipp_action 368 0 0 0 0 0
ipp_packet 64 0 0 0 0 0
sfmmuid_cache 80 628 1818 147456 99769763 0
sfmmu8_cache 312 225077 268320 84541440 13146634 0
sfmmu1_cache 88 85817 86204 7675904 303263740 0
ism_blk_cache 208 0 0 0 0 0
ism_ment_cache 32 0 0 0 0 0
seg_cache 72 12892 35030 2539520 2960733430 0
dev_info_node_cache 464 273 306 147456 373 0
segkp_8192 8192 55 128 1048576 49636804 0
segkp_16384 16384 28 32 524288 28 0
segkp_24576 24576 1621 2640 69206016 2271477 0
segkp_32768 32768 0 0 0 0 0
segkp_40960 40960 0 24 1048576 980 0
thread_cache 672 789 1848 1261568 51062322 0
lwp_cache 896 789 1845 1679360 8064872 0
turnstile_cache 64 1608 2794 180224 50084052 0
cred_cache 132 75 236 32768 148760 0
rctl_cache 40 9502 25578 1032192 751647089 0
rctl_val_cache 64 17104 46228 2981888 1343272478 0
task_cache 80 15 202 16384 4148 0
cyclic_id_cache 64 3 127 8192 3 0
dnlc_space_cache 24 4 339 8192 1138 0
vn_cache 136 1083305 1087044 212025344 949777010 0
file_cache 56 1798 3770 212992 2129707746 0
stream_head_cache 376 820 1995 778240 310241380 0
queue_cache 640 1769 4092 2793472 656777797 0
syncq_cache 160 18 150 24576 7622 0
qband_cache 64 1 127 8192 2 0
linkinfo_cache 48 11 169 8192 12 0
ciputctrl_cache 1024 0 0 0 0 0
as_cache 152 624 1855 286720 99769898 0
marker_cache 96 0 504 49152 521498194 0
anon_cache 48 25347 54249 2629632 2459420029 0
anonmap_cache 56 7422 20155 1138688 1131652604 0
segvn_cache 104 12798 35022 3678208 2623996682 0
flk_edges 48 0 0 0 0 0
fdb_cache 104 0 0 0 0 0
timer_cache 112 1 72 8192 23 0
physio_buf_cache 224 0 36 8192 24 0
snode_cache 152 223 371 57344 768694842 0
ufs_inode_cache 360 29515 29524 10993664 29890 0
directio_buf_cache 248 0 0 0 0 0
sbus1_dvma_8192 8192 7 256 2097152 7214174 0
sbus1_dvma_16384 16384 0 16 262144 288075 0
sbus1_dvma_24576 24576 1 20 524288 67007 0
sbus1_dvma_32768 32768 0 8 262144 35761 0
sbus1_dvma_40960 40960 0 6 262144 14189 0
sbus1_dvma_49152 49152 0 10 524288 6954 0
sbus1_dvma_57344 57344 0 8 524288 2216 0
sbus1_dvma_65536 65536 0 8 524288 702259 0
isp0_cache 368 4 264 98304 5639139 0
dv_node cache 104 108 156 16384 197 0
kcf_sreq_cache 48 0 0 0 0 0
kcf_areq_cache 272 0 0 0 0 0
kcf_context_cache 88 0 0 0 0 0
ipsec_actions 72 0 113 8192 60260 0
ipsec_selectors 64 0 0 0 0 0
ipsec_policy 40 0 0 0 0 0
ipsec_info 272 0 58 16384 60260 0
ip_minor_arena_1 1 716 1984 1984 92424621 0
ipcl_conn_cache 392 65 360 163840 47244112 0
ipcl_tcpconn_cache 1416 818 2057 3063808 31373796 0
ire_cache 336 890 936 319488 20154 0
tcp_timercache 88 4096 4324 385024 1643439 0
tcp_sack_info_cache 80 783 2121 172032 28887722 0
tcp_iphc_cache 120 818 2144 262144 29036792 0
clnt_clts_endpnt_cache 88 21 92 8192 21 0
fas0_cache 264 2 30 8192 25 0
sbus3_dvma_8192 8192 1 32 262144 17 0
sbus3_dvma_16384 16384 0 0 0 0 0
sbus3_dvma_24576 24576 0 0 0 0 0
sbus3_dvma_32768 32768 0 0 0 0 0
sbus3_dvma_40960 40960 0 0 0 0 0
sbus3_dvma_49152 49152 0 0 0 0 0
sbus3_dvma_57344 57344 0 0 0 0 0
sbus3_dvma_65536 65536 0 0 0 0 0
fas1_cache 264 0 30 8192 15 0
md_stripe_parent 96 0 336 32768 3896376 0
md_stripe_child 288 0 252 73728 4535849 0
md_mirror_parent 160 0 250 40960 2092699 0
md_mirror_child 280 0 261 73728 4711878 0
md_mirror_wow 16440 0 16 278528 32141 0
sock_cache 344 680 1771 630784 97493069 0
sock_unix_cache 344 59 552 196608 217593582 0
ncafs_cache 392 0 0 0 0 0
squeue_cache 120 14 63 8192 14 0
process_cache 2600 647 1704 4653056 52562754 0
exacct_object_cache 40 0 0 0 0 0
sf_private_cache 40 14 127 8192 14 0
sbus0_dvma_8192 8192 5 32 262144 7 0
sbus0_dvma_16384 16384 0 0 0 0 0
sbus0_dvma_24576 24576 2 10 262144 2 0
sbus0_dvma_32768 32768 1 8 262144 1 0
sbus0_dvma_40960 40960 0 6 262144 2 0
sbus0_dvma_49152 49152 0 0 0 0 0
sbus0_dvma_57344 57344 0 0 0 0 0
sbus0_dvma_65536 65536 0 0 0 0 0
sbus2_dvma_8192 8192 6 64 524288 181818156 0
sbus2_dvma_16384 16384 0 32 524288 6545622 0
sbus2_dvma_24576 24576 2 10 262144 2 0
sbus2_dvma_32768 32768 1 8 262144 1 0
sbus2_dvma_40960 40960 0 6 262144 2 0
sbus2_dvma_49152 49152 0 0 0 0 0
sbus2_dvma_57344 57344 0 0 0 0 0
sbus2_dvma_65536 65536 0 0 0 0 0
sf1_cache 536 0 0 0 0 0
sf4_cache 536 0 0 0 0 0
sf0_cache 536 0 0 0 0 0
fnode_cache 160 13 50 8192 776 0
pipe_cache 288 16 112 32768 2150307 0
sf2_cache 536 0 0 0 0 0
keysock_1 1 0 0 0 0 0
fctl_cache 112 0 0 0 0 0
spdsock_1 1 0 64 64 1 0
ip_minor_1 1 0 0 0 0 0
ar_minor_1 1 0 0 0 0 0
lufs save 24 0 339 8192 85585 0
lufs bufs 232 0 105 24576 91304 0
lufs_mapentry_cache 112 50 7776 884736 3358514 0
udp_minor_1 1 0 0 0 0 0
icmp_minor_1 1 0 0 0 0 0
authkern_cache 72 586 2147 155648 797753679 0
authloopback_cache 72 0 0 0 0 0
authdes_cache_handle 80 0 0 0 0 0
rnode_cache 608 1048576 1048710 660848640 10341685 0
nfs_access_cache 56 976031 1029355 58155008 677013406 0
client_handle_cache 32 1652 1778 57344 1652 0
rnode4_cache 856 0 0 0 0 0
nfs4_access_cache 56 0 0 0 0 0
client_handle4_cache 32 0 0 0 0 0
u2s_cache 48 0 0 0 0 0
s2u_cache 48 0 0 0 0 0
g2s_cache 48 0 0 0 0 0
s2g_cache 48 0 0 0 0 0
lm_vnode 168 0 0 0 0 0
lm_xprt 32 0 0 0 0 0
lm_sysid 144 4 56 8192 4 0
lm_client 128 540 567 73728 540 0
lm_async 32 0 0 0 0 0
lm_sleep 96 1122 1176 114688 1122 0
lm_config 80 1 101 8192 1 0
md_raid_parent 120 0 0 0 0 0
md_raid_child 968 0 0 0 0 0
md_raid_cbufs 352 0 0 0 0 0
md_trans_parent 80 0 0 0 0 0
md_trans_child 224 0 0 0 0 0
pty_map 48 7 169 8192 164 0
md_softpart_parent 88 0 0 0 0 0
md_softpart_child 280 0 0 0 0 0
------------------------- ------ ------ ------ --------- --------- -----
Total [hat_memload] 84541440 13146634 0
Total [kmem_msb] 135938048 3588448 0
Total [kmem_va] 1166540800 1200475 0
Total [kmem_default] 1171431424 3049231990 0
Total [bp_map] 17301504 448779207 0
Total [id32] 8192 258 0
Total [segkp] 71827456 51909289 0
Total [sbus1_dvma] 4980736 8330635 0
Total [ip_minor_arena] 1984 92424621 0
Total [sbus3_dvma] 262144 17 0
Total [sbus0_dvma] 1048576 12 0
Total [sbus2_dvma] 1835008 188363783 0
Total [spdsock] 64 1 0
------------------------- ------ ------ ------ --------- --------- -----
vmem memory memory memory alloc alloc
name in use total import succeed fail
------------------------- --------- ---------- --------- --------- -----
heap 10310033408 4398046511104 0 49669358 0
vmem_metadata 155115520 155189248 155189248 18307 0
vmem_seg 149520384 149520384 149520384 18252 0
vmem_hash 5423616 5439488 5439488 65 0
vmem_vmem 167040 190448 155648 66 0
hat_memload 84541440 84541440 84541440 10335 0
kstat 385648 401408 335872 985 0
kmem_metadata 154107904 154140672 154140672 16714 0
kmem_msb 135938048 135938048 135938048 16599 0
kmem_cache 610848 688128 688128 316 0
kmem_hash 17467904 17481728 17481728 157 0
kmem_log 1970976 1974272 1974272 6 0
kmem_firewall_va 116883456 116883456 116883456 49620153 0
kmem_firewall 0 0 0 0 0
kmem_oversize 116660884 116883456 116883456 49620155 0
mod_sysfile 1570 8192 8192 46 0
kmem_va 9763340288 9763340288 9763340288 37282 0
kmem_default 9761366016 9761366016 9761366016 1195859 0
little_endian 0 0 0 0 0
bp_map 17301504 17301504 17301504 44 0
ksyms 1004536 1081344 1081344 137 0
ctf 740522 851968 851968 136 0
heap32 2342944 67108864 0 72 0
id32 8192 8192 8192 1 0
module_text 3226476 3678208 1122304 132 0
module_data 838414 1392640 1130496 183 0
promplat 0 0 0 19 0
segkp 71827456 2147483648 0 548 0
rctl_ids 22 32767 0 22 0
taskid_space 15 999999 0 4052 0
sbus1_dvma 10231808 1073741824 0 24461 0
ip_minor_arena 1984 262140 0 31 0
sbus3_dvma 5513216 1073741824 0 4 0
sbus0_dvma 13762560 1073741824 0 7 0
sbus2_dvma 14548992 1073741824 0 10 0
keysock 0 4294967295 0 0 0
spdsock 64 4294967295 0 1 0
ip_minor 0 262142 0 0 0
ar_minor 0 262142 0 0 0
devfsadm_event_channel 0 101 0 0 0
devfsadm_event_channel 1 2 0 1 0
syseventconfd_event_channel 0 101 0 0 0
syseventconfd_event_channel 1 2 0 1 0
syseventd_channel 0 101 0 0 0
syseventd_channel 1 2 0 1 0
udp_minor 0 262142 0 0 0
icmp_minor 0 262142 0 0 0
ptms_minor 7 16 0 164 0
------------------------- --------- ---------- --------- --------- -----
>
--
Robert Milkowski
rmilkowski at wp-sa.pl
Comments
Got something to say?
You must be logged in to post a comment.
