# /usr/share/dtrace/toolkit/hotkernel Sampling... Hit Ctrl-C to end. ^C FUNCTION COUNT PCNT kernel`vm_object_backing_scan 1 0.0% kernel`rtalloc1_fib 1 0.0% kernel`rtalloc_ign_fib 1 0.0% kernel`udp6_append 1 0.0% kernel`vm_map_pmap_enter 1 0.0% kernel`vholdl 1 0.0% zfs.ko`arc_release 1 0.0% kernel`unlock_mtx 1 0.0% kernel`softclock 1 0.0% zfs.ko`dmu_tx_count_write 1 0.0% kernel`in6_lltable_lookup 1 0.0% kernel`ip6_savecontrol 1 0.0% kernel`init_sin6 1 0.0% zfs.ko`vdev_queue_io_done 1 0.0% kernel`_vn_lock 1 0.0% zfs.ko`zfs_fuid_overquota 1 0.0% kernel`VOP_IS_TEXT_APV 1 0.0% zfs.ko`zio_add_child 1 0.0% kernel`adastart 1 0.0% zfs.ko`txg_list_add 1 0.0% zfs.ko`taskq_dispatch_ent 1 0.0% kernel`igb_msix_que 1 0.0% zfs.ko`vdev_geom_io_intr 1 0.0% kernel`sx_downgrade_ 1 0.0% kernel`udp6_input 1 0.0% zfs.ko`dsl_dataset_dirty 1 0.0% kernel`dev_lock 1 0.0% zfs.ko`vdev_queue_offset_compare 1 0.0% zfs.ko`spa_get_random 1 0.0% kernel`sosend_dgram 1 0.0% kernel`sa6_recoverscope 1 0.0% kernel`pfslowtimo 1 0.0% zfs.ko`metaslab_alloc 1 0.0% zfs.ko`range_tree_remove 1 0.0% kernel`mac_vnode_check_open 1 0.0% zfs.ko`vdev_lookup_top 1 0.0% kernel`rijndaelEncrypt 1 0.0% kernel`devfs_ioctl_f 1 0.0% zfs.ko`vdev_mirror_io_done 1 0.0% kernel`mac_ifnet_create_mbuf 1 0.0% kernel`arpintr 1 0.0% kernel`VOP_ISLOCKED_APV 1 0.0% kernel`vn_write 1 0.0% kernel`mfi_data_cb 1 0.0% kernel`cache_zap 1 0.0% zfs.ko`avl_create 1 0.0% kernel`cpu_tickrate 1 0.0% zfs.ko`zio_write_bp_init 1 0.0% zfs.ko`spa_config_enter 1 0.0% kernel`wakeup_one 1 0.0% kernel`bioq_insert_tail 1 0.0% kernel`mnt_vnode_next_active 1 0.0% kernel`vputx 1 0.0% zfs.ko`buf_hash_insert 1 0.0% kernel`sleeplk 1 0.0% kernel`vm_object_shadow 1 0.0% kernel`vm_object_madvise 1 0.0% kernel`g_is_geom_thread 1 0.0% zfs.ko`arc_freed 1 0.0% kernel`in6_pcblookup_hash_locked 1 0.0% kernel`mac_inpcb_check_deliver 1 0.0% kernel`mac_pipe_check_poll 1 0.0% kernel`release_page 1 0.0% zfs.ko`trim_map_write_done 1 0.0% zfs.ko`dmu_buf_will_dirty 1 0.0% zfs.ko`zio_vdev_io_done 1 0.0% zfs.ko`spa_config_exit 1 0.0% kernel`VOP_ACCESS_APV 1 0.0% kernel`mb_ctor_pack 1 0.0% zfs.ko`vdev_accessible 1 0.0% kernel`_cv_timedwait_sig_sbt 1 0.0% kernel`m_tag_delete_nonpersistent 1 0.0% zfs.ko`avl_first 1 0.0% kernel`read_random 1 0.0% zfs.ko`dbuf_destroy 1 0.0% zfs.ko`spa_last_synced_txg 1 0.0% kernel`scrn_update 1 0.0% kernel`taskqueue_member 1 0.0% kernel`mfi_read_fw_status_ppc 1 0.0% kernel`g_clone_bio 1 0.0% zfs.ko`zfs_freebsd_write 1 0.0% kernel`vfs_ref 1 0.0% kernel`fdunused 1 0.0% kernel`vm_map_insert 1 0.0% kernel`crfree 1 0.0% zfs.ko`atomic_cas_64 1 0.0% zfs.ko`zio_nowait 1 0.0% kernel`kern_openat 1 0.0% kernel`atkbd_check_char 1 0.0% kernel`pmap_allocpde 1 0.0% kernel`pmap_remove 1 0.0% kernel`random_yarrow_read 1 0.0% kernel`intr_event_schedule_thread 1 0.0% kernel`devstat_start_transaction_bio 1 0.0% kernel`vn_finished_write 1 0.0% zfs.ko`arc_cksum_compute 1 0.0% kernel`finit 1 0.0% zfs.ko`zfs_link_destroy 1 0.0% kernel`ifnet_byindex 1 0.0% kernel`bus_dmamap_load 1 0.0% kernel`SHA256_Update 1 0.0% kernel`mfi_disk_complete 1 0.0% zfs.ko`bp_get_dsize_sync 1 0.0% kernel`g_io_schedule_up 1 0.0% zfs.ko`spa_writeable 1 0.0% kernel`randomdev_hash_iterate 1 0.0% kernel`g_slice_start 1 0.0% kernel`tcp_hc_lookup 1 0.0% kernel`udp6_send 1 0.0% kernel`swapper 1 0.0% kernel`kern_sigprocmask 1 0.0% kernel`vm_map_delete 1 0.0% kernel`lookup 1 0.0% kernel`rijndael_blockEncrypt 1 0.0% kernel`devfs_get_cdevpriv 1 0.0% kernel`mac_netinet6_nd6_send 1 0.0% kernel`vm_radix_lookup_ge 2 0.0% zfs.ko`zio_vdev_io_assess 2 0.0% kernel`uhci_root_intr 2 0.0% zfs.ko`vdev_mirror_io_start 2 0.0% kernel`callout_lock 2 0.0% kernel`pipe_poll 2 0.0% kernel`zone_timeout 2 0.0% kernel`cpu_idle_acpi 2 0.0% kernel`biodone 2 0.0% kernel`if_simloop 2 0.0% kernel`in6_selecthlim 2 0.0% kernel`bcmp 2 0.0% zfs.ko`zio_vdev_child_io 2 0.0% dtrace.ko`dtrace_ioctl 2 0.0% zfs.ko`arc_write_physdone 2 0.0% kernel`nd6_output_lle 2 0.0% kernel`netisr_queue 2 0.0% kernel`swi_net 2 0.0% kernel`mac_ifnet_check_transmit 2 0.0% kernel`ifa_ref 2 0.0% zfs.ko`zfs_zaccess 2 0.0% kernel`vm_page_zero_idle_wakeup 2 0.0% zfs.ko`zfs_acl_next_ace 2 0.0% kernel`mfi_disk_strategy 2 0.0% dtrace.ko`dtrace_state_clean 2 0.0% kernel`vm_radix_reclaim_allnodes_int 2 0.0% kernel`pmap_zero_page 2 0.0% kernel`g_part_start 2 0.0% kernel`sys_mmap 2 0.0% kernel`mld_fasttimo 2 0.0% kernel`faithprefix 2 0.0% kernel`VOP_GETWRITEMOUNT_APV 2 0.0% kernel`random_harvest 2 0.0% kernel`pmap_copy 2 0.0% kernel`ip6_savecontrol_v4 2 0.0% zfs.ko`zfs_freebsd_access 2 0.0% kernel`ifa_free 2 0.0% kernel`fixjobc 2 0.0% kernel`ether_demux 2 0.0% kernel`kern_ioctl 2 0.0% kernel`devstat_end_transaction 2 0.0% kernel`kbdc_data_ready 2 0.0% zfs.ko`dbuf_dirty 2 0.0% zfs.ko`range_tree_seg_compare 3 0.0% kernel`cpu_activeclock 3 0.0% zfs.ko`zfs_zaccess_aces_check 3 0.0% kernel`g_disk_done_single 3 0.0% kernel`devvn_refthread 3 0.0% kernel`vm_page_free 3 0.0% zfs.ko`spa_get_dsl 3 0.0% kernel`soreceive_dgram 3 0.0% kernel`free_pv_entry 3 0.0% zfs.ko`dbuf_write_physdone 3 0.0% kernel`rn_match 3 0.0% kernel`bounce_bus_dmamap_load_buffer 3 0.0% zfs.ko`zio_notify_parent 3 0.0% kernel`mfi_startio 3 0.0% kernel`g_disk_start 3 0.0% kernel`ip6_getpmtu 3 0.0% kernel`softclock_call_cc 3 0.0% kernel`acpi_cpu_idle 4 0.0% kernel`mfi_check_clear_intr_ppc 4 0.0% kernel`scrn_timer 4 0.0% kernel`_sx_sunlock_hard 4 0.0% kernel`g_io_deliver 4 0.0% zfs.ko`dnode_block_freed 4 0.0% kernel`in6_ifawithifp 4 0.0% zfs.ko`ddt_get_dedup_stats 4 0.0% kernel`selectroute 4 0.0% kernel`lockstat_nsecs 4 0.0% zfs.ko`zio_vdev_io_start 4 0.0% kernel`__mnt_vnode_next_all 4 0.0% kernel`ip6_input 4 0.0% kernel`vfs_stdsync 5 0.0% kernel`pipe_read 5 0.0% kernel`taskqueue_enqueue 5 0.0% kernel`vm_radix_insert 5 0.0% kernel`mfi_intr 5 0.0% zfs.ko`dmu_zfetch_dofetch 6 0.0% kernel`pmap_promote_pde 6 0.0% kernel`rdtsc 6 0.0% kernel`sx_try_upgrade_ 6 0.0% kernel`taskqueue_run_locked 6 0.0% kernel`pagecopy 7 0.0% kernel`random_process_event 7 0.0% kernel`sys_poll 7 0.0% kernel`g_io_request 9 0.0% kernel`vm_object_terminate 10 0.0% zfs.ko`avl_nearest 11 0.0% kernel`intr_event_execute_handlers 11 0.0% kernel`vm_pageout_page_lock 11 0.0% dtrace.ko`dtrace_dynvar_clean 11 0.0% kernel`smp_masked_invlpg 11 0.0% zfs.ko`SHA256_Transform 12 0.0% kernel`cpu_idleclock 12 0.0% kernel`vm_page_xunbusy_hard 12 0.0% kernel`turnstile_chain_lock 12 0.0% zfs.ko`fletcher_4_native 12 0.0% kernel`ip6_output 13 0.0% kernel`doreti_iret 13 0.0% zfs.ko`avl_walk 13 0.0% kernel`vm_page_remove 14 0.0% zfs.ko`avl_find 14 0.0% kernel`ithread_loop 17 0.0% kernel`vm_map_entry_splay 19 0.0% kernel`SHA256_Transform 19 0.0% kernel`vm_reserv_alloc_page 22 0.0% kernel`_sx_slock_hard 22 0.0% kernel`vm_reserv_free_page 22 0.0% kernel`vm_radix_lookup_le 23 0.0% kernel`dump_drop_page 24 0.0% kernel`turnstile_trywait 24 0.0% kernel`tsc_get_timecount_low_lfence 24 0.0% zfs.ko`zfs_range_compare 24 0.0% kernel`igb_update_stats_counters 26 0.0% kernel`binuptime 29 0.0% kernel`vm_page_busy_sleep 30 0.0% zfs.ko`lzjb_compress 31 0.0% kernel`vm_page_unwire 37 0.0% kernel`vm_phys_alloc_pages 38 0.0% kernel`ast 40 0.0% kernel`lock_destroy 42 0.0% zfs.ko`xva_getxoptattr 48 0.0% kernel`_pmap_allocpte 49 0.0% kernel`wakeup 51 0.0% kernel`_pmap_unwire_ptp 51 0.0% zfs.ko`list_destroy 51 0.0% kernel`sx_destroy 55 0.0% kernel`smp_targeted_tlb_shootdown 63 0.0% kernel`cv_destroy 63 0.0% kernel`calltrap 68 0.0% kernel`cv_init 70 0.0% zfs.ko`dnode_hold 77 0.0% kernel`vm_page_requeue_locked 77 0.0% kernel`pmap_invalidate_page 79 0.0% zfs.ko`dbuf_whichblock 79 0.0% zfs.ko`zio_root 84 0.0% kernel`prison_allow 86 0.0% kernel`dump_add_page 86 0.0% kernel`sleepq_wait_sig 86 0.0% kernel`fget_read 92 0.0% kernel`vop_lock_post 101 0.0% opensolaris.ko`kmem_cache_free 102 0.0% zfs.ko`zrl_remove 104 0.0% kernel`ipcperm 110 0.0% zfs.ko`dbuf_rele 133 0.0% zfs.ko`avl_numnodes 138 0.0% zfs.ko`dbuf_hold 141 0.0% kernel`vop_unlock_post 145 0.0% kernel`vm_map_lookup_done 146 0.0% zfs.ko`dnode_rele 151 0.0% kernel`vm_page_sleep_if_busy 154 0.0% opensolaris.ko`zfs_kmem_free 157 0.0% zfs.ko`arc_buf_eviction_needed 166 0.0% kernel`vm_phys_alloc_domain_pages 181 0.0% zfs.ko`xva_init 182 0.0% zfs.ko`list_head 184 0.0% zfs.ko`arc_access 184 0.0% kernel`vop_unlock_pre 187 0.0% kernel`selwakeuppri 190 0.0% kernel`cap_rights 195 0.0% kernel`mac_sysvsem_check_semop 198 0.0% zfs.ko`arc_released 200 0.0% zfs.ko`list_create 202 0.0% kernel`sx_try_xlock_ 210 0.0% zfs.ko`dmu_buf_get_user 213 0.0% kernel`vm_phys_free_pages 213 0.0% zfs.ko`avl_insert 219 0.0% kernel`sbwait 223 0.0% kernel`cv_broadcastpri 228 0.0% kernel`sx_init_flags 229 0.0% kernel`vop_lock_pre 233 0.0% kernel`pagezero 237 0.0% kernel`free_pv_chunk 238 0.0% zfs.ko`zfs_tstamp_update_setup 238 0.0% kernel`getnanotime 243 0.0% zfs.ko`zio_wait 243 0.0% zfs.ko`avl_add 247 0.0% zfs.ko`sa_object_size 256 0.0% opensolaris.ko`zfs_kmem_alloc 274 0.0% kernel`lock_init 277 0.0% opensolaris.ko`kmem_cache_alloc 279 0.0% kernel`vm_pageout 281 0.0% kernel`get_advice 286 0.0% zfs.ko`zio_gang_tree_free 290 0.0% zfs.ko`remove_reference 303 0.0% zfs.ko`list_insert_head 306 0.0% kernel`sleepq_lock 309 0.0% kernel`pmap_kextract 313 0.0% zfs.ko`add_reference 324 0.0% kernel`sys_select 335 0.0% zfs.ko`sa_bulk_lookup 365 0.0% zfs.ko`arc_buf_freeze 367 0.0% kernel`mb_dtor_mbuf 369 0.0% kernel`vm_page_lookup 376 0.0% kernel`vm_page_alloc 380 0.0% kernel`vm_page_free_toq 394 0.0% kernel`getbinuptime 405 0.0% kernel`dofileread 416 0.0% zfs.ko`avl_remove 418 0.0% kernel`sys_read 434 0.0% kernel`vop_stdunlock 435 0.0% kernel`seltdwait 457 0.0% kernel`_fget 462 0.0% zfs.ko`zfs_fuid_map_ids 469 0.0% kernel`soreceive 469 0.0% zfs.ko`atomic_cas_32 484 0.0% zfs.ko`list_remove 517 0.0% kernel`atomic_fetchadd_int 519 0.0% zfs.ko`rrw_enter_read 522 0.0% zfs.ko`dmu_read_uio 524 0.0% kernel`sys_sendto 544 0.0% kernel`syscall_thread_exit 547 0.0% kernel`kern_readv 556 0.0% kernel`vm_object_set_writeable_dirty 576 0.0% kernel`mac_vnode_check_read 597 0.0% kernel`soo_poll 621 0.0% zfs.ko`zfs_range_unlock 626 0.0% kernel`uiomove 630 0.0% kernel`sopoll 635 0.0% kernel`mb_ctor_mbuf 649 0.0% zfs.ko`vdev_stat_update 650 0.0% zfs.ko`rrw_exit 658 0.0% kernel`vm_page_activate 661 0.0% zfs.ko`zfs_range_lock 666 0.0% zfs.ko`dbuf_read 679 0.0% kernel`sbunlock 682 0.0% kernel`sys_semop 707 0.0% kernel`_sx_sunlock 726 0.0% kernel`_mtx_trylock_flags_ 746 0.0% kernel`vm_map_process_deferred 752 0.0% kernel`sys_recvfrom 782 0.0% kernel`fget 814 0.0% kernel`critical_enter 824 0.0% zfs.ko`zio_execute 830 0.0% kernel`_vm_map_unlock_read 844 0.0% zfs.ko`dmu_buf_hold_array_by_dnode 845 0.0% zfs.ko`dbuf_rele_and_unlock 851 0.0% zfs.ko`zio_ready 857 0.0% kernel`mac_socket_check_poll 870 0.0% kernel`_cv_wait_sig 896 0.0% zfs.ko`zio_create 896 0.0% kernel`VOP_READ_APV 930 0.1% kernel`vn_io_fault 952 0.1% kernel`critical_exit 971 0.1% kernel`vm_map_lookup_entry 1059 0.1% kernel`memcpy 1074 0.1% kernel`vn_read 1086 0.1% kernel`selrecord 1090 0.1% kernel`vop_stdlock 1096 0.1% kernel`sosend 1098 0.1% kernel`wakeupshlk 1114 0.1% zfs.ko`arc_buf_remove_ref 1116 0.1% kernel`spinlock_enter 1116 0.1% kernel`vm_object_reference_locked 1120 0.1% kernel`knote 1127 0.1% kernel`sendit 1130 0.1% kernel`selfdfree 1134 0.1% kernel`bcopy 1143 0.1% kernel`malloc 1152 0.1% zfs.ko`atomic_add_64_nv 1177 0.1% kernel`mac_socket_check_receive 1183 0.1% kernel`m_uiotombuf 1183 0.1% kernel`vm_fault 1196 0.1% kernel`cap_check 1218 0.1% kernel`syscall_thread_enter 1228 0.1% kernel`thread_lock_flags_ 1257 0.1% kernel`copyin 1260 0.1% kernel`sbappend_locked 1261 0.1% kernel`mac_socket_check_send 1311 0.1% kernel`_sleep 1327 0.1% zfs.ko`dbuf_hold_impl 1387 0.1% kernel`sys_lseek 1400 0.1% kernel`vm_object_pip_add 1438 0.1% kernel`_sx_xlock_hard 1480 0.1% kernel`vm_object_pip_wakeup 1503 0.1% kernel`VOP_GETATTR_APV 1517 0.1% zfs.ko`zio_done 1561 0.1% zfs.ko`dmu_object_size_from_db 1593 0.1% kernel`cap_rights_contains 1608 0.1% kernel`foffset_unlock 1615 0.1% kernel`sowakeup 1630 0.1% kernel`mtx_pool_find 1670 0.1% kernel`_sx_slock 1681 0.1% kernel`sched_userret 1681 0.1% zfs.ko`dnode_hold_impl 1719 0.1% kernel`maybe_yield 1737 0.1% kernel`PHYS_TO_VM_PAGE 1758 0.1% kernel`_mtx_lock_spin_cookie 1880 0.1% zfs.ko`arc_buf_add_ref 1895 0.1% zfs.ko`list_next 1909 0.1% kernel`sblock 1953 0.1% kernel`sbcompress 1972 0.1% kernel`m_getm2 1990 0.1% kernel`uiomove_faultflag 2159 0.1% kernel`VOP_UNLOCK_APV 2261 0.1% kernel`lock_mtx 2284 0.1% kernel`userret 2314 0.1% kernel`sopoll_generic 2344 0.1% kernel`__cap_rights_is_set 2346 0.1% zfs.ko`zrl_add 2375 0.1% kernel`pmap_ts_referenced 2539 0.1% zfs.ko`zfs_freebsd_read 2640 0.1% kernel`kern_sendit 2778 0.2% kernel`free 2879 0.2% kernel`VOP_LOCK1_APV 2892 0.2% kernel`cpu_set_syscall_retval 2961 0.2% kernel`sched_idletd 3031 0.2% kernel`vn_seek 3112 0.2% dtrace.ko`dtrace_trap 3210 0.2% kernel`foffset_lock 3235 0.2% zfs.ko`dbuf_find 3331 0.2% kernel`fpudna 3398 0.2% kernel`vm_map_lookup 3519 0.2% kernel`get_pv_entry 3537 0.2% kernel`vm_page_enqueue 3663 0.2% kernel`kern_recvit 3799 0.2% zfs.ko`sa_attr_op 3812 0.2% kernel`doselwakeup 3918 0.2% kernel`atomic_add_long 4003 0.2% kernel`uma_zfree_arg 4280 0.2% kernel`__cap_rights_init 4293 0.2% kernel`bzero 4372 0.2% kernel`unlock_and_deallocate 4402 0.2% kernel`sosend_generic 4445 0.2% kernel`vm_page_dequeue 4702 0.3% kernel`pmap_try_insert_pv_entry 4791 0.3% kernel`trap_pfault 4808 0.3% kernel`uma_zalloc_arg 5058 0.3% kernel`cpu_fetch_syscall_args 5140 0.3% kernel`pmap_enter_quick_locked 5149 0.3% kernel`_sx_xunlock 5495 0.3% zfs.ko`dmu_zfetch 5777 0.3% kernel`__lockmgr_args 6089 0.3% kernel`uipc_send 6103 0.3% kernel`trap 6427 0.4% kernel`kern_select 6452 0.4% kernel`uipc_rcvd 6804 0.4% zfs.ko`zfs_freebsd_getattr 6921 0.4% kernel`vm_object_deallocate 7601 0.4% kernel`pmap_is_prefaultable 8093 0.4% kernel`pmap_enter 8746 0.5% kernel`pmap_enter_quick 9673 0.5% kernel`soreceive_generic 9949 0.5% kernel`__mtx_lock_sleep 11447 0.6% kernel`fget_unlocked 11560 0.6% kernel`_sx_xlock 12206 0.7% kernel`amd64_syscall 17807 1.0% kernel`vm_fault_hold 21926 1.2% kernel`0xffffffff80 23611 1.3% kernel`copyout 25849 1.4% kernel`_rw_runlock_cookie 25912 1.4% kernel`vm_radix_lookup 28910 1.6% kernel`pmap_remove_pages 29904 1.6% kernel`acpi_cpu_c1 119477 6.5% kernel`spinlock_exit 155967 8.5% kernel`__rw_rlock 230311 12.6% kernel`__rw_wlock_hard 299915 16.4% kernel`cpu_idle 476633 26.0%