bonnie vbox fixed size Version 1.97 ------Sequential Output------ --Sequential Input- --Random- Concurrency 1 -Per Chr- --Block-- -Rewrite- -Per Chr- --Block-- --Seeks-- Machine Size K/sec %CP K/sec %CP K/sec %CP K/sec %CP K/sec %CP /sec %CP vboxvm01 8G 1027 99 51933 4 106805 13 1673 99 1051989 83 +++++ +++ Latency 9013us 226ms 3737ms 5946us 2193us 1369us Version 1.97 ------Sequential Create------ --------Random Create-------- vboxvm01 -Create-- --Read--- -Delete-- -Create-- --Read--- -Delete-- files /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP 16 +++++ +++ +++++ +++ +++++ +++ +++++ +++ +++++ +++ +++++ +++ Latency 33056us 124us 101us 25000us 111us 77us 1.97,1.97,vboxvm01,1,1477879091,8G,,1027,99,51933,4,106805,13,1673,99,1051989,83,+++++,+++,16,,,,,+++++,+++,+++++,+++,+++++,+++,+++++,+++,+++++,+++,+++++,+++,9013us,226ms,3737ms,5946us,2193us,1369us,33056us,124us,101us,25000us,111us,77us bonnie vbox dynamic size Version 1.97 ------Sequential Output------ --Sequential Input- --Random- Concurrency 1 -Per Chr- --Block-- -Rewrite- -Per Chr- --Block-- --Seeks-- Machine Size K/sec %CP K/sec %CP K/sec %CP K/sec %CP K/sec %CP /sec %CP vboxvm01 8G 1031 99 49915 4 96102 11 1679 99 988546 78 15676 220 Latency 8740us 255ms 3319ms 5720us 407ms 63135us Version 1.97 ------Sequential Create------ --------Random Create-------- vboxvm01 -Create-- --Read--- -Delete-- -Create-- --Read--- -Delete-- files /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP 16 +++++ +++ +++++ +++ +++++ +++ +++++ +++ +++++ +++ +++++ +++ Latency 88037us 426us 885us 23564us 513us 504us 1.97,1.97,vboxvm01,1,1477861991,8G,,1031,99,49915,4,96102,11,1679,99,988546,78,15676,220,16,,,,,+++++,+++,+++++,+++,+++++,+++,+++++,+++,+++++,+++,+++++,+++,8740us,255ms,3319ms,5720us,407ms,63135us,88037us,426us,885us,23564us,513us,504us bonnie bhyve ahci Version 1.97 ------Sequential Output------ --Sequential Input- --Random- Concurrency 1 -Per Chr- --Block-- -Rewrite- -Per Chr- --Block-- --Seeks-- Machine Size K/sec %CP K/sec %CP K/sec %CP K/sec %CP K/sec %CP /sec %CP vboxvm01 8G 1045 99 37376 4 79729 12 1677 98 508504 72 2359 48 Latency 7964us 1659ms 10388ms 5469us 270ms 3182ms Version 1.97 ------Sequential Create------ --------Random Create-------- vboxvm01 -Create-- --Read--- -Delete-- -Create-- --Read--- -Delete-- files /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP 16 +++++ +++ +++++ +++ +++++ +++ +++++ +++ +++++ +++ +++++ +++ Latency 99018us 65us 46us 21761us 30us 36us 1.97,1.97,vboxvm01,1,1478025220,8G,,1045,99,37376,4,79729,12,1677,98,508504,72,2359,48,16,,,,,+++++,+++,+++++,+++,+++++,+++,+++++,+++,+++++,+++,+++++,+++,7964us,1659ms,10388ms,5469us,270ms,3182ms,99018us,65us,46us,21761us,30us,36us root@vboxvm01:~ # dumpfs / | grep "bsize" bsize 32768 shift 15 mask 0xffff8000 maxbsize 32768 maxbpg 4096 maxcontig 4 contigsumsize 4 sbsize 4096 cgsize 32768 csaddr 5056 cssize 4096 root@vboxvm01:~ # bonnie bhyve virtio Version 1.97 ------Sequential Output------ --Sequential Input- --Random- Concurrency 1 -Per Chr- --Block-- -Rewrite- -Per Chr- --Block-- --Seeks-- Machine Size K/sec %CP K/sec %CP K/sec %CP K/sec %CP K/sec %CP /sec %CP vboxvm01 8G 1016 99 108874 14 72602 10 1680 99 404693 53 +++++ +++ Latency 8242us 3112ms 9206ms 5187us 4100ms 14464us Version 1.97 ------Sequential Create------ --------Random Create-------- vboxvm01 -Create-- --Read--- -Delete-- -Create-- --Read--- -Delete-- files /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP 16 +++++ +++ +++++ +++ +++++ +++ +++++ +++ +++++ +++ +++++ +++ Latency 15728us 55us 34us 17376us 30us 36us 1.97,1.97,vboxvm01,1,1478028052,8G,,1016,99,108874,14,72602,10,1680,99,404693,53,+++++,+++,16,,,,,+++++,+++,+++++,+++,+++++,+++,+++++,+++,+++++,+++,+++++,+++,8242us,3112ms,9206ms,5187us,4100ms,14464us,15728us,55us,34us,17376us,30us,36us FreeBSD 11.0-RELEASE-p1 May 16, 2013 FreeBSD 11.0-RELEASE-p1 root@vboxvm01:~ # dumpfs / | grep "bsize" bsize 32768 shift 15 mask 0xffff8000 maxbsize 32768 maxbpg 4096 maxcontig 4 contigsumsize 4 sbsize 4096 cgsize 32768 csaddr 5056 cssize 4096 root@vboxvm01:~ # # vbox (19:53) novel@romashka:~ %> # virtio-net (19:53) novel@romashka:~ %> seq 1 8 | xargs -I % sh -c "iperf -c 192.168.87.52|tail -1" [ 3] 0.0-10.0 sec 537 MBytes 450 Mbits/sec [ 3] 0.0-10.0 sec 520 MBytes 436 Mbits/sec [ 3] 0.0-10.0 sec 520 MBytes 436 Mbits/sec [ 3] 0.0-10.0 sec 494 MBytes 415 Mbits/sec [ 3] 0.0-10.0 sec 509 MBytes 426 Mbits/sec [ 3] 0.0-10.0 sec 518 MBytes 434 Mbits/sec [ 3] 0.0-10.0 sec 525 MBytes 440 Mbits/sec [ 3] 0.0-10.0 sec 507 MBytes 425 Mbits/sec (19:55) novel@romashka:~ %> # e1000 (19:57) novel@romashka:~ %> seq 1 8 | xargs -I % sh -c "iperf -c 192.168.87.52|tail -1" [ 3] 0.0-10.0 sec 1.24 GBytes 1.06 Gbits/sec [ 3] 0.0-10.0 sec 1.76 GBytes 1.51 Gbits/sec [ 3] 0.0-10.0 sec 1.77 GBytes 1.52 Gbits/sec [ 3] 0.0-10.0 sec 1.75 GBytes 1.50 Gbits/sec [ 3] 0.0-10.0 sec 1.59 GBytes 1.36 Gbits/sec [ 3] 0.0-10.0 sec 1.70 GBytes 1.46 Gbits/sec [ 3] 0.0-10.0 sec 1.74 GBytes 1.49 Gbits/sec [ 3] 0.0-10.0 sec 1.78 GBytes 1.53 Gbits/sec (19:58) novel@romashka:~ %> # bhyve (18:14) novel@romashka:~ %> # virtio-net (18:14) novel@romashka:~ %> seq 1 8 | xargs -I % sh -c "iperf -c 192.168.87.47|tail -1" [ 3] 0.0-10.0 sec 2.81 GBytes 2.42 Gbits/sec [ 3] 0.0-10.0 sec 2.78 GBytes 2.39 Gbits/sec [ 3] 0.0-10.0 sec 2.80 GBytes 2.40 Gbits/sec [ 3] 0.0-10.0 sec 2.80 GBytes 2.41 Gbits/sec [ 3] 0.0-10.0 sec 2.80 GBytes 2.40 Gbits/sec [ 3] 0.0-10.0 sec 2.79 GBytes 2.39 Gbits/sec [ 3] 0.0-10.0 sec 2.79 GBytes 2.39 Gbits/sec [ 3] 0.0-10.0 sec 2.79 GBytes 2.40 Gbits/sec (18:16) novel@romashka:~ %> # e1000 (18:20) novel@romashka:~ %> seq 1 8 | xargs -I % sh -c "iperf -c 192.168.87.47|tail -1" [ 3] 0.0-10.0 sec 2.02 GBytes 1.73 Gbits/sec [ 3] 0.0-10.0 sec 1.62 GBytes 1.39 Gbits/sec [ 3] 0.0-10.0 sec 1.74 GBytes 1.50 Gbits/sec [ 3] 0.0-10.0 sec 1.75 GBytes 1.50 Gbits/sec [ 3] 0.0-10.0 sec 1.90 GBytes 1.63 Gbits/sec [ 3] 0.0-10.0 sec 1.86 GBytes 1.60 Gbits/sec [ 3] 0.0-10.0 sec 1.94 GBytes 1.67 Gbits/sec [ 3] 0.0-10.0 sec 1.83 GBytes 1.57 Gbits/sec (18:22) novel@romashka:~ %> diskinfo test vbox fixed root@vboxvm01:~ # seq 1 16 | xargs -L 1 sh -c "diskinfo -tv ada0 | grep middle" | tee diskinfo.log middle: 102400 kbytes in 0.082394 sec = 1242809 kbytes/sec middle: 102400 kbytes in 0.083074 sec = 1232636 kbytes/sec middle: 102400 kbytes in 0.081656 sec = 1254041 kbytes/sec middle: 102400 kbytes in 0.082273 sec = 1244637 kbytes/sec middle: 102400 kbytes in 0.083307 sec = 1229188 kbytes/sec middle: 102400 kbytes in 0.082682 sec = 1238480 kbytes/sec middle: 102400 kbytes in 0.082370 sec = 1243171 kbytes/sec middle: 102400 kbytes in 0.082350 sec = 1243473 kbytes/sec middle: 102400 kbytes in 0.082723 sec = 1237866 kbytes/sec middle: 102400 kbytes in 0.084452 sec = 1212523 kbytes/sec middle: 102400 kbytes in 0.086314 sec = 1186366 kbytes/sec middle: 102400 kbytes in 0.083240 sec = 1230178 kbytes/sec middle: 102400 kbytes in 0.082242 sec = 1245106 kbytes/sec middle: 102400 kbytes in 0.081656 sec = 1254041 kbytes/sec middle: 102400 kbytes in 0.085815 sec = 1193265 kbytes/sec middle: 102400 kbytes in 0.083213 sec = 1230577 kbytes/sec root@vboxvm01:~ # avg: 1232397 kbytes/sec root@vboxvm01:~ # root@vboxvm01:~ # seq 1 16 | xargs -t -L 1 sh -c "dd bs=1M count=2048 if=/dev/zero of=test conv=sync; sync; rm test" |& grep secs| tee dd_write.log 2147483648 bytes transferred in 46.937001 secs (45752468 bytes/sec) 2147483648 bytes transferred in 46.367979 secs (46313937 bytes/sec) 2147483648 bytes transferred in 19.166360 secs (112044416 bytes/sec) 2147483648 bytes transferred in 18.439280 secs (116462447 bytes/sec) 2147483648 bytes transferred in 16.980203 secs (126469845 bytes/sec) 2147483648 bytes transferred in 16.550426 secs (129753982 bytes/sec) 2147483648 bytes transferred in 16.894997 secs (127107667 bytes/sec) 2147483648 bytes transferred in 16.295449 secs (131784260 bytes/sec) 2147483648 bytes transferred in 16.930381 secs (126842017 bytes/sec) 2147483648 bytes transferred in 16.373250 secs (131158055 bytes/sec) 2147483648 bytes transferred in 16.921449 secs (126908969 bytes/sec) 2147483648 bytes transferred in 16.801550 secs (127814612 bytes/sec) 2147483648 bytes transferred in 16.660474 secs (128896915 bytes/sec) 2147483648 bytes transferred in 16.786854 secs (127926511 bytes/sec) 2147483648 bytes transferred in 16.383077 secs (131079381 bytes/sec) 2147483648 bytes transferred in 16.889635 secs (127148023 bytes/sec) root@vboxvm01:~ # avg: 113737 kbytes/sec vbox dynamic root@vboxvm01:~ # seq 1 16 | xargs -L 1 sh -c "diskinfo -tv ada0 | grep middle" | tee diskinfo.log middle: 102400 kbytes in 0.066579 sec = 1538022 kbytes/sec middle: 102400 kbytes in 0.066664 sec = 1536061 kbytes/sec middle: 102400 kbytes in 0.066665 sec = 1536038 kbytes/sec middle: 102400 kbytes in 0.067122 sec = 1525580 kbytes/sec middle: 102400 kbytes in 0.066040 sec = 1550575 kbytes/sec middle: 102400 kbytes in 0.066706 sec = 1535094 kbytes/sec middle: 102400 kbytes in 0.066606 sec = 1537399 kbytes/sec middle: 102400 kbytes in 0.066375 sec = 1542750 kbytes/sec middle: 102400 kbytes in 0.067910 sec = 1507878 kbytes/sec middle: 102400 kbytes in 0.068195 sec = 1501576 kbytes/sec middle: 102400 kbytes in 0.066551 sec = 1538670 kbytes/sec middle: 102400 kbytes in 0.067128 sec = 1525444 kbytes/sec middle: 102400 kbytes in 0.068425 sec = 1496529 kbytes/sec middle: 102400 kbytes in 0.067823 sec = 1509812 kbytes/sec middle: 102400 kbytes in 0.067214 sec = 1523492 kbytes/sec middle: 102400 kbytes in 0.065829 sec = 1555545 kbytes/sec root@vboxvm01:~ # avg: 1528779.0625 kbytes/sec root@vboxvm01:~ # seq 1 16 | xargs -t -L 1 sh -c "dd bs=1M count=2048 if=/dev/zero of=test conv=sync; sync; rm test" |& grep secs| tee dd_write.log 2147483648 bytes transferred in 41.548831 secs (51685778 bytes/sec) 2147483648 bytes transferred in 29.564844 secs (72636393 bytes/sec) 2147483648 bytes transferred in 14.580334 secs (147286310 bytes/sec) 2147483648 bytes transferred in 13.710771 secs (156627486 bytes/sec) 2147483648 bytes transferred in 14.584449 secs (147244756 bytes/sec) 2147483648 bytes transferred in 16.280686 secs (131903755 bytes/sec) 2147483648 bytes transferred in 12.660335 secs (169622968 bytes/sec) 2147483648 bytes transferred in 16.266103 secs (132022014 bytes/sec) 2147483648 bytes transferred in 12.682943 secs (169320615 bytes/sec) 2147483648 bytes transferred in 16.293783 secs (131797731 bytes/sec) 2147483648 bytes transferred in 12.712190 secs (168931054 bytes/sec) 2147483648 bytes transferred in 16.273954 secs (131958320 bytes/sec) 2147483648 bytes transferred in 12.676930 secs (169400925 bytes/sec) 2147483648 bytes transferred in 16.287683 secs (131847095 bytes/sec) 2147483648 bytes transferred in 12.702897 secs (169054632 bytes/sec) 2147483648 bytes transferred in 16.275932 secs (131942287 bytes/sec) root@vboxvm01:~ # avg: 135088 kbytes/sec bhyve ahci root@vboxvm01:~ # seq 1 16 | xargs -L 1 sh -c "diskinfo -tv ada0 | grep middle" | tee diskinfo.log middle: 102400 kbytes in 0.097547 sec = 1049750 kbytes/sec middle: 102400 kbytes in 0.090026 sec = 1137449 kbytes/sec middle: 102400 kbytes in 0.077193 sec = 1326545 kbytes/sec middle: 102400 kbytes in 0.077182 sec = 1326734 kbytes/sec middle: 102400 kbytes in 0.077139 sec = 1327474 kbytes/sec middle: 102400 kbytes in 0.077026 sec = 1329421 kbytes/sec middle: 102400 kbytes in 0.077405 sec = 1322912 kbytes/sec middle: 102400 kbytes in 0.077736 sec = 1317279 kbytes/sec middle: 102400 kbytes in 0.077382 sec = 1323305 kbytes/sec middle: 102400 kbytes in 0.077491 sec = 1321444 kbytes/sec middle: 102400 kbytes in 0.077392 sec = 1323134 kbytes/sec middle: 102400 kbytes in 0.076875 sec = 1332033 kbytes/sec middle: 102400 kbytes in 0.077266 sec = 1325292 kbytes/sec middle: 102400 kbytes in 0.077695 sec = 1317974 kbytes/sec middle: 102400 kbytes in 0.077043 sec = 1329128 kbytes/sec middle: 102400 kbytes in 0.077166 sec = 1327009 kbytes/sec root@vboxvm01:~ # avg: 1296055 kbytes/sec root@vboxvm01:~ # seq 1 16 | xargs -t -L 1 sh -c "dd bs=1M count=2048 if=/dev/zero of=test conv=sync; sync; rm test" |& grep secs| tee dd_write.log 2147483648 bytes transferred in 20.497597 secs (104767581 bytes/sec) 2147483648 bytes transferred in 19.483842 secs (110218694 bytes/sec) 2147483648 bytes transferred in 17.593280 secs (122062719 bytes/sec) 2147483648 bytes transferred in 17.718765 secs (121198268 bytes/sec) 2147483648 bytes transferred in 17.638532 secs (121749566 bytes/sec) 2147483648 bytes transferred in 17.836826 secs (120396064 bytes/sec) 2147483648 bytes transferred in 19.994826 secs (107401968 bytes/sec) 2147483648 bytes transferred in 17.899001 secs (119977852 bytes/sec) 2147483648 bytes transferred in 20.136477 secs (106646445 bytes/sec) 2147483648 bytes transferred in 17.638520 secs (121749653 bytes/sec) 2147483648 bytes transferred in 17.931280 secs (119761873 bytes/sec) 2147483648 bytes transferred in 17.503575 secs (122688287 bytes/sec) 2147483648 bytes transferred in 20.520099 secs (104652696 bytes/sec) 2147483648 bytes transferred in 17.513938 secs (122615693 bytes/sec) 2147483648 bytes transferred in 17.966409 secs (119527703 bytes/sec) 2147483648 bytes transferred in 17.815531 secs (120539973 bytes/sec) root@vboxvm01:~ # avg: 113889 kbytes/sec bhyve virtio root@vboxvm01:~ # seq 1 16 | xargs -L 1 sh -c "diskinfo -tv vtbd0 | grep middle" | tee diskinfo.log middle: 102400 kbytes in 0.038236 sec = 2678104 kbytes/sec middle: 102400 kbytes in 0.038830 sec = 2637136 kbytes/sec middle: 102400 kbytes in 0.038545 sec = 2656635 kbytes/sec middle: 102400 kbytes in 0.038846 sec = 2636050 kbytes/sec middle: 102400 kbytes in 0.038818 sec = 2637951 kbytes/sec middle: 102400 kbytes in 0.039273 sec = 2607389 kbytes/sec middle: 102400 kbytes in 0.038827 sec = 2637340 kbytes/sec middle: 102400 kbytes in 0.038616 sec = 2651751 kbytes/sec middle: 102400 kbytes in 0.039154 sec = 2615314 kbytes/sec middle: 102400 kbytes in 0.038641 sec = 2650035 kbytes/sec middle: 102400 kbytes in 0.038148 sec = 2684282 kbytes/sec middle: 102400 kbytes in 0.039064 sec = 2621339 kbytes/sec middle: 102400 kbytes in 0.038606 sec = 2652437 kbytes/sec middle: 102400 kbytes in 0.038526 sec = 2657945 kbytes/sec middle: 102400 kbytes in 0.038509 sec = 2659119 kbytes/sec middle: 102400 kbytes in 0.038207 sec = 2680137 kbytes/sec root@vboxvm01:~ # avg: 2647685.25 kbytes/sec seq 1 16 | xargs -t -L 1 sh -c "dd bs=1M count=2048 if=/dev/zero of=test conv=sync; sync; rm test" | tee dd_write.log root@vboxvm01:~ # seq 1 16 | xargs -t -L 1 sh -c "dd bs=1M count=2048 if=/dev/zero of=test conv=sync; sync; rm test" |& grep secs| tee dd_write.log 2147483648 bytes transferred in 17.195208 secs (124888498 bytes/sec) 2147483648 bytes transferred in 17.490449 secs (122780360 bytes/sec) 2147483648 bytes transferred in 17.348501 secs (123784972 bytes/sec) 2147483648 bytes transferred in 21.853299 secs (98268169 bytes/sec) 2147483648 bytes transferred in 17.383257 secs (123537475 bytes/sec) 2147483648 bytes transferred in 17.638768 secs (121747940 bytes/sec) 2147483648 bytes transferred in 21.275358 secs (100937605 bytes/sec) 2147483648 bytes transferred in 17.516020 secs (122601116 bytes/sec) 2147483648 bytes transferred in 17.645046 secs (121704622 bytes/sec) 2147483648 bytes transferred in 17.098440 secs (125595298 bytes/sec) 2147483648 bytes transferred in 17.604795 secs (121982884 bytes/sec) 2147483648 bytes transferred in 17.416624 secs (123300797 bytes/sec) 2147483648 bytes transferred in 17.410222 secs (123346139 bytes/sec) 2147483648 bytes transferred in 21.468304 secs (100030429 bytes/sec) 2147483648 bytes transferred in 17.394110 secs (123460390 bytes/sec) 2147483648 bytes transferred in 17.700210 secs (121325322 bytes/sec) root@vboxvm01:~ # avg: 115924 kbytes/sec