HW>#lshw
# fio --version
fio 1.38
# cat test.sh
#!/bin/sh
# ------ General parameter define. ------
FIO=/usr/bin/fio
NAME=zfs-bench
OUTPUT=$1
SIZE=18G
BS=512k
JOBS=1
RW=randwrite

# ------ single I/O ------
$FIO -filename=$OUTPUT -rw=write -bs=$BS -size=$SIZE -numjobs=$JOBS -startdelay=5 -name=$NAME
$FIO -filename=$OUTPUT -rw=read  -bs=$BS -size=$SIZE -numjobs=$JOBS -startdelay=5 -name=$NAME

# ------ Random 512K Read,Write. ------
$FIO -filename=$OUTPUT -rw=randwrite -bs=$BS -size=$SIZE -numjobs=$JOBS -startdelay=5 -name=$NAME
$FIO -filename=$OUTPUT -rw=randread  -bs=$BS -size=$SIZE -numjobs=$JOBS -startdelay=5 -name=$NAME

# ------ Random 4KiB Read,Write. ------
BS=4096
RUNTIME=600
$FIO -filename=$OUTPUT -rw=randwrite -bs=$BS -size=$SIZE -numjobs=$JOBS -runtime=$RUNTIME -startdelay=5 -name=$NAME
$FIO -filename=$OUTPUT -rw=randread  -bs=$BS -size=$SIZE -numjobs=$JOBS -runtime=$RUNTIME -startdelay=5 -name=$NAME

# ------ Random 4KiB Read,Write. ------
IO=posixaio
QUEUE=32
$FIO -filename=$OUTPUT -rw=randwrite -ioengine=$IO -iodepth=$QUEUE -bs=$BS -size=$SIZE -numjobs=$JOBS -runtime=$RUNTIME -startdelay=5 -name=$NAME
$FIO -filename=$OUTPUT -rw=randread  -ioengine=$IO -iodepth=$QUEUE -bs=$BS -size=$SIZE -numjobs=$JOBS -runtime=$RUNTIME -startdelay=5 -unlink=1 -name=$NAME

# ------ 32 multiple I/O ------
$FIO -filename=$OUTPUT -rw=write -bs=$BS -size=589824k -numjobs=32 -runtime=$RUNTIME -startdelay=5 -name=$NAME
$FIO -filename=$OUTPUT -rw=read  -bs=$BS -size=589824k -numjobs=32 -runtime=$RUNTIME -startdelay=5 -name=$NAME
RAIDZ-1 RAIDZ-2
RAIDZ-1 zfs-bench: (g=0): rw=write, bs=512K-512K/512K-512K, ioengine=sync, iodepth=1 Starting 1 process zfs-bench: Laying out IO file(s) (1 file(s) / 18432MB) zfs-bench: (groupid=0, jobs=1): err= 0: pid=14953 write: io=18432MB, bw=108802KB/s, iops=212, runt=173474msec clat (usec): min=175, max=5382K, avg=4703.00, stdev=82547.90 bw (KB/s) : min=43213, max=1432576, per=114.94%, avg=125052.94, stdev=148798.08 cpu : usr=0.08%, sys=4.90%, ctx=599, majf=0, minf=29 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/36864, short=0/0 lat (usec): 250=68.93%, 500=30.38%, 750=0.06%, 1000=0.03% lat (msec): 2=0.07%, 4=0.04%, 10=0.09%, 250=0.01%, 500=0.01% lat (msec): 750=0.02%, 1000=0.16%, 2000=0.19%, >=2000=0.02% Run status group 0 (all jobs): WRITE: io=18432MB, aggrb=108802KB/s, minb=111413KB/s, maxb=111413KB/s, mint=173474msec, maxt=173474msec zfs-bench: (g=0): rw=read, bs=512K-512K/512K-512K, ioengine=sync, iodepth=1 Starting 1 process zfs-bench: (groupid=0, jobs=1): err= 0: pid=14957 read : io=18432MB, bw=3380MB/s, iops=6760, runt= 5453msec clat (usec): min=137, max=3876, avg=146.03, stdev=25.13 bw (KB/s) : min=3383296, max=3477504, per=100.00%, avg=3461222.40, stdev=28616.89 cpu : usr=2.05%, sys=97.65%, ctx=22, majf=0, minf=153 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=36864/0, short=0/0 lat (usec): 250=98.41%, 500=1.59%, 1000=0.01% lat (msec): 4=0.01% Run status group 0 (all jobs): READ: io=18432MB, aggrb=3380MB/s, minb=3461MB/s, maxb=3461MB/s, mint=5453msec, maxt=5453msec zfs-bench: (g=0): rw=randwrite, bs=512K-512K/512K-512K, ioengine=sync, iodepth=1 Starting 1 process zfs-bench: (groupid=0, jobs=1): err= 0: pid=14959 write: io=18432MB, bw=73272KB/s, iops=143, runt=257593msec clat (usec): min=177, max=5063K, avg=6983.70, stdev=94486.44 bw (KB/s) : min=39132, max=984083, per=105.73%, avg=77467.53, stdev=82811.68 cpu : usr=0.06%, sys=3.40%, ctx=595, majf=0, minf=33 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/36864, short=0/0 lat (usec): 250=63.12%, 500=36.05%, 750=0.05%, 1000=0.04% lat (msec): 2=0.04%, 4=0.03%, 10=0.06%, 500=0.02%, 750=0.01% lat (msec): 1000=0.23%, 2000=0.35%, >=2000=0.02% Run status group 0 (all jobs): WRITE: io=18432MB, aggrb=73272KB/s, minb=75030KB/s, maxb=75030KB/s, mint=257593msec, maxt=257593msec zfs-bench: (g=0): rw=randread, bs=512K-512K/512K-512K, ioengine=sync, iodepth=1 Starting 1 process zfs-bench: (groupid=0, jobs=1): err= 0: pid=14963 read : io=18432MB, bw=3260MB/s, iops=6519, runt= 5654msec clat (usec): min=142, max=4021, avg=150.66, stdev=23.64 bw (KB/s) : min=3198976, max=3388416, per=99.99%, avg=3337774.55, stdev=66098.96 cpu : usr=1.42%, sys=98.28%, ctx=39, majf=0, minf=151 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=36864/0, short=0/0 lat (usec): 250=99.84%, 500=0.14%, 750=0.01% lat (msec): 2=0.01%, 10=0.01% Run status group 0 (all jobs): READ: io=18432MB, aggrb=3260MB/s, minb=3338MB/s, maxb=3338MB/s, mint=5654msec, maxt=5654msec zfs-bench: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1 Starting 1 process zfs-bench: (groupid=0, jobs=1): err= 0: pid=14965 write: io=1722MB, bw=2936KB/s, iops=734, runt=600661msec clat (usec): min=12, max=4426K, avg=1360.68, stdev=44705.59 bw (KB/s) : min= 1101, max=31781, per=112.79%, avg=3311.48, stdev=2810.88 cpu : usr=0.12%, sys=1.85%, ctx=876, majf=0, minf=1617 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/440935, short=0/0 lat (usec): 20=38.04%, 50=60.01%, 100=0.26%, 250=1.38%, 500=0.11% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.05%, 10=0.03%, 20=0.01%, 500=0.01% lat (msec): 750=0.01%, 1000=0.02%, 2000=0.06%, >=2000=0.01% Run status group 0 (all jobs): WRITE: io=1722MB, aggrb=2936KB/s, minb=3006KB/s, maxb=3006KB/s, mint=600661msec, maxt=600661msec zfs-bench: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1 Starting 1 process zfs-bench: (groupid=0, jobs=1): err= 0: pid=14971 read : io=18432MB, bw=459723KB/s, iops=114930, runt= 41056msec clat (usec): min=5, max=2204, avg= 7.47, stdev= 1.81 bw (KB/s) : min=380392, max=475728, per=100.13%, avg=460303.12, stdev=13071.67 cpu : usr=15.68%, sys=84.14%, ctx=72, majf=0, minf=322 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=4718592/0, short=0/0 lat (usec): 10=99.63%, 20=0.21%, 50=0.14%, 100=0.02%, 250=0.01% lat (usec): 500=0.01% lat (msec): 2=0.01%, 4=0.01% Run status group 0 (all jobs): READ: io=18432MB, aggrb=459722KB/s, minb=470755KB/s, maxb=470755KB/s, mint=41056msec, maxt=41056msec zfs-bench: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=posixaio, iodepth=32 Starting 1 process zfs-bench: (groupid=0, jobs=1): err= 0: pid=14973 write: io=1612MB, bw=2743KB/s, iops=685, runt=601693msec slat (usec): min=0, max=3849, avg= 0.71, stdev=12.96 clat (usec): min=21, max=4940K, avg=46669.55, stdev=270763.62 bw (KB/s) : min= 932, max=31250, per=119.21%, avg=3268.75, stdev=2783.00 cpu : usr=0.16%, sys=1.73%, ctx=25885, majf=0, minf=1644 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued r/w: total=0/412549, short=0/0 lat (usec): 50=0.09%, 100=0.11%, 250=0.10%, 500=2.78%, 750=48.89% lat (usec): 1000=19.46% lat (msec): 2=19.20%, 4=3.83%, 10=1.92%, 20=0.01%, 50=0.01% lat (msec): 250=0.03%, 500=0.19%, 750=0.31%, 1000=0.73%, 2000=1.98% lat (msec): >=2000=0.39% Run status group 0 (all jobs): WRITE: io=1612MB, aggrb=2742KB/s, minb=2808KB/s, maxb=2808KB/s, mint=601693msec, maxt=601693msec zfs-bench: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=posixaio, iodepth=32 Starting 1 process zfs-bench: (groupid=0, jobs=1): err= 0: pid=14981 read : io=18432MB, bw=491712KB/s, iops=122928, runt= 38385msec slat (usec): min=0, max=1024, avg= 0.78, stdev= 1.65 clat (usec): min=7, max=117409, avg=258.80, stdev=277.77 bw (KB/s) : min=468264, max=516032, per=100.25%, avg=492937.68, stdev=17620.95 cpu : usr=26.50%, sys=95.22%, ctx=453934, majf=0, minf=379 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued r/w: total=4718592/0, short=0/0 lat (usec): 10=0.02%, 20=0.17%, 50=0.58%, 100=1.42%, 250=51.34% lat (usec): 500=46.08%, 750=0.38%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): READ: io=18432MB, aggrb=491712KB/s, minb=503513KB/s, maxb=503513KB/s, mint=38385msec, maxt=38385msec zfs-bench: (g=0): rw=write, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1 ... zfs-bench: (g=0): rw=write, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1 Starting 32 processes zfs-bench: Laying out IO file(s) (1 file(s) / 576MB) zfs-bench: (groupid=0, jobs=1): err= 0: pid=14984 write: io=589824KB, bw=14584KB/s, iops=3646, runt= 40443msec clat (usec): min=12, max=132820, avg=272.96, stdev=2980.88 bw (KB/s) : min= 6102, max=23699, per=3.19%, avg=14625.33, stdev=3570.60 cpu : usr=0.47%, sys=23.84%, ctx=3599, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.70%, 50=33.24%, 100=50.93%, 250=14.26%, 500=0.10% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.04%, 10=0.12%, 20=0.15%, 50=0.32% lat (msec): 100=0.10%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=14985 write: io=589824KB, bw=14815KB/s, iops=3703, runt= 39813msec clat (usec): min=11, max=128098, avg=268.44, stdev=2941.02 bw (KB/s) : min= 5996, max=24880, per=3.24%, avg=14855.36, stdev=4091.71 cpu : usr=0.47%, sys=24.23%, ctx=3407, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.42%, 50=33.68%, 100=50.81%, 250=14.18%, 500=0.12% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.13%, 20=0.16%, 50=0.31% lat (msec): 100=0.10%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=14986 write: io=589824KB, bw=15549KB/s, iops=3887, runt= 37934msec clat (usec): min=11, max=119851, avg=255.71, stdev=2813.80 bw (KB/s) : min= 9135, max=22939, per=3.39%, avg=15541.14, stdev=2953.61 cpu : usr=0.53%, sys=25.61%, ctx=2958, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.46%, 50=33.07%, 100=50.91%, 250=14.74%, 500=0.12% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.03%, 10=0.11%, 20=0.12%, 50=0.30% lat (msec): 100=0.10%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=14987 write: io=589824KB, bw=17044KB/s, iops=4260, runt= 34606msec clat (usec): min=11, max=109115, avg=233.47, stdev=2596.35 bw (KB/s) : min=10157, max=27509, per=3.72%, avg=17049.22, stdev=3661.00 cpu : usr=0.54%, sys=28.04%, ctx=2825, majf=0, minf=28 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.63%, 50=33.38%, 100=50.75%, 250=14.39%, 500=0.20% lat (usec): 750=0.02%, 1000=0.01% lat (msec): 2=0.02%, 4=0.03%, 10=0.10%, 20=0.14%, 50=0.27% lat (msec): 100=0.07%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=14988 write: io=589824KB, bw=15742KB/s, iops=3935, runt= 37468msec clat (usec): min=11, max=124063, avg=252.74, stdev=2805.10 bw (KB/s) : min= 7601, max=27602, per=3.45%, avg=15788.54, stdev=3813.68 cpu : usr=0.47%, sys=26.15%, ctx=3073, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.45%, 50=32.98%, 100=51.00%, 250=14.60%, 500=0.27% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.03%, 10=0.12%, 20=0.13%, 50=0.29% lat (msec): 100=0.09%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=14989 write: io=589824KB, bw=14919KB/s, iops=3729, runt= 39536msec clat (usec): min=12, max=142712, avg=266.56, stdev=2903.32 bw (KB/s) : min= 5933, max=24214, per=3.26%, avg=14933.72, stdev=3904.58 cpu : usr=0.48%, sys=24.56%, ctx=3443, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.32%, 50=33.30%, 100=51.02%, 250=14.50%, 500=0.09% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.04%, 10=0.13%, 20=0.15%, 50=0.31% lat (msec): 100=0.10%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=14990 write: io=589824KB, bw=14815KB/s, iops=3703, runt= 39813msec clat (usec): min=11, max=106695, avg=268.85, stdev=2907.86 bw (KB/s) : min= 5581, max=26070, per=3.23%, avg=14771.90, stdev=3728.59 cpu : usr=0.52%, sys=24.44%, ctx=3437, majf=0, minf=28 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.40%, 50=32.62%, 100=51.44%, 250=14.68%, 500=0.10% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.03%, 10=0.12%, 20=0.16%, 50=0.31% lat (msec): 100=0.10%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=14991 write: io=589824KB, bw=15026KB/s, iops=3756, runt= 39254msec clat (usec): min=11, max=119336, avg=265.18, stdev=2871.63 bw (KB/s) : min= 7691, max=23288, per=3.28%, avg=15038.93, stdev=3645.13 cpu : usr=0.51%, sys=24.54%, ctx=3477, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.59%, 50=33.60%, 100=50.62%, 250=14.31%, 500=0.08% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.05%, 10=0.13%, 20=0.17%, 50=0.30% lat (msec): 100=0.09%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=14992 write: io=589824KB, bw=14311KB/s, iops=3577, runt= 41215msec clat (usec): min=11, max=154026, avg=278.25, stdev=3103.98 bw (KB/s) : min= 5944, max=38600, per=3.01%, avg=13779.61, stdev=4468.32 cpu : usr=0.39%, sys=23.09%, ctx=3666, majf=0, minf=28 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=1.17%, 50=34.52%, 100=49.50%, 250=13.96%, 500=0.06% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.04%, 10=0.13%, 20=0.15%, 50=0.31% lat (msec): 100=0.11%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=14993 write: io=589824KB, bw=14315KB/s, iops=3578, runt= 41204msec clat (usec): min=11, max=189310, avg=277.84, stdev=3031.33 bw (KB/s) : min= 6502, max=41424, per=3.03%, avg=13891.33, stdev=5122.49 cpu : usr=0.37%, sys=23.27%, ctx=3964, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.49%, 50=34.56%, 100=50.16%, 250=13.93%, 500=0.05% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.14%, 20=0.16%, 50=0.32% lat (msec): 100=0.10%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=14994 write: io=589824KB, bw=14365KB/s, iops=3591, runt= 41061msec clat (usec): min=14, max=177659, avg=277.24, stdev=3050.76 bw (KB/s) : min= 7128, max=34424, per=3.11%, avg=14229.53, stdev=4364.80 cpu : usr=0.26%, sys=23.90%, ctx=3583, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.46%, 50=32.75%, 100=51.23%, 250=14.71%, 500=0.05% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.03%, 10=0.13%, 20=0.17%, 50=0.30% lat (msec): 100=0.11%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=14995 write: io=589824KB, bw=14911KB/s, iops=3727, runt= 39555msec clat (usec): min=11, max=126784, avg=267.25, stdev=2917.66 bw (KB/s) : min= 7304, max=24613, per=3.24%, avg=14841.07, stdev=4077.70 cpu : usr=0.51%, sys=24.39%, ctx=3508, majf=0, minf=28 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.57%, 50=33.36%, 100=50.90%, 250=14.30%, 500=0.09% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.04%, 10=0.12%, 20=0.17%, 50=0.30% lat (msec): 100=0.09%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=14996 write: io=589824KB, bw=14492KB/s, iops=3622, runt= 40700msec clat (usec): min=11, max=116570, avg=274.69, stdev=2938.28 bw (KB/s) : min= 7537, max=39944, per=3.17%, avg=14495.81, stdev=4762.79 cpu : usr=0.52%, sys=23.69%, ctx=3678, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.50%, 50=33.42%, 100=50.92%, 250=14.29%, 500=0.08% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.03%, 10=0.13%, 20=0.16%, 50=0.33% lat (msec): 100=0.09%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=14997 write: io=589824KB, bw=14382KB/s, iops=3595, runt= 41012msec clat (usec): min=12, max=121959, avg=276.87, stdev=3017.68 bw (KB/s) : min= 7056, max=24144, per=3.07%, avg=14081.09, stdev=3671.90 cpu : usr=0.40%, sys=23.75%, ctx=3650, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.45%, 50=32.90%, 100=51.14%, 250=14.66%, 500=0.05% lat (usec): 750=0.02%, 1000=0.01% lat (msec): 2=0.03%, 4=0.03%, 10=0.12%, 20=0.16%, 50=0.30% lat (msec): 100=0.11%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=14998 write: io=589824KB, bw=14701KB/s, iops=3675, runt= 40120msec clat (usec): min=12, max=175260, avg=270.70, stdev=2972.93 bw (KB/s) : min= 7134, max=22059, per=3.20%, avg=14659.92, stdev=3497.96 cpu : usr=0.38%, sys=24.32%, ctx=3567, majf=0, minf=28 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.47%, 50=33.09%, 100=51.00%, 250=14.55%, 500=0.11% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.05%, 10=0.12%, 20=0.17%, 50=0.30% lat (msec): 100=0.10%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=14999 write: io=589824KB, bw=14315KB/s, iops=3578, runt= 41204msec clat (usec): min=11, max=115189, avg=278.30, stdev=2971.85 bw (KB/s) : min= 6949, max=47648, per=3.11%, avg=14247.36, stdev=5728.35 cpu : usr=0.45%, sys=23.25%, ctx=4060, majf=0, minf=28 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.47%, 50=34.31%, 100=50.47%, 250=13.84%, 500=0.07% lat (usec): 750=0.02%, 1000=0.01% lat (msec): 2=0.02%, 4=0.05%, 10=0.14%, 20=0.17%, 50=0.31% lat (msec): 100=0.11%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15000 write: io=589824KB, bw=14653KB/s, iops=3663, runt= 40254msec clat (usec): min=11, max=139587, avg=271.78, stdev=3002.25 bw (KB/s) : min= 6244, max=27200, per=3.18%, avg=14565.44, stdev=3569.17 cpu : usr=0.38%, sys=24.24%, ctx=3590, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.40%, 50=33.10%, 100=50.92%, 250=14.70%, 500=0.10% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.04%, 10=0.14%, 20=0.16%, 50=0.30% lat (msec): 100=0.10%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15001 write: io=589824KB, bw=15017KB/s, iops=3754, runt= 39276msec clat (usec): min=11, max=118739, avg=265.14, stdev=2863.66 bw (KB/s) : min= 5472, max=25976, per=3.27%, avg=14982.66, stdev=3660.41 cpu : usr=0.43%, sys=24.82%, ctx=3481, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.47%, 50=32.85%, 100=51.44%, 250=14.32%, 500=0.17% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.03%, 10=0.11%, 20=0.16%, 50=0.31% lat (msec): 100=0.10%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15002 write: io=589824KB, bw=14357KB/s, iops=3589, runt= 41082msec clat (usec): min=12, max=142204, avg=277.52, stdev=3003.22 bw (KB/s) : min= 6507, max=27782, per=3.11%, avg=14224.95, stdev=4033.03 cpu : usr=0.38%, sys=23.63%, ctx=3521, majf=0, minf=28 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.35%, 50=33.24%, 100=51.27%, 250=14.28%, 500=0.07% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.11%, 20=0.17%, 50=0.32% lat (msec): 100=0.11%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15003 write: io=589824KB, bw=14402KB/s, iops=3600, runt= 40954msec clat (usec): min=11, max=151842, avg=276.15, stdev=3032.72 bw (KB/s) : min= 8384, max=36080, per=3.14%, avg=14400.00, stdev=4555.11 cpu : usr=0.37%, sys=23.81%, ctx=3823, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.37%, 50=32.89%, 100=51.37%, 250=14.52%, 500=0.05% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.05%, 10=0.13%, 20=0.15%, 50=0.31% lat (msec): 100=0.11%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15004 write: io=589824KB, bw=14357KB/s, iops=3589, runt= 41083msec clat (usec): min=11, max=136304, avg=276.93, stdev=2961.63 bw (KB/s) : min= 5535, max=35600, per=3.09%, avg=14152.11, stdev=5081.51 cpu : usr=0.45%, sys=23.50%, ctx=3811, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.75%, 50=32.94%, 100=51.09%, 250=14.33%, 500=0.05% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.05%, 10=0.14%, 20=0.16%, 50=0.33% lat (msec): 100=0.11%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15005 write: io=589824KB, bw=14400KB/s, iops=3600, runt= 40959msec clat (usec): min=12, max=150620, avg=276.43, stdev=3039.50 bw (KB/s) : min= 5423, max=31904, per=3.14%, avg=14380.33, stdev=4367.89 cpu : usr=0.45%, sys=23.67%, ctx=3703, majf=0, minf=28 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.45%, 50=32.87%, 100=51.36%, 250=14.45%, 500=0.06% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.15%, 20=0.15%, 50=0.31% lat (msec): 100=0.10%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15006 write: io=589824KB, bw=14492KB/s, iops=3622, runt= 40700msec clat (usec): min=12, max=124038, avg=274.84, stdev=2928.59 bw (KB/s) : min= 7503, max=24408, per=3.16%, avg=14456.10, stdev=3519.39 cpu : usr=0.48%, sys=23.64%, ctx=3679, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.70%, 50=33.35%, 100=50.81%, 250=14.25%, 500=0.08% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.04%, 10=0.13%, 20=0.17%, 50=0.32% lat (msec): 100=0.10%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15007 write: io=589824KB, bw=14402KB/s, iops=3600, runt= 40953msec clat (usec): min=12, max=140444, avg=276.42, stdev=3080.99 bw (KB/s) : min= 4847, max=28735, per=3.09%, avg=14160.08, stdev=4451.85 cpu : usr=0.54%, sys=23.71%, ctx=3713, majf=0, minf=28 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.43%, 50=32.77%, 100=51.29%, 250=14.62%, 500=0.11% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.12%, 20=0.16%, 50=0.30% lat (msec): 100=0.11%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15008 write: io=589824KB, bw=14649KB/s, iops=3662, runt= 40263msec clat (usec): min=11, max=107217, avg=271.72, stdev=2916.70 bw (KB/s) : min= 6984, max=25704, per=3.18%, avg=14567.96, stdev=3738.04 cpu : usr=0.41%, sys=24.04%, ctx=3388, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.58%, 50=33.14%, 100=51.17%, 250=14.26%, 500=0.05% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.12%, 20=0.17%, 50=0.31% lat (msec): 100=0.11%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15009 write: io=589824KB, bw=15710KB/s, iops=3927, runt= 37545msec clat (usec): min=13, max=121622, avg=253.17, stdev=2793.49 bw (KB/s) : min= 8169, max=25720, per=3.44%, avg=15766.21, stdev=3765.05 cpu : usr=0.56%, sys=25.91%, ctx=3100, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.34%, 50=32.92%, 100=51.38%, 250=14.52%, 500=0.15% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.03%, 10=0.11%, 20=0.13%, 50=0.29% lat (msec): 100=0.09%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15010 write: io=589824KB, bw=17002KB/s, iops=4250, runt= 34691msec clat (usec): min=11, max=147279, avg=233.81, stdev=2683.85 bw (KB/s) : min= 6731, max=32150, per=3.73%, avg=17067.94, stdev=4317.91 cpu : usr=0.54%, sys=27.65%, ctx=2696, majf=0, minf=28 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.95%, 50=33.42%, 100=50.64%, 250=14.33%, 500=0.03% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.03%, 10=0.10%, 20=0.13%, 50=0.26% lat (msec): 100=0.08%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15011 write: io=589824KB, bw=14365KB/s, iops=3591, runt= 41060msec clat (usec): min=11, max=122090, avg=277.28, stdev=3047.50 bw (KB/s) : min= 8186, max=29048, per=3.12%, avg=14308.87, stdev=3672.42 cpu : usr=0.40%, sys=23.61%, ctx=3524, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.55%, 50=33.13%, 100=51.10%, 250=14.40%, 500=0.05% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.03%, 10=0.13%, 20=0.15%, 50=0.31% lat (msec): 100=0.11%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15012 write: io=589824KB, bw=14637KB/s, iops=3659, runt= 40297msec clat (usec): min=14, max=137965, avg=272.08, stdev=2954.36 bw (KB/s) : min= 7241, max=24254, per=3.18%, avg=14548.74, stdev=3519.59 cpu : usr=0.42%, sys=24.11%, ctx=3620, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.38%, 50=33.22%, 100=50.87%, 250=14.68%, 500=0.06% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.13%, 20=0.16%, 50=0.31% lat (msec): 100=0.10%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15013 write: io=589824KB, bw=14560KB/s, iops=3639, runt= 40510msec clat (usec): min=11, max=132728, avg=273.54, stdev=2990.58 bw (KB/s) : min= 6904, max=22920, per=3.14%, avg=14376.04, stdev=3658.79 cpu : usr=0.45%, sys=23.88%, ctx=3661, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.45%, 50=33.31%, 100=51.12%, 250=14.25%, 500=0.07% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.04%, 10=0.14%, 20=0.16%, 50=0.32% lat (msec): 100=0.10%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15014 write: io=589824KB, bw=15259KB/s, iops=3814, runt= 38654msec clat (usec): min=11, max=148262, avg=260.62, stdev=2911.64 bw (KB/s) : min= 7545, max=26257, per=3.33%, avg=15229.25, stdev=3965.07 cpu : usr=0.43%, sys=25.00%, ctx=3375, majf=0, minf=28 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.52%, 50=33.58%, 100=50.71%, 250=14.34%, 500=0.12% lat (usec): 750=0.02%, 1000=0.01% lat (msec): 2=0.03%, 4=0.03%, 10=0.12%, 20=0.14%, 50=0.28% lat (msec): 100=0.11%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15015 write: io=589824KB, bw=14322KB/s, iops=3580, runt= 41182msec clat (usec): min=11, max=145917, avg=278.17, stdev=3051.81 bw (KB/s) : min= 3930, max=49584, per=3.11%, avg=14233.86, stdev=6141.29 cpu : usr=0.30%, sys=23.50%, ctx=3775, majf=0, minf=28 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.38%, 50=33.87%, 100=50.77%, 250=14.13%, 500=0.05% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.12%, 20=0.17%, 50=0.32% lat (msec): 100=0.10%, 250=0.01% Run status group 0 (all jobs): WRITE: io=18432MB, aggrb=457948KB/s, minb=14654KB/s, maxb=17453KB/s, mint=34606msec, maxt=41215msec zfs-bench: (g=0): rw=read, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1 ... zfs-bench: (g=0): rw=read, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1 Starting 32 processes zfs-bench: (groupid=0, jobs=1): err= 0: pid=15017 read : io=589824KB, bw=57253KB/s, iops=14313, runt= 10302msec clat (usec): min=5, max=104062, avg=68.30, stdev=1430.68 bw (KB/s) : min=39824, max=85415, per=3.31%, avg=57063.11, stdev=14351.67 cpu : usr=1.32%, sys=24.89%, ctx=533, majf=0, minf=25 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.38%, 20=74.64%, 50=24.73%, 100=0.06%, 250=0.01% lat (usec): 500=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.05%, 50=0.07% lat (msec): 100=0.03%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15018 read : io=589824KB, bw=58730KB/s, iops=14682, runt= 10043msec clat (usec): min=6, max=87316, avg=66.89, stdev=1391.80 bw (KB/s) : min=35776, max=88880, per=3.41%, avg=58660.84, stdev=14561.55 cpu : usr=1.55%, sys=25.37%, ctx=524, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.44%, 20=74.27%, 50=25.05%, 100=0.06%, 250=0.01% lat (usec): 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.05%, 50=0.08% lat (msec): 100=0.02% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15019 read : io=589824KB, bw=54918KB/s, iops=13729, runt= 10740msec clat (usec): min=5, max=94474, avg=71.42, stdev=1492.86 bw (KB/s) : min=35329, max=87680, per=3.13%, avg=53988.50, stdev=13195.40 cpu : usr=1.38%, sys=23.65%, ctx=561, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.42%, 20=75.23%, 50=24.10%, 100=0.06%, 250=0.01% lat (usec): 500=0.01%, 750=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.04%, 50=0.07% lat (msec): 100=0.04% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15020 read : io=589824KB, bw=53821KB/s, iops=13455, runt= 10959msec clat (usec): min=8, max=88199, avg=73.19, stdev=1550.92 bw (KB/s) : min=33490, max=79640, per=2.99%, avg=51441.81, stdev=11461.05 cpu : usr=1.20%, sys=23.40%, ctx=568, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.03%, 20=75.03%, 50=24.69%, 100=0.06%, 250=0.01% lat (usec): 500=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.04%, 50=0.08% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15021 read : io=589824KB, bw=55529KB/s, iops=13882, runt= 10622msec clat (usec): min=6, max=79581, avg=70.94, stdev=1497.29 bw (KB/s) : min=35389, max=76520, per=3.20%, avg=55133.05, stdev=10335.41 cpu : usr=1.69%, sys=23.46%, ctx=591, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.87%, 20=75.12%, 50=23.75%, 100=0.06%, 250=0.01% lat (usec): 500=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.04%, 50=0.07% lat (msec): 100=0.04% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15022 read : io=589824KB, bw=54457KB/s, iops=13614, runt= 10831msec clat (usec): min=6, max=88713, avg=71.79, stdev=1494.64 bw (KB/s) : min=33615, max=70762, per=3.07%, avg=52836.40, stdev=11099.40 cpu : usr=0.96%, sys=23.86%, ctx=586, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.27%, 20=75.39%, 50=24.08%, 100=0.07%, 250=0.01% lat (usec): 500=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.05%, 50=0.09% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15023 read : io=589824KB, bw=55294KB/s, iops=13823, runt= 10667msec clat (usec): min=7, max=103209, avg=70.85, stdev=1479.80 bw (KB/s) : min=32876, max=81398, per=3.18%, avg=54847.75, stdev=10845.86 cpu : usr=1.20%, sys=23.96%, ctx=534, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.28%, 20=75.40%, 50=24.06%, 100=0.06%, 250=0.01% lat (usec): 500=0.01%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.05%, 50=0.07% lat (msec): 100=0.03%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15024 read : io=589824KB, bw=56410KB/s, iops=14102, runt= 10456msec clat (usec): min=7, max=80050, avg=69.81, stdev=1445.78 bw (KB/s) : min=41952, max=67256, per=3.23%, avg=55545.20, stdev=7831.99 cpu : usr=1.30%, sys=24.29%, ctx=561, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.41%, 20=75.78%, 50=23.55%, 100=0.06%, 250=0.01% lat (usec): 500=0.01%, 750=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.05%, 50=0.08% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15025 read : io=589824KB, bw=55654KB/s, iops=13913, runt= 10598msec clat (usec): min=8, max=103253, avg=70.68, stdev=1480.87 bw (KB/s) : min=38139, max=96048, per=3.21%, avg=55216.50, stdev=14563.43 cpu : usr=0.87%, sys=24.54%, ctx=572, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.05%, 20=75.11%, 50=24.58%, 100=0.06%, 250=0.01% lat (usec): 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.04%, 50=0.09% lat (msec): 100=0.03%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15026 read : io=589824KB, bw=55602KB/s, iops=13900, runt= 10608msec clat (usec): min=5, max=89086, avg=70.17, stdev=1471.90 bw (KB/s) : min=35326, max=73667, per=3.21%, avg=55261.70, stdev=11591.82 cpu : usr=1.13%, sys=24.17%, ctx=586, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.20%, 20=75.54%, 50=24.01%, 100=0.06%, 250=0.01% lat (usec): 500=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.05%, 50=0.08% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15027 read : io=589824KB, bw=55036KB/s, iops=13759, runt= 10717msec clat (usec): min=7, max=88795, avg=71.28, stdev=1463.88 bw (KB/s) : min=31423, max=73432, per=3.15%, avg=54255.00, stdev=9677.69 cpu : usr=1.27%, sys=23.81%, ctx=580, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.21%, 20=75.30%, 50=24.22%, 100=0.07%, 250=0.01% lat (usec): 500=0.01%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.05%, 50=0.08% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15028 read : io=589824KB, bw=54352KB/s, iops=13587, runt= 10852msec clat (usec): min=6, max=103773, avg=71.99, stdev=1463.80 bw (KB/s) : min=31198, max=73016, per=3.05%, avg=52598.50, stdev=8774.86 cpu : usr=1.18%, sys=23.63%, ctx=574, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.03%, 20=75.31%, 50=24.40%, 100=0.06%, 250=0.01% lat (usec): 500=0.01%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.05%, 50=0.09% lat (msec): 100=0.02%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15029 read : io=589824KB, bw=54083KB/s, iops=13520, runt= 10906msec clat (usec): min=7, max=87555, avg=72.68, stdev=1527.42 bw (KB/s) : min=21255, max=89928, per=3.04%, avg=52277.81, stdev=16497.97 cpu : usr=1.39%, sys=23.11%, ctx=519, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.10%, 20=76.10%, 50=23.55%, 100=0.05%, 250=0.01% lat (usec): 500=0.01%, 1000=0.01% lat (msec): 4=0.01%, 10=0.03%, 20=0.04%, 50=0.07%, 100=0.04% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15030 read : io=589824KB, bw=54939KB/s, iops=13734, runt= 10736msec clat (usec): min=6, max=80744, avg=70.94, stdev=1464.85 bw (KB/s) : min=36822, max=70086, per=3.10%, avg=53428.90, stdev=8192.28 cpu : usr=1.27%, sys=23.77%, ctx=568, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.45%, 20=75.16%, 50=24.14%, 100=0.05%, 250=0.01% lat (usec): 500=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.04%, 50=0.09% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15031 read : io=589824KB, bw=54705KB/s, iops=13676, runt= 10782msec clat (usec): min=6, max=94298, avg=72.07, stdev=1505.68 bw (KB/s) : min=36320, max=84456, per=3.14%, avg=54087.24, stdev=11936.48 cpu : usr=1.26%, sys=23.56%, ctx=584, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.24%, 20=76.01%, 50=23.49%, 100=0.05%, 250=0.01% lat (usec): 500=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.03%, 50=0.09% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15032 read : io=589824KB, bw=57809KB/s, iops=14452, runt= 10203msec clat (usec): min=7, max=111058, avg=67.82, stdev=1441.12 bw (KB/s) : min=43303, max=95009, per=3.39%, avg=58345.00, stdev=13489.20 cpu : usr=1.29%, sys=25.21%, ctx=534, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.22%, 20=74.61%, 50=24.94%, 100=0.06%, 250=0.01% lat (usec): 500=0.01% lat (msec): 4=0.01%, 10=0.02%, 20=0.04%, 50=0.08%, 100=0.03% lat (msec): 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15033 read : io=589824KB, bw=58671KB/s, iops=14667, runt= 10053msec clat (usec): min=6, max=95470, avg=67.15, stdev=1421.39 bw (KB/s) : min=39216, max=100352, per=3.34%, avg=57593.84, stdev=12920.94 cpu : usr=1.15%, sys=25.67%, ctx=508, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.27%, 20=74.82%, 50=24.67%, 100=0.05%, 250=0.01% lat (usec): 500=0.01%, 750=0.01%, 1000=0.01% lat (msec): 4=0.01%, 10=0.02%, 20=0.05%, 50=0.07%, 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15034 read : io=589824KB, bw=55902KB/s, iops=13975, runt= 10551msec clat (usec): min=7, max=84490, avg=70.71, stdev=1482.36 bw (KB/s) : min=38756, max=100864, per=3.24%, avg=55786.55, stdev=13164.10 cpu : usr=0.87%, sys=24.68%, ctx=538, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.22%, 20=74.71%, 50=24.82%, 100=0.06%, 250=0.01% lat (usec): 500=0.01% lat (msec): 4=0.01%, 10=0.02%, 20=0.04%, 50=0.09%, 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15035 read : io=589824KB, bw=58025KB/s, iops=14506, runt= 10165msec clat (usec): min=7, max=96359, avg=67.77, stdev=1435.04 bw (KB/s) : min=33432, max=86928, per=3.33%, avg=57302.95, stdev=10993.54 cpu : usr=1.10%, sys=25.34%, ctx=551, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.07%, 20=75.69%, 50=23.99%, 100=0.06%, 250=0.01% lat (usec): 500=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.04%, 50=0.08% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15036 read : io=589824KB, bw=54755KB/s, iops=13688, runt= 10772msec clat (usec): min=7, max=88647, avg=71.52, stdev=1468.05 bw (KB/s) : min=36581, max=72008, per=3.13%, avg=53901.10, stdev=10226.13 cpu : usr=1.52%, sys=23.47%, ctx=606, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.06%, 20=75.45%, 50=24.23%, 100=0.06%, 250=0.01% lat (usec): 500=0.01%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.05%, 50=0.08% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15037 read : io=589824KB, bw=54137KB/s, iops=13534, runt= 10895msec clat (usec): min=6, max=88507, avg=73.04, stdev=1528.58 bw (KB/s) : min=32296, max=67728, per=3.03%, avg=52183.76, stdev=8954.87 cpu : usr=1.10%, sys=23.35%, ctx=515, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.13%, 20=76.34%, 50=23.27%, 100=0.06%, 250=0.01% lat (usec): 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.05%, 50=0.08% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15038 read : io=589824KB, bw=56046KB/s, iops=14011, runt= 10524msec clat (usec): min=7, max=91943, avg=70.13, stdev=1473.17 bw (KB/s) : min=33848, max=77354, per=3.22%, avg=55537.60, stdev=12255.07 cpu : usr=1.14%, sys=24.40%, ctx=538, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.06%, 20=75.48%, 50=24.22%, 100=0.04%, 250=0.01% lat (usec): 500=0.01%, 750=0.01% lat (msec): 4=0.01%, 10=0.03%, 20=0.04%, 50=0.08%, 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15039 read : io=589824KB, bw=54197KB/s, iops=13549, runt= 10883msec clat (usec): min=7, max=84623, avg=71.68, stdev=1476.32 bw (KB/s) : min=26490, max=90568, per=3.11%, avg=53479.62, stdev=15082.87 cpu : usr=1.47%, sys=23.19%, ctx=572, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.22%, 20=75.44%, 50=24.10%, 100=0.05%, 250=0.01% lat (usec): 500=0.01%, 750=0.01% lat (msec): 4=0.01%, 10=0.02%, 20=0.06%, 50=0.08%, 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15040 read : io=589824KB, bw=54771KB/s, iops=13692, runt= 10769msec clat (usec): min=7, max=82818, avg=71.60, stdev=1486.12 bw (KB/s) : min=28837, max=73703, per=3.08%, avg=53078.45, stdev=12342.13 cpu : usr=1.37%, sys=23.48%, ctx=575, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.61%, 20=75.13%, 50=24.00%, 100=0.06%, 250=0.01% lat (usec): 500=0.01%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.04%, 50=0.08% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15041 read : io=589824KB, bw=54043KB/s, iops=13510, runt= 10914msec clat (usec): min=5, max=99728, avg=73.05, stdev=1514.60 bw (KB/s) : min=37032, max=65250, per=3.01%, avg=51850.71, stdev=7416.72 cpu : usr=1.21%, sys=23.38%, ctx=513, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.29%, 20=75.56%, 50=23.90%, 100=0.05%, 250=0.01% lat (usec): 500=0.01%, 750=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.04%, 50=0.09% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15042 read : io=589824KB, bw=54212KB/s, iops=13552, runt= 10880msec clat (usec): min=7, max=96763, avg=73.06, stdev=1536.18 bw (KB/s) : min=31768, max=63718, per=2.94%, avg=50695.20, stdev=9293.85 cpu : usr=1.21%, sys=23.49%, ctx=531, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.27%, 20=75.11%, 50=24.36%, 100=0.06%, 250=0.01% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.04%, 50=0.08% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15043 read : io=589824KB, bw=55654KB/s, iops=13913, runt= 10598msec clat (usec): min=7, max=83829, avg=70.29, stdev=1471.76 bw (KB/s) : min=34417, max=67792, per=3.20%, avg=55061.55, stdev=9552.81 cpu : usr=1.09%, sys=24.31%, ctx=589, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.03%, 20=75.43%, 50=24.29%, 100=0.06%, 250=0.01% lat (usec): 500=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.04%, 50=0.08% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15044 read : io=589824KB, bw=55222KB/s, iops=13805, runt= 10681msec clat (usec): min=7, max=97309, avg=70.47, stdev=1459.27 bw (KB/s) : min=40883, max=72976, per=3.18%, avg=54839.15, stdev=8402.43 cpu : usr=1.61%, sys=23.60%, ctx=567, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.05%, 20=75.47%, 50=24.22%, 100=0.05%, 250=0.01% lat (usec): 500=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.05%, 50=0.08% lat (msec): 100=0.02% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15045 read : io=589824KB, bw=53860KB/s, iops=13465, runt= 10951msec clat (usec): min=4, max=96307, avg=73.15, stdev=1583.03 bw (KB/s) : min=27001, max=103936, per=3.10%, avg=53376.14, stdev=16508.59 cpu : usr=1.39%, sys=22.87%, ctx=551, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=1.34%, 20=74.85%, 50=23.57%, 100=0.05%, 250=0.01% lat (usec): 500=0.01%, 750=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.04%, 50=0.07% lat (msec): 100=0.04% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15046 read : io=589824KB, bw=55103KB/s, iops=13775, runt= 10704msec clat (usec): min=7, max=128031, avg=71.01, stdev=1518.16 bw (KB/s) : min=34598, max=90672, per=3.21%, avg=55218.48, stdev=12853.06 cpu : usr=1.23%, sys=23.84%, ctx=545, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.40%, 20=75.17%, 50=24.18%, 100=0.05%, 250=0.01% lat (usec): 500=0.01%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.05%, 50=0.07% lat (msec): 100=0.03%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15047 read : io=589824KB, bw=54142KB/s, iops=13535, runt= 10894msec clat (usec): min=6, max=128031, avg=72.70, stdev=1576.15 bw (KB/s) : min=36675, max=94400, per=3.16%, avg=54410.00, stdev=15588.09 cpu : usr=1.10%, sys=23.68%, ctx=546, majf=0, minf=28 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.28%, 20=74.66%, 50=24.80%, 100=0.05%, 250=0.01% lat (usec): 500=0.01%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.05%, 50=0.07% lat (msec): 100=0.03%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15048 read : io=589824KB, bw=57544KB/s, iops=14385, runt= 10250msec clat (usec): min=5, max=122544, avg=68.30, stdev=1431.10 bw (KB/s) : min=32864, max=95541, per=3.39%, avg=58329.37, stdev=14553.76 cpu : usr=1.17%, sys=25.10%, ctx=520, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.61%, 20=74.53%, 50=24.60%, 100=0.06%, 250=0.01% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.05%, 50=0.08% lat (msec): 100=0.03%, 250=0.01% Run status group 0 (all jobs): READ: io=18432MB, aggrb=1682MB/s, minb=55112KB/s, maxb=60139KB/s, mint=10043msec, maxt=10959msec RAIDZ-2 zfs-bench: (g=0): rw=write, bs=512K-512K/512K-512K, ioengine=sync, iodepth=1 Starting 1 process zfs-bench: Laying out IO file(s) (1 file(s) / 18432MB) zfs-bench: (groupid=0, jobs=1): err= 0: pid=15403 write: io=18432MB, bw=154631KB/s, iops=302, runt=122061msec clat (usec): min=178, max=1804K, avg=3308.60, stdev=48906.91 bw (KB/s) : min= 9017, max=630784, per=110.81%, avg=171352.39, stdev=117052.12 cpu : usr=0.14%, sys=10.88%, ctx=1133, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/36864, short=0/0 lat (usec): 250=69.19%, 500=21.62%, 750=0.24%, 1000=0.31% lat (msec): 2=3.58%, 4=3.14%, 10=0.50%, 20=0.40%, 50=0.52% lat (msec): 100=0.09%, 250=0.09%, 500=0.04%, 750=0.04%, 1000=0.14% lat (msec): 2000=0.08% Run status group 0 (all jobs): WRITE: io=18432MB, aggrb=154630KB/s, minb=158341KB/s, maxb=158341KB/s, mint=122061msec, maxt=122061msec zfs-bench: (g=0): rw=read, bs=512K-512K/512K-512K, ioengine=sync, iodepth=1 Starting 1 process zfs-bench: (groupid=0, jobs=1): err= 0: pid=15406 read : io=18432MB, bw=979418KB/s, iops=1912, runt= 19271msec clat (usec): min=502, max=8673, avg=520.95, stdev=46.54 bw (KB/s) : min=939008, max=986112, per=100.05%, avg=979941.05, stdev=7582.93 cpu : usr=0.37%, sys=99.37%, ctx=63, majf=0, minf=153 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=36864/0, short=0/0 lat (usec): 750=99.98%, 1000=0.01% lat (msec): 2=0.01%, 10=0.01% Run status group 0 (all jobs): READ: io=18432MB, aggrb=979418KB/s, minb=979MB/s, maxb=979MB/s, mint=19271msec, maxt=19271msec zfs-bench: (g=0): rw=randwrite, bs=512K-512K/512K-512K, ioengine=sync, iodepth=1 Starting 1 process zfs-bench: (groupid=0, jobs=1): err= 0: pid=15408 write: io=18432MB, bw=93818KB/s, iops=183, runt=201181msec clat (usec): min=182, max=3449K, avg=5453.63, stdev=85535.47 bw (KB/s) : min=32104, max=656455, per=112.13%, avg=105199.01, stdev=76396.61 cpu : usr=0.10%, sys=4.56%, ctx=605, majf=0, minf=31 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/36864, short=0/0 lat (usec): 250=47.30%, 500=51.74%, 750=0.08%, 1000=0.04% lat (msec): 2=0.12%, 4=0.07%, 10=0.21%, 500=0.01%, 750=0.05% lat (msec): 1000=0.12%, 2000=0.23%, >=2000=0.04% Run status group 0 (all jobs): WRITE: io=18432MB, aggrb=93817KB/s, minb=96069KB/s, maxb=96069KB/s, mint=201181msec, maxt=201181msec zfs-bench: (g=0): rw=randread, bs=512K-512K/512K-512K, ioengine=sync, iodepth=1 Starting 1 process zfs-bench: (groupid=0, jobs=1): err= 0: pid=15413 read : io=18432MB, bw=817639KB/s, iops=1596, runt= 23084msec clat (usec): min=519, max=2025, avg=623.43, stdev=14.52 bw (KB/s) : min=804864, max=832512, per=100.07%, avg=818220.52, stdev=7595.59 cpu : usr=0.42%, sys=99.38%, ctx=49, majf=0, minf=151 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=36864/0, short=0/0 lat (usec): 750=99.96%, 1000=0.04% lat (msec): 2=0.01%, 4=0.01% Run status group 0 (all jobs): READ: io=18432MB, aggrb=817638KB/s, minb=837261KB/s, maxb=837261KB/s, mint=23084msec, maxt=23084msec zfs-bench: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1 Starting 1 process zfs-bench: (groupid=0, jobs=1): err= 0: pid=15415 write: io=1824MB, bw=3111KB/s, iops=777, runt=600432msec clat (usec): min=12, max=4830K, avg=1284.34, stdev=41668.65 bw (KB/s) : min= 813, max=16759, per=108.83%, avg=3384.58, stdev=1704.02 cpu : usr=0.13%, sys=1.91%, ctx=836, majf=0, minf=1617 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/466966, short=0/0 lat (usec): 20=28.31%, 50=69.91%, 100=0.62%, 250=0.91%, 500=0.07% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.04%, 10=0.03%, 20=0.01%, 500=0.01% lat (msec): 750=0.01%, 1000=0.03%, 2000=0.07%, >=2000=0.01% Run status group 0 (all jobs): WRITE: io=1824MB, aggrb=3110KB/s, minb=3185KB/s, maxb=3185KB/s, mint=600432msec, maxt=600432msec zfs-bench: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1 Starting 1 process zfs-bench: (groupid=0, jobs=1): err= 0: pid=15419 read : io=18432MB, bw=442230KB/s, iops=110557, runt= 42680msec clat (usec): min=5, max=3469, avg= 7.82, stdev= 2.00 bw (KB/s) : min=423240, max=457368, per=100.28%, avg=443463.81, stdev=8308.32 cpu : usr=15.82%, sys=83.96%, ctx=88, majf=0, minf=323 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=4718592/0, short=0/0 lat (usec): 10=99.64%, 20=0.25%, 50=0.10%, 100=0.01%, 250=0.01% lat (usec): 500=0.01%, 750=0.01% lat (msec): 4=0.01% Run status group 0 (all jobs): READ: io=18432MB, aggrb=442229KB/s, minb=452843KB/s, maxb=452843KB/s, mint=42680msec, maxt=42680msec zfs-bench: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=posixaio, iodepth=32 Starting 1 process zfs-bench: (groupid=0, jobs=1): err= 0: pid=15421 write: io=1010MB, bw=1721KB/s, iops=430, runt=600671msec slat (usec): min=0, max=4091, avg= 0.79, stdev=16.46 clat (usec): min=10, max=3979K, avg=74357.39, stdev=320568.36 bw (KB/s) : min= 764, max=15529, per=114.03%, avg=1962.41, stdev=1266.26 cpu : usr=0.11%, sys=1.04%, ctx=17150, majf=0, minf=2078 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued r/w: total=0/258495, short=0/0 lat (usec): 20=0.01%, 50=0.08%, 100=0.10%, 250=0.12%, 500=1.37% lat (usec): 750=50.11%, 1000=16.71% lat (msec): 2=18.70%, 4=4.77%, 10=1.98%, 20=0.02%, 250=0.04% lat (msec): 500=0.11%, 750=0.48%, 1000=1.45%, 2000=3.44%, >=2000=0.51% Run status group 0 (all jobs): WRITE: io=1010MB, aggrb=1721KB/s, minb=1762KB/s, maxb=1762KB/s, mint=600671msec, maxt=600671msec zfs-bench: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=posixaio, iodepth=32 Starting 1 process zfs-bench: (groupid=0, jobs=1): err= 0: pid=15427 read : io=18432MB, bw=463789KB/s, iops=115947, runt= 40696msec slat (usec): min=0, max=496, avg= 1.00, stdev= 1.89 clat (usec): min=8, max=117871, avg=274.25, stdev=273.41 bw (KB/s) : min=458600, max=471160, per=100.23%, avg=464862.62, stdev=3204.74 cpu : usr=27.69%, sys=95.58%, ctx=472023, majf=0, minf=380 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued r/w: total=4718592/0, short=0/0 lat (usec): 10=0.01%, 20=0.02%, 50=0.19%, 100=0.89%, 250=32.96% lat (usec): 500=64.16%, 750=1.77%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): READ: io=18432MB, aggrb=463789KB/s, minb=474920KB/s, maxb=474920KB/s, mint=40696msec, maxt=40696msec zfs-bench: (g=0): rw=write, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1 ... zfs-bench: (g=0): rw=write, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1 Starting 32 processes zfs-bench: Laying out IO file(s) (1 file(s) / 576MB) zfs-bench: (groupid=0, jobs=1): err= 0: pid=15430 write: io=589824KB, bw=16505KB/s, iops=4126, runt= 35737msec clat (usec): min=11, max=671844, avg=240.91, stdev=4773.35 bw (KB/s) : min= 77, max=29392, per=3.22%, avg=16733.59, stdev=6757.34 cpu : usr=0.44%, sys=19.94%, ctx=2858, majf=0, minf=25 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.71%, 50=59.25%, 100=37.39%, 250=1.74%, 500=0.22% lat (usec): 750=0.02%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.13%, 20=0.13%, 50=0.24% lat (msec): 100=0.07%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15431 write: io=589824KB, bw=17101KB/s, iops=4275, runt= 34491msec clat (usec): min=12, max=629018, avg=232.67, stdev=4477.79 bw (KB/s) : min= 2499, max=35037, per=3.37%, avg=17492.33, stdev=6911.74 cpu : usr=0.56%, sys=20.47%, ctx=2664, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.96%, 50=60.32%, 100=35.96%, 250=1.72%, 500=0.36% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.13%, 20=0.15%, 50=0.24% lat (msec): 100=0.07%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15432 write: io=589824KB, bw=16787KB/s, iops=4196, runt= 35135msec clat (usec): min=14, max=651329, avg=236.89, stdev=4716.49 bw (KB/s) : min= 1467, max=31691, per=3.33%, avg=17306.06, stdev=6763.44 cpu : usr=0.52%, sys=20.27%, ctx=2917, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.58%, 50=59.16%, 100=37.57%, 250=1.75%, 500=0.25% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.13%, 20=0.15%, 50=0.24% lat (msec): 100=0.07%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15433 write: io=589824KB, bw=16547KB/s, iops=4136, runt= 35645msec clat (usec): min=12, max=589687, avg=240.34, stdev=4562.85 bw (KB/s) : min= 1514, max=28262, per=3.23%, avg=16756.98, stdev=5638.47 cpu : usr=0.46%, sys=19.73%, ctx=2740, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=1.68%, 50=59.46%, 100=36.17%, 250=1.73%, 500=0.26% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.13%, 20=0.15%, 50=0.26% lat (msec): 100=0.06%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15434 write: io=589824KB, bw=18500KB/s, iops=4624, runt= 31883msec clat (usec): min=11, max=594007, avg=214.60, stdev=4430.76 bw (KB/s) : min= 376, max=36328, per=3.64%, avg=18902.81, stdev=7288.59 cpu : usr=0.70%, sys=21.83%, ctx=2305, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=1.01%, 50=59.29%, 100=37.40%, 250=1.71%, 500=0.03% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.03%, 10=0.10%, 20=0.12%, 50=0.20% lat (msec): 100=0.06%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15435 write: io=589824KB, bw=16253KB/s, iops=4063, runt= 36291msec clat (usec): min=13, max=711944, avg=244.27, stdev=4753.34 bw (KB/s) : min= 1052, max=34440, per=3.08%, avg=15999.70, stdev=6576.23 cpu : usr=0.54%, sys=19.27%, ctx=3022, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.96%, 50=59.72%, 100=36.81%, 250=1.71%, 500=0.09% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.12%, 20=0.15%, 50=0.25% lat (msec): 100=0.07%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15436 write: io=589824KB, bw=16313KB/s, iops=4078, runt= 36156msec clat (usec): min=11, max=603384, avg=244.01, stdev=4725.09 bw (KB/s) : min= 776, max=38416, per=3.13%, avg=16249.10, stdev=7849.13 cpu : usr=0.46%, sys=19.33%, ctx=2797, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=3.33%, 50=57.45%, 100=36.54%, 250=1.77%, 500=0.21% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.05%, 10=0.13%, 20=0.14%, 50=0.24% lat (msec): 100=0.07%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15437 write: io=589824KB, bw=16390KB/s, iops=4097, runt= 35986msec clat (usec): min=12, max=610820, avg=242.93, stdev=4705.43 bw (KB/s) : min= 1836, max=39470, per=3.23%, avg=16789.91, stdev=7227.86 cpu : usr=0.48%, sys=19.35%, ctx=2831, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.98%, 50=60.67%, 100=35.86%, 250=1.65%, 500=0.15% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.05%, 10=0.13%, 20=0.15%, 50=0.24% lat (msec): 100=0.08%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15438 write: io=589824KB, bw=17199KB/s, iops=4299, runt= 34295msec clat (usec): min=11, max=664993, avg=231.21, stdev=4541.61 bw (KB/s) : min= 1845, max=32704, per=3.35%, avg=17393.97, stdev=6190.05 cpu : usr=0.48%, sys=20.76%, ctx=2658, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.77%, 50=59.53%, 100=36.99%, 250=1.81%, 500=0.26% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.03%, 10=0.12%, 20=0.15%, 50=0.23% lat (msec): 100=0.06%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15439 write: io=589824KB, bw=16306KB/s, iops=4076, runt= 36173msec clat (usec): min=12, max=674556, avg=243.74, stdev=4693.88 bw (KB/s) : min= 927, max=46111, per=3.12%, avg=16218.39, stdev=7584.28 cpu : usr=0.54%, sys=19.41%, ctx=2945, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=1.51%, 50=59.04%, 100=36.90%, 250=1.65%, 500=0.20% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.13%, 20=0.15%, 50=0.23% lat (msec): 100=0.07%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15440 write: io=589824KB, bw=16267KB/s, iops=4066, runt= 36259msec clat (usec): min=11, max=627181, avg=244.02, stdev=4878.45 bw (KB/s) : min= 629, max=35992, per=3.10%, avg=16119.34, stdev=6575.91 cpu : usr=0.42%, sys=19.54%, ctx=2673, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.92%, 50=59.71%, 100=36.78%, 250=1.72%, 500=0.18% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.15%, 20=0.13%, 50=0.24% lat (msec): 100=0.07%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15441 write: io=589824KB, bw=16537KB/s, iops=4134, runt= 35666msec clat (usec): min=12, max=642196, avg=240.53, stdev=4672.29 bw (KB/s) : min= 2644, max=34216, per=3.26%, avg=16930.02, stdev=7105.04 cpu : usr=0.46%, sys=20.08%, ctx=2794, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.94%, 50=58.56%, 100=37.72%, 250=1.84%, 500=0.24% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.12%, 20=0.17%, 50=0.25% lat (msec): 100=0.06%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15442 write: io=589824KB, bw=16544KB/s, iops=4135, runt= 35652msec clat (usec): min=11, max=635723, avg=240.19, stdev=4786.00 bw (KB/s) : min= 1249, max=32267, per=3.24%, avg=16830.69, stdev=6654.73 cpu : usr=0.43%, sys=19.95%, ctx=2804, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.98%, 50=59.36%, 100=37.02%, 250=1.74%, 500=0.24% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.11%, 20=0.14%, 50=0.23% lat (msec): 100=0.07%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15443 write: io=589824KB, bw=16399KB/s, iops=4099, runt= 35967msec clat (usec): min=11, max=605401, avg=242.57, stdev=4732.12 bw (KB/s) : min= 1501, max=29888, per=3.23%, avg=16781.28, stdev=6356.58 cpu : usr=0.52%, sys=19.85%, ctx=2810, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.81%, 50=58.84%, 100=37.54%, 250=1.86%, 500=0.25% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.05%, 10=0.13%, 20=0.15%, 50=0.24% lat (msec): 100=0.07%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15444 write: io=589824KB, bw=16409KB/s, iops=4102, runt= 35945msec clat (usec): min=12, max=654326, avg=242.31, stdev=4712.58 bw (KB/s) : min= 661, max=31793, per=3.20%, avg=16642.35, stdev=6540.69 cpu : usr=0.56%, sys=19.78%, ctx=2782, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=1.03%, 50=58.80%, 100=37.39%, 250=1.83%, 500=0.25% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.05%, 10=0.14%, 20=0.14%, 50=0.24% lat (msec): 100=0.07%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15445 write: io=589824KB, bw=16235KB/s, iops=4058, runt= 36330msec clat (usec): min=11, max=634372, avg=245.06, stdev=4708.26 bw (KB/s) : min= 2176, max=37584, per=3.03%, avg=15747.76, stdev=6669.55 cpu : usr=0.56%, sys=19.14%, ctx=3004, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=1.27%, 50=60.25%, 100=35.83%, 250=1.69%, 500=0.21% lat (usec): 750=0.02%, 1000=0.01% lat (msec): 2=0.04%, 4=0.05%, 10=0.14%, 20=0.17%, 50=0.24% lat (msec): 100=0.07%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15446 write: io=589824KB, bw=16687KB/s, iops=4171, runt= 35346msec clat (usec): min=11, max=612038, avg=238.44, stdev=4723.26 bw (KB/s) : min= 1944, max=31800, per=3.26%, avg=16926.62, stdev=6869.40 cpu : usr=0.45%, sys=20.02%, ctx=2906, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=1.33%, 50=59.05%, 100=36.93%, 250=1.80%, 500=0.21% lat (usec): 750=0.02%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.12%, 20=0.15%, 50=0.23% lat (msec): 100=0.06%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15447 write: io=589824KB, bw=16420KB/s, iops=4105, runt= 35920msec clat (usec): min=11, max=583878, avg=242.53, stdev=4593.80 bw (KB/s) : min= 1067, max=28916, per=3.19%, avg=16579.30, stdev=6297.48 cpu : usr=0.35%, sys=19.86%, ctx=2751, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.97%, 50=60.15%, 100=36.08%, 250=1.77%, 500=0.34% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.12%, 20=0.16%, 50=0.24% lat (msec): 100=0.07%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15448 write: io=589824KB, bw=16862KB/s, iops=4215, runt= 34980msec clat (usec): min=12, max=653944, avg=235.87, stdev=4701.00 bw (KB/s) : min= 1323, max=36097, per=3.30%, avg=17138.23, stdev=6916.47 cpu : usr=0.40%, sys=20.16%, ctx=2657, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.80%, 50=60.07%, 100=36.65%, 250=1.67%, 500=0.13% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.03%, 10=0.12%, 20=0.16%, 50=0.24% lat (msec): 100=0.06%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15449 write: io=589824KB, bw=16249KB/s, iops=4062, runt= 36300msec clat (usec): min=12, max=662318, avg=244.05, stdev=4750.57 bw (KB/s) : min= 1538, max=42120, per=3.06%, avg=15893.16, stdev=6768.08 cpu : usr=0.57%, sys=19.04%, ctx=2920, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=1.33%, 50=60.58%, 100=35.52%, 250=1.71%, 500=0.17% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.05%, 10=0.13%, 20=0.13%, 50=0.25% lat (msec): 100=0.07%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15450 write: io=589824KB, bw=17108KB/s, iops=4277, runt= 34476msec clat (usec): min=11, max=569040, avg=232.28, stdev=4528.76 bw (KB/s) : min= 1561, max=45234, per=3.35%, avg=17392.89, stdev=7080.52 cpu : usr=0.59%, sys=20.26%, ctx=2655, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=2.97%, 50=58.10%, 100=36.19%, 250=1.79%, 500=0.28% lat (usec): 750=0.02%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.12%, 20=0.15%, 50=0.24% lat (msec): 100=0.06%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15451 write: io=589824KB, bw=16399KB/s, iops=4099, runt= 35967msec clat (usec): min=11, max=583253, avg=242.23, stdev=4592.45 bw (KB/s) : min= 1093, max=33896, per=3.20%, avg=16633.85, stdev=6938.88 cpu : usr=0.47%, sys=19.31%, ctx=2736, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=1.76%, 50=59.93%, 100=35.80%, 250=1.70%, 500=0.15% lat (usec): 750=0.02%, 1000=0.01% lat (msec): 2=0.02%, 4=0.04%, 10=0.11%, 20=0.13%, 50=0.25% lat (msec): 100=0.08%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15452 write: io=589824KB, bw=16400KB/s, iops=4099, runt= 35965msec clat (usec): min=12, max=654185, avg=242.18, stdev=4669.41 bw (KB/s) : min= 2847, max=38184, per=3.25%, avg=16851.12, stdev=6276.66 cpu : usr=0.50%, sys=19.88%, ctx=2836, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.80%, 50=58.87%, 100=37.53%, 250=1.84%, 500=0.26% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.13%, 20=0.15%, 50=0.25% lat (msec): 100=0.07%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15453 write: io=589824KB, bw=17891KB/s, iops=4472, runt= 32968msec clat (usec): min=11, max=627640, avg=222.11, stdev=4511.63 bw (KB/s) : min= 1288, max=32536, per=3.54%, avg=18373.18, stdev=7067.03 cpu : usr=0.56%, sys=21.49%, ctx=2408, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=2.34%, 50=59.19%, 100=35.47%, 250=1.84%, 500=0.55% lat (usec): 750=0.02%, 1000=0.01% lat (msec): 2=0.02%, 4=0.04%, 10=0.11%, 20=0.13%, 50=0.21% lat (msec): 100=0.06%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15454 write: io=589824KB, bw=16547KB/s, iops=4136, runt= 35646msec clat (usec): min=11, max=598892, avg=240.62, stdev=4547.94 bw (KB/s) : min= 2968, max=46737, per=3.24%, avg=16816.45, stdev=6696.65 cpu : usr=0.55%, sys=19.42%, ctx=2984, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=2.55%, 50=58.59%, 100=36.33%, 250=1.68%, 500=0.14% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.13%, 20=0.16%, 50=0.24% lat (msec): 100=0.07%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15455 write: io=589824KB, bw=16784KB/s, iops=4196, runt= 35141msec clat (usec): min=12, max=637461, avg=236.99, stdev=4627.83 bw (KB/s) : min= 648, max=30096, per=3.29%, avg=17100.26, stdev=6849.90 cpu : usr=0.51%, sys=20.27%, ctx=2750, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.88%, 50=59.28%, 100=37.00%, 250=1.86%, 500=0.27% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.05%, 10=0.13%, 20=0.14%, 50=0.24% lat (msec): 100=0.07%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15456 write: io=589824KB, bw=16300KB/s, iops=4074, runt= 36186msec clat (usec): min=12, max=627181, avg=244.27, stdev=4697.37 bw (KB/s) : min= 2352, max=40448, per=3.11%, avg=16149.45, stdev=6726.09 cpu : usr=0.35%, sys=19.46%, ctx=2992, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=2.32%, 50=58.43%, 100=36.61%, 250=1.76%, 500=0.16% lat (usec): 750=0.02%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.13%, 20=0.15%, 50=0.24% lat (msec): 100=0.08%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15457 write: io=589824KB, bw=16227KB/s, iops=4056, runt= 36348msec clat (usec): min=11, max=686209, avg=245.29, stdev=4886.01 bw (KB/s) : min= 2086, max=39650, per=3.04%, avg=15802.36, stdev=7156.33 cpu : usr=0.67%, sys=19.02%, ctx=2921, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.97%, 50=60.36%, 100=36.08%, 250=1.78%, 500=0.14% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.03%, 10=0.12%, 20=0.15%, 50=0.23% lat (msec): 100=0.08%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15458 write: io=589824KB, bw=17910KB/s, iops=4477, runt= 32932msec clat (usec): min=11, max=605880, avg=222.09, stdev=4523.62 bw (KB/s) : min= 2664, max=37368, per=3.52%, avg=18295.28, stdev=6929.15 cpu : usr=0.51%, sys=21.80%, ctx=2419, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=1.12%, 50=59.18%, 100=36.74%, 250=1.89%, 500=0.47% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.04%, 10=0.10%, 20=0.12%, 50=0.22% lat (msec): 100=0.06%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15459 write: io=589824KB, bw=18181KB/s, iops=4545, runt= 32441msec clat (usec): min=11, max=641203, avg=218.59, stdev=4385.21 bw (KB/s) : min= 1489, max=35225, per=3.62%, avg=18796.07, stdev=7129.86 cpu : usr=0.75%, sys=22.05%, ctx=2622, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.99%, 50=58.86%, 100=37.21%, 250=1.81%, 500=0.52% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.03%, 4=0.03%, 10=0.11%, 20=0.14%, 50=0.21% lat (msec): 100=0.06%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15460 write: io=589824KB, bw=16461KB/s, iops=4115, runt= 35832msec clat (usec): min=13, max=596257, avg=241.62, stdev=4661.63 bw (KB/s) : min= 1296, max=29504, per=3.21%, avg=16686.42, stdev=6251.96 cpu : usr=0.59%, sys=19.74%, ctx=2838, majf=0, minf=28 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=0.56%, 50=58.86%, 100=37.98%, 250=1.81%, 500=0.10% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.03%, 10=0.12%, 20=0.17%, 50=0.24% lat (msec): 100=0.07%, 250=0.01%, 500=0.01%, 750=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15461 write: io=589824KB, bw=18118KB/s, iops=4529, runt= 32555msec clat (usec): min=11, max=627781, avg=219.74, stdev=4558.12 bw (KB/s) : min= 1783, max=42587, per=3.60%, avg=18699.51, stdev=7698.18 cpu : usr=0.50%, sys=21.83%, ctx=2317, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/147456, short=0/0 lat (usec): 20=1.46%, 50=59.24%, 100=36.58%, 250=1.79%, 500=0.31% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.03%, 10=0.13%, 20=0.13%, 50=0.20% lat (msec): 100=0.07%, 250=0.01%, 500=0.01%, 750=0.01% Run status group 0 (all jobs): WRITE: io=18432MB, aggrb=519268KB/s, minb=16616KB/s, maxb=18943KB/s, mint=31883msec, maxt=36348msec zfs-bench: (g=0): rw=read, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1 ... zfs-bench: (g=0): rw=read, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1 Starting 32 processes zfs-bench: (groupid=0, jobs=1): err= 0: pid=15464 read : io=589824KB, bw=55435KB/s, iops=13858, runt= 10640msec clat (usec): min=6, max=83016, avg=71.08, stdev=1522.53 bw (KB/s) : min=28031, max=84760, per=3.12%, avg=54672.40, stdev=15974.25 cpu : usr=1.09%, sys=23.72%, ctx=621, majf=0, minf=25 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.23%, 20=77.84%, 50=21.64%, 100=0.09%, 250=0.01% lat (usec): 500=0.01%, 750=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.03%, 50=0.08% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15465 read : io=589824KB, bw=57443KB/s, iops=14360, runt= 10268msec clat (usec): min=6, max=101418, avg=68.47, stdev=1535.66 bw (KB/s) : min=33662, max=77080, per=3.28%, avg=57444.75, stdev=11103.62 cpu : usr=1.05%, sys=24.54%, ctx=608, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.94%, 20=77.31%, 50=21.50%, 100=0.08%, 250=0.01% lat (usec): 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.03%, 50=0.07% lat (msec): 100=0.03%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15466 read : io=589824KB, bw=55955KB/s, iops=13988, runt= 10541msec clat (usec): min=7, max=83756, avg=70.70, stdev=1521.66 bw (KB/s) : min=36565, max=83808, per=3.17%, avg=55571.20, stdev=14092.63 cpu : usr=0.91%, sys=24.29%, ctx=634, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.88%, 20=77.64%, 50=21.02%, 100=0.07%, 250=0.22% lat (usec): 500=0.01%, 1000=0.01% lat (msec): 4=0.01%, 10=0.02%, 20=0.03%, 50=0.08%, 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15467 read : io=589824KB, bw=58759KB/s, iops=14689, runt= 10038msec clat (usec): min=7, max=111990, avg=67.23, stdev=1493.27 bw (KB/s) : min=35632, max=91418, per=3.36%, avg=59019.47, stdev=13435.51 cpu : usr=1.16%, sys=25.03%, ctx=621, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.52%, 20=78.01%, 50=21.20%, 100=0.10%, 250=0.01% lat (usec): 500=0.01%, 750=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.05%, 50=0.07% lat (msec): 100=0.03%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15468 read : io=589824KB, bw=59344KB/s, iops=14836, runt= 9939msec clat (usec): min=7, max=99424, avg=66.16, stdev=1479.19 bw (KB/s) : min=37695, max=82472, per=3.39%, avg=59505.95, stdev=12603.03 cpu : usr=1.09%, sys=25.48%, ctx=595, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.31%, 20=77.76%, 50=21.67%, 100=0.08%, 250=0.01% lat (usec): 750=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.04%, 50=0.06% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15469 read : io=589824KB, bw=55062KB/s, iops=13765, runt= 10712msec clat (usec): min=7, max=106782, avg=71.17, stdev=1576.99 bw (KB/s) : min=26671, max=104872, per=3.14%, avg=55068.10, stdev=17588.26 cpu : usr=1.27%, sys=23.19%, ctx=654, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.90%, 20=77.77%, 50=21.06%, 100=0.06%, 250=0.03% lat (usec): 500=0.01%, 750=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.04%, 50=0.07% lat (msec): 100=0.04%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15470 read : io=589824KB, bw=57690KB/s, iops=14422, runt= 10224msec clat (usec): min=7, max=115110, avg=68.44, stdev=1515.33 bw (KB/s) : min=43384, max=91952, per=3.31%, avg=58096.79, stdev=11160.31 cpu : usr=1.25%, sys=24.61%, ctx=606, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.08%, 20=77.99%, 50=21.68%, 100=0.07%, 250=0.01% lat (usec): 500=0.01%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.04%, 50=0.06% lat (msec): 100=0.04%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15471 read : io=589824KB, bw=57815KB/s, iops=14453, runt= 10202msec clat (usec): min=7, max=92660, avg=67.73, stdev=1497.17 bw (KB/s) : min=35390, max=72500, per=3.27%, avg=57425.84, stdev=10408.25 cpu : usr=1.37%, sys=24.39%, ctx=577, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.20%, 20=78.45%, 50=21.10%, 100=0.07%, 250=0.01% lat (usec): 500=0.01%, 750=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.04%, 50=0.06% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15472 read : io=589824KB, bw=56254KB/s, iops=14063, runt= 10485msec clat (usec): min=6, max=101910, avg=70.17, stdev=1542.45 bw (KB/s) : min=34630, max=102520, per=3.20%, avg=56123.40, stdev=14902.10 cpu : usr=1.22%, sys=24.04%, ctx=575, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.06%, 20=77.70%, 50=22.00%, 100=0.06%, 250=0.01% lat (usec): 750=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.03%, 50=0.08% lat (msec): 100=0.03%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15473 read : io=589824KB, bw=54954KB/s, iops=13738, runt= 10733msec clat (usec): min=7, max=101304, avg=71.82, stdev=1599.65 bw (KB/s) : min=34008, max=82459, per=3.03%, avg=53094.60, stdev=13338.02 cpu : usr=1.53%, sys=22.92%, ctx=660, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.44%, 20=78.19%, 50=21.10%, 100=0.08%, 250=0.01% lat (usec): 500=0.01% lat (msec): 4=0.01%, 10=0.03%, 20=0.04%, 50=0.07%, 100=0.04% lat (msec): 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15474 read : io=589824KB, bw=59148KB/s, iops=14787, runt= 9972msec clat (usec): min=7, max=112106, avg=66.63, stdev=1503.37 bw (KB/s) : min=40198, max=81646, per=3.38%, avg=59229.00, stdev=10443.60 cpu : usr=1.00%, sys=25.43%, ctx=549, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.10%, 20=78.15%, 50=21.51%, 100=0.07%, 250=0.01% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 4=0.01%, 10=0.02%, 20=0.04%, 50=0.06%, 100=0.03% lat (msec): 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15475 read : io=589824KB, bw=54857KB/s, iops=13714, runt= 10752msec clat (usec): min=7, max=111976, avg=71.66, stdev=1574.49 bw (KB/s) : min=35856, max=91864, per=3.09%, avg=54130.00, stdev=14320.06 cpu : usr=1.00%, sys=23.74%, ctx=640, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.11%, 20=78.18%, 50=21.38%, 100=0.06%, 250=0.07% lat (usec): 500=0.02%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.04%, 50=0.07% lat (msec): 100=0.03%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15476 read : io=589824KB, bw=54995KB/s, iops=13748, runt= 10725msec clat (usec): min=7, max=80978, avg=71.48, stdev=1567.70 bw (KB/s) : min=30272, max=89376, per=3.11%, avg=54598.76, stdev=16250.34 cpu : usr=1.23%, sys=23.35%, ctx=593, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.07%, 20=78.28%, 50=21.39%, 100=0.08%, 250=0.01% lat (usec): 500=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.04%, 50=0.07% lat (msec): 100=0.04% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15477 read : io=589824KB, bw=55279KB/s, iops=13819, runt= 10670msec clat (usec): min=7, max=120070, avg=71.13, stdev=1547.60 bw (KB/s) : min=30679, max=84688, per=3.13%, avg=54834.70, stdev=13491.49 cpu : usr=1.35%, sys=23.24%, ctx=650, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.12%, 20=78.80%, 50=20.82%, 100=0.07%, 250=0.01% lat (usec): 500=0.01%, 750=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.04%, 50=0.07% lat (msec): 100=0.03%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15478 read : io=589824KB, bw=55098KB/s, iops=13774, runt= 10705msec clat (usec): min=7, max=92278, avg=71.54, stdev=1561.98 bw (KB/s) : min=25402, max=91504, per=3.03%, avg=53058.55, stdev=15915.80 cpu : usr=0.86%, sys=23.92%, ctx=592, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.89%, 20=77.02%, 50=21.79%, 100=0.08%, 250=0.03% lat (usec): 500=0.01%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.04%, 50=0.07% lat (msec): 100=0.04% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15479 read : io=589824KB, bw=55707KB/s, iops=13926, runt= 10588msec clat (usec): min=7, max=83606, avg=70.60, stdev=1550.77 bw (KB/s) : min=37507, max=91890, per=3.13%, avg=54973.30, stdev=13704.86 cpu : usr=1.81%, sys=23.08%, ctx=589, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.11%, 20=78.10%, 50=21.54%, 100=0.06%, 250=0.01% lat (usec): 500=0.01%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.04%, 50=0.07% lat (msec): 100=0.04% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15480 read : io=589824KB, bw=56319KB/s, iops=14079, runt= 10473msec clat (usec): min=5, max=100898, avg=69.86, stdev=1529.77 bw (KB/s) : min=30882, max=76139, per=3.21%, avg=56295.55, stdev=13169.49 cpu : usr=1.26%, sys=23.87%, ctx=674, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.48%, 20=77.81%, 50=21.45%, 100=0.07%, 250=0.01% lat (usec): 500=0.01%, 750=0.01%,1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.04%, 50=0.07% lat (msec): 100=0.03%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15481 read : io=589824KB, bw=58503KB/s, iops=14625, runt= 10082msec clat (usec): min=7, max=103937, avg=66.94, stdev=1520.97 bw (KB/s) : min=30579, max=88784, per=3.36%, avg=58972.26, stdev=15956.98 cpu : usr=1.55%, sys=24.44%, ctx=542, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.25%, 20=78.42%, 50=21.09%, 100=0.07%, 250=0.01% lat (usec): 500=0.01%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.03%, 50=0.07% lat (msec): 100=0.03%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15482 read : io=589824KB, bw=57437KB/s, iops=14359, runt= 10269msec clat (usec): min=7, max=99301, avg=68.46, stdev=1490.73 bw (KB/s) : min=28593, max=124776, per=3.27%, avg=57309.21, stdev=21863.14 cpu : usr=1.17%, sys=24.50%, ctx=713, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.11%, 20=78.32%, 50=21.28%, 100=0.10%, 250=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.04%, 50=0.07% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15483 read : io=589824KB, bw=58346KB/s, iops=14586, runt= 10109msec clat (usec): min=7, max=88493, avg=66.16, stdev=1491.49 bw (KB/s) : min=26355, max=107730, per=3.29%, avg=57734.42, stdev=19169.66 cpu : usr=1.54%, sys=24.61%, ctx=594, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.09%, 20=77.97%, 50=21.70%, 100=0.08%, 250=0.01% lat (usec): 500=0.01%, 750=0.01%, 1000=0.01% lat (msec): 4=0.01%, 10=0.03%, 20=0.03%, 50=0.06%, 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15484 read : io=589824KB, bw=55660KB/s, iops=13914, runt= 10597msec clat (usec): min=6, max=88472, avg=70.70, stdev=1544.55 bw (KB/s) : min=31206, max=82685, per=3.10%, avg=54398.45, stdev=11967.21 cpu : usr=0.87%, sys=24.05%, ctx=612, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.07%, 20=78.34%, 50=21.33%, 100=0.07%, 250=0.01% lat (usec): 500=0.01%, 750=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.03%, 50=0.08% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15485 read : io=589824KB, bw=54842KB/s, iops=13710, runt= 10755msec clat (usec): min=4, max=101231, avg=71.93, stdev=1574.96 bw (KB/s) : min=33729, max=72661, per=2.87%, avg=50253.85, stdev=10611.72 cpu : usr=1.12%, sys=22.69%, ctx=613, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=5.75%, 20=73.65%, 50=20.26%, 100=0.07%, 250=0.08% lat (usec): 500=0.01%, 750=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.04%, 50=0.07% lat (msec): 100=0.04%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15486 read : io=589824KB, bw=55077KB/s, iops=13769, runt= 10709msec clat (usec): min=7, max=112060, avg=71.61, stdev=1575.06 bw (KB/s) : min=34942, max=79223, per=2.99%, avg=52424.60, stdev=14041.69 cpu : usr=1.46%, sys=23.20%, ctx=646, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.14%, 20=78.06%, 50=21.51%, 100=0.09%, 250=0.02% lat (usec): 500=0.01%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.04%, 50=0.07% lat (msec): 100=0.03%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15487 read : io=589824KB, bw=55016KB/s, iops=13753, runt= 10721msec clat (usec): min=7, max=99275, avg=71.73, stdev=1538.50 bw (KB/s) : min=34502, max=67034, per=2.96%, avg=51924.65, stdev=9079.88 cpu : usr=1.38%, sys=23.06%, ctx=614, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.97%, 20=77.62%, 50=21.14%, 100=0.08%, 250=0.01% lat (usec): 500=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.04%, 50=0.08% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15488 read : io=589824KB, bw=58202KB/s, iops=14550, runt= 10134msec clat (usec): min=7, max=94483, avg=67.85, stdev=1526.63 bw (KB/s) : min=41411, max=84819, per=3.30%, avg=57930.11, stdev=12080.19 cpu : usr=1.30%, sys=24.63%, ctx=577, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.57%, 20=78.09%, 50=21.09%, 100=0.09%, 250=0.01% lat (usec): 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.04%, 50=0.06% lat (msec): 100=0.04% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15489 read : io=589824KB, bw=54959KB/s, iops=13739, runt= 10732msec clat (usec): min=5, max=112564, avg=71.27, stdev=1558.47 bw (KB/s) : min=35858, max=68268, per=2.97%, avg=52049.90, stdev=10507.78 cpu : usr=1.19%, sys=23.56%, ctx=652, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.05%, 20=78.17%, 50=21.45%, 100=0.10%, 250=0.04% lat (usec): 500=0.02%, 750=0.01% lat (msec): 4=0.01%, 10=0.02%, 20=0.03%, 50=0.08%, 100=0.03% lat (msec): 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15490 read : io=589824KB, bw=56867KB/s, iops=14216, runt= 10372msec clat (usec): min=5, max=101835, avg=69.27, stdev=1522.70 bw (KB/s) : min=38548, max=88576, per=3.23%, avg=56692.50, stdev=13250.76 cpu : usr=1.43%, sys=24.07%, ctx=623, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.09%, 20=78.06%, 50=21.56%, 100=0.09%, 250=0.03% lat (usec): 500=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.04%, 50=0.07% lat (msec): 100=0.03%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15491 read : io=589824KB, bw=58254KB/s, iops=14563, runt= 10125msec clat (usec): min=7, max=86630, avg=67.31, stdev=1507.01 bw (KB/s) : min=36358, max=81042, per=3.32%, avg=58207.74, stdev=12132.48 cpu : usr=1.62%, sys=24.10%, ctx=566, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=1.04%, 20=78.47%, 50=20.24%, 100=0.08%, 250=0.01% lat (usec): 500=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.03%, 50=0.06% lat (msec): 100=0.04% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15492 read : io=589824KB, bw=55006KB/s, iops=13751, runt= 10723msec clat (usec): min=5, max=112615, avg=71.93, stdev=1562.61 bw (KB/s) : min=35163, max=71112, per=2.97%, avg=52089.45, stdev=10937.48 cpu : usr=1.16%, sys=23.43%, ctx=756, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.48%, 20=78.00%, 50=21.26%, 100=0.08%, 250=0.01% lat (usec): 750=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.04%, 50=0.07% lat (msec): 100=0.03%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15493 read : io=589824KB, bw=54811KB/s, iops=13702, runt= 10761msec clat (usec): min=5, max=112038, avg=71.90, stdev=1561.91 bw (KB/s) : min=35292, max=65876, per=2.96%, avg=51850.65, stdev=8528.56 cpu : usr=1.04%, sys=23.38%, ctx=633, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=1.35%, 20=77.17%, 50=21.19%, 100=0.09%, 250=0.02% lat (usec): 500=0.01%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.04%, 50=0.07% lat (msec): 100=0.03%, 250=0.01% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15494 read : io=589824KB, bw=55508KB/s, iops=13876, runt= 10626msec clat (usec): min=7, max=88123, avg=70.86, stdev=1519.39 bw (KB/s) : min=33687, max=80608, per=3.11%, avg=54568.65, stdev=12835.30 cpu : usr=1.43%, sys=23.45%, ctx=587, majf=0, minf=28 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.10%, 20=78.05%, 50=21.56%, 100=0.11%, 250=0.02% lat (usec): 500=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.03%, 50=0.08% lat (msec): 100=0.03% zfs-bench: (groupid=0, jobs=1): err= 0: pid=15495 read : io=589824KB, bw=55934KB/s, iops=13983, runt= 10545msec clat (usec): min=6, max=101304, avg=70.07, stdev=1499.91 bw (KB/s) : min=37000, max=81241, per=3.16%, avg=55365.10, stdev=13071.17 cpu : usr=0.99%, sys=24.01%, ctx=653, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147456/0, short=0/0 lat (usec): 10=0.08%, 20=78.26%, 50=21.40%, 100=0.08%, 250=0.01% lat (usec): 500=0.01%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.04%, 50=0.08% lat (msec): 100=0.03%, 250=0.01% Run status group 0 (all jobs): READ: io=18432MB, aggrb=1713MB/s, minb=56126KB/s, maxb=60768KB/s, mint=9939msec, maxt=10761msec