[readtest]
blocksize=4k
filename=/storage/fio
rw=randread
buffered=0
direct=0
ioengine=libaio
iodepth=32
[writetest]
blocksize=4k
filename=/storage/fio
rw=randwrite
buffered=0
direct=0
ioengine=libaio
iodepth=32

zpool status
pool: storage
state: ONLINE
scan: none requested
config:

NAME STATE READ WRITE CKSUM
storage ONLINE 0 0 0
mirror-0 ONLINE 0 0 0
sda3 ONLINE 0 0 0
sdb3 ONLINE 0 0 0
logs
nvme0n1p7 ONLINE 0 0 0
cache
nvme0n1p1 ONLINE 0 0 0

errors: No known data errors

NAME PROPERTY VALUE SOURCE
storage type filesystem -
storage creation Sun Sep 2 17:30 2018 -
storage used 4.00G -
storage available 3.33T -
storage referenced 4.00G -
storage compressratio 1.00x -
storage mounted yes -
storage quota none default
storage reservation none default
storage recordsize 128K default
storage mountpoint /storage default
storage sharenfs off default
storage checksum on default
storage compression off local
storage atime on default
storage devices on default
storage exec on default
storage setuid on default
storage readonly off default
storage zoned off default
storage snapdir hidden default
storage aclinherit restricted default
storage createtxg 1 -
storage canmount on default
storage xattr on default
storage copies 1 default
storage version 5 -
storage utf8only off -
storage normalization none -
storage casesensitivity sensitive -
storage vscan off default
storage nbmand off default
storage sharesmb off default
storage refquota none default
storage refreservation none default
storage guid 7508850733284762342 -
storage primarycache all default
storage secondarycache all default
storage usedbysnapshots 0B -
storage usedbydataset 4.00G -
storage usedbychildren 552K -
storage usedbyrefreservation 0B -
storage logbias latency default
storage dedup off default
storage mlslabel none default
storage sync standard default
storage dnodesize legacy default
storage refcompressratio 1.00x -
storage written 4.00G -
storage logicalused 4.00G -
storage logicalreferenced 4.00G -
storage volmode default default
storage filesystem_limit none default
storage snapshot_limit none default
storage filesystem_count none default
storage snapshot_count none default
storage snapdev hidden default
storage acltype off default
storage context none default
storage fscontext none default
storage defcontext none default
storage rootcontext none default
storage relatime off default
storage redundant_metadata all default
storage overlay off default

readtest: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=32
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=32
fio-2.16
Starting 2 processes
^Cbs: 1 (f=1): [_(1),w(1)] [32.4% done] [0KB/7859KB/0KB /s] [0/1964/0 iops] [eta 04m:36s]
fio: terminating on signal 2

readtest: (groupid=0, jobs=1): err= 0: pid=28111: Sun Sep 2 17:36:29 2018
read : io=4096.0MB, bw=117738KB/s, iops=29434, runt= 35624msec
slat (usec): min=4, max=101871, avg=26.71, stdev=224.71
clat (usec): min=3, max=103574, avg=1055.01, stdev=1264.95
lat (usec): min=14, max=103689, avg=1082.95, stdev=1285.41
clat percentiles (usec):
| 1.00th=[ 604], 5.00th=[ 700], 10.00th=[ 756], 20.00th=[ 820],
| 30.00th=[ 860], 40.00th=[ 900], 50.00th=[ 948], 60.00th=[ 1004],
| 70.00th=[ 1128], 80.00th=[ 1304], 90.00th=[ 1496], 95.00th=[ 1608],
| 99.00th=[ 1800], 99.50th=[ 1896], 99.90th=[ 2128], 99.95th=[ 2224],
| 99.99th=[101888]
lat (usec) : 4=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01%
lat (usec) : 750=9.43%, 1000=49.95%
lat (msec) : 2=40.38%, 4=0.22%, 250=0.01%
cpu : usr=9.13%, sys=78.20%, ctx=32903, majf=0, minf=40
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
issued : total=r=1048576/w=0/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0
latency : target=0, window=0, percentile=100.00%, depth=32
writetest: (groupid=0, jobs=1): err= 0: pid=28112: Sun Sep 2 17:36:29 2018
write: io=1328.1MB, bw=10265KB/s, iops=2566, runt=132565msec
slat (usec): min=11, max=102140, avg=379.49, stdev=407.43
clat (usec): min=5, max=114059, avg=12082.00, stdev=4575.54
lat (usec): min=496, max=114476, avg=12463.18, stdev=4705.60
clat percentiles (usec):
| 1.00th=[ 1624], 5.00th=[ 1928], 10.00th=[ 7456], 20.00th=[ 9152],
| 30.00th=[10304], 40.00th=[11584], 50.00th=[12736], 60.00th=[13760],
| 70.00th=[14784], 80.00th=[15680], 90.00th=[16512], 95.00th=[17024],
| 99.00th=[17792], 99.50th=[17792], 99.90th=[18048], 99.95th=[18304],
| 99.99th=[112128]
lat (usec) : 10=0.01%, 750=0.01%
lat (msec) : 2=5.83%, 4=2.24%, 10=19.33%, 20=72.56%, 250=0.05%
cpu : usr=1.79%, sys=14.94%, ctx=341834, majf=0, minf=9
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
issued : total=r=0/w=340209/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0
latency : target=0, window=0, percentile=100.00%, depth=32

Run status group 0 (all jobs):
READ: io=4096.0MB, aggrb=117738KB/s, minb=117738KB/s, maxb=117738KB/s, mint=35624msec, maxt=35624msec
WRITE: io=1328.1MB, aggrb=10265KB/s, minb=10265KB/s, maxb=10265KB/s, mint=132565msec, maxt=132565msec

[ 72.722759] flashcache: flashcache-3.1.1 initialized
[ 72.725351] device-mapper: flashcache: flashcache_writeback_create: md_blocks = 102000, md_sectors = 816000

[ 72.725353] device-mapper: flashcache: Allocate 560989KB (22B per) mem for 26111488-entry cache(capacity:102396MB, associativity:512, block size:8 sectors(4KB))
[ 73.142239] flashcache-dbg: cachedev check - speedy speedy
[ 117.788263] EXT4-fs (dm-1): mounted filesystem with ordered data mode. Opts: (null)

cat test.ini
[readtest]
blocksize=4k
filename=/fast/fio
rw=randread
direct=1
buffered=0
ioengine=libaio
iodepth=32
[writetest]
blocksize=4k
filename=/fast/fio
rw=randwrite
direct=1
buffered=0
ioengine=libaio
iodepth=32


fio test.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=32
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=32
fio-2.16
Starting 2 processes
Jobs: 1 (f=1): [_(1),w(1)] [100.0% done] [0KB/173.2MB/0KB /s] [0/44.4K/0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=2462: Mon Sep 3 00:33:56 2018
read : io=4096.0MB, bw=182226KB/s, iops=45556, runt= 23017msec
slat (usec): min=4, max=11240, avg=13.26, stdev=42.79
clat (usec): min=38, max=616232, avg=682.97, stdev=4329.67
lat (usec): min=53, max=616248, avg=697.57, stdev=4329.98
clat percentiles (usec):
| 1.00th=[ 105], 5.00th=[ 245], 10.00th=[ 310], 20.00th=[ 382],
| 30.00th=[ 430], 40.00th=[ 466], 50.00th=[ 498], 60.00th=[ 524],
| 70.00th=[ 556], 80.00th=[ 588], 90.00th=[ 636], 95.00th=[ 708],
| 99.00th=[ 1912], 99.50th=[ 3248], 99.90th=[69120], 99.95th=[102912],
| 99.99th=[177152]
lat (usec) : 50=0.04%, 100=0.89%, 250=4.35%, 500=45.36%, 750=45.46%
lat (usec) : 1000=1.97%
lat (msec) : 2=0.97%, 4=0.60%, 10=0.12%, 20=0.02%, 50=0.09%
lat (msec) : 100=0.09%, 250=0.05%, 500=0.01%, 750=0.01%
cpu : usr=16.45%, sys=72.34%, ctx=326198, majf=0, minf=40
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
issued : total=r=1048576/w=0/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0
latency : target=0, window=0, percentile=100.00%, depth=32
writetest: (groupid=0, jobs=1): err= 0: pid=2463: Mon Sep 3 00:33:56 2018
write: io=4096.0MB, bw=163476KB/s, iops=40869, runt= 25657msec
slat (usec): min=4, max=11240, avg=12.58, stdev=45.75
clat (usec): min=2, max=267463, avg=760.61, stdev=2852.86
lat (usec): min=23, max=267477, avg=774.35, stdev=2853.30
clat percentiles (usec):
| 1.00th=[ 213], 5.00th=[ 410], 10.00th=[ 498], 20.00th=[ 564],
| 30.00th=[ 604], 40.00th=[ 636], 50.00th=[ 660], 60.00th=[ 684],
| 70.00th=[ 708], 80.00th=[ 732], 90.00th=[ 780], 95.00th=[ 828],
| 99.00th=[ 1672], 99.50th=[ 3184], 99.90th=[38144], 99.95th=[66048],
| 99.99th=[132096]
lat (usec) : 4=0.01%, 10=0.01%, 20=0.06%, 50=0.09%, 100=0.12%
lat (usec) : 250=1.19%, 500=8.81%, 750=73.80%, 1000=14.00%
lat (msec) : 2=1.08%, 4=0.50%, 10=0.18%, 20=0.03%, 50=0.07%
lat (msec) : 100=0.06%, 250=0.02%, 500=0.01%
cpu : usr=15.74%, sys=68.48%, ctx=512779, majf=0, minf=9
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
issued : total=r=0/w=1048576/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0
latency : target=0, window=0, percentile=100.00%, depth=32

Run status group 0 (all jobs):
READ: io=4096.0MB, aggrb=182226KB/s, minb=182226KB/s, maxb=182226KB/s, mint=23017msec, maxt=23017msec
WRITE: io=4096.0MB, aggrb=163476KB/s, minb=163476KB/s, maxb=163476KB/s, mint=25657msec, maxt=25657msec

Disk stats (read/write):
dm-1: ios=1048576/1049142, merge=0/0, ticks=264228/121672, in_queue=388304, util=99.71%, aggrios=572400/642282, aggrmerge=58249/231213, aggrticks=131630/253438, aggrin_queue=373106, aggrutil=99.83%
nvme0n1: ios=1142080/1268216, merge=116469/259190, ticks=95976/14100, in_queue=84436, util=89.03%
sdb: ios=2721/16349, merge=30/203237, ticks=167284/492776, in_queue=661776, util=99.83%


dmsetup status
speedy: 0 7423020687 flashcache stats:
reads(2098176), writes(7206128)
read hits(2088953), read hit percent(99)
write hits(2075059) write hit percent(28)
dirty write hits(1786857) dirty write hit percent(24)
replacement(5716), write replacement(26429)
write invalidates(5), read invalidates(3)
pending enqueues(1544), pending inval(1542)
metadata dirties(3153544), metadata cleans(481491)
metadata batch(3282775) metadata ssd writes(352260)
cleanings(481491) fallow cleanings(0)
no room(70067) front merge(12067) back merge(445672)
force_clean_block(0)
disk reads(9224), disk writes(2748643) ssd reads(2570429) ssd writes(5299350)
uncached reads(2267), uncached writes(2267160), uncached IO requeue(1)
disk read errors(0), disk write errors(0) ssd read errors(0) ssd write errors(0)
uncached sequential reads(0), uncached sequential writes(0)
pid_adds(0), pid_dels(0), pid_drops(0) pid_expiry(0)
lru hot blocks(13055744), lru warm blocks(13055744)
lru promotions(0), lru demotions(0)