|
Packit Service |
0e769b |
#!/usr/bin/python2.7
|
|
Packit Service |
0e769b |
# Note: this script is python2 and python3 compatible.
|
|
Packit Service |
0e769b |
#
|
|
Packit Service |
0e769b |
# fio_jsonplus_clat2csv
|
|
Packit Service |
0e769b |
#
|
|
Packit Service |
0e769b |
# This script converts fio's json+ completion latency data to CSV format.
|
|
Packit Service |
0e769b |
#
|
|
Packit Service |
0e769b |
# For example:
|
|
Packit Service |
0e769b |
#
|
|
Packit Service |
0e769b |
# Run the following fio jobs:
|
|
Packit Service |
0e769b |
# ../fio --output=fio-jsonplus.output --output-format=json+ --name=test1
|
|
Packit Service |
0e769b |
# --ioengine=null --time_based --runtime=5s --size=1G --rw=randrw
|
|
Packit Service |
0e769b |
# --name=test2 --ioengine=null --time_based --runtime=3s --size=1G
|
|
Packit Service |
0e769b |
# --rw=read --name=test3 --ioengine=null --time_based --runtime=4s
|
|
Packit Service |
0e769b |
# --size=8G --rw=write
|
|
Packit Service |
0e769b |
#
|
|
Packit Service |
0e769b |
# Then run:
|
|
Packit Service |
0e769b |
# fio_jsonplus_clat2csv fio-jsonplus.output fio-latency.csv
|
|
Packit Service |
0e769b |
#
|
|
Packit Service |
0e769b |
# You will end up with the following 3 files
|
|
Packit Service |
0e769b |
#
|
|
Packit Service |
0e769b |
# -rw-r--r-- 1 root root 6467 Jun 27 14:57 fio-latency_job0.csv
|
|
Packit Service |
0e769b |
# -rw-r--r-- 1 root root 3985 Jun 27 14:57 fio-latency_job1.csv
|
|
Packit Service |
0e769b |
# -rw-r--r-- 1 root root 4490 Jun 27 14:57 fio-latency_job2.csv
|
|
Packit Service |
0e769b |
#
|
|
Packit Service |
0e769b |
# fio-latency_job0.csv will look something like:
|
|
Packit Service |
0e769b |
#
|
|
Packit Service |
0e769b |
# clat_nsec, read_count, read_cumulative, read_percentile, write_count,
|
|
Packit Service |
0e769b |
# write_cumulative, write_percentile, trim_count, trim_cumulative,
|
|
Packit Service |
0e769b |
# trim_percentile,
|
|
Packit Service |
0e769b |
# 25, 1, 1, 1.50870705013e-07, , , , , , ,
|
|
Packit Service |
0e769b |
# 26, 12, 13, 1.96131916517e-06, 947, 947, 0.000142955890032, , , ,
|
|
Packit Service |
0e769b |
# 27, 843677, 843690, 0.127288105112, 838347, 839294, 0.126696959629, , , ,
|
|
Packit Service |
0e769b |
# 28, 1877982, 2721672, 0.410620573454, 1870189, 2709483, 0.409014312345, , , ,
|
|
Packit Service |
0e769b |
# 29, 4471, 2726143, 0.411295116376, 7718, 2717201, 0.410179395301, , , ,
|
|
Packit Service |
0e769b |
# 30, 2142885, 4869028, 0.734593687087, 2138164, 4855365, 0.732949340025, , , ,
|
|
Packit Service |
0e769b |
# ...
|
|
Packit Service |
0e769b |
# 2544, , , , 2, 6624404, 0.999997433738, , , ,
|
|
Packit Service |
0e769b |
# 2576, 3, 6628178, 0.99999788781, 4, 6624408, 0.999998037564, , , ,
|
|
Packit Service |
0e769b |
# 2608, 4, 6628182, 0.999998491293, 4, 6624412, 0.999998641391, , , ,
|
|
Packit Service |
0e769b |
# 2640, 3, 6628185, 0.999998943905, 2, 6624414, 0.999998943304, , , ,
|
|
Packit Service |
0e769b |
# 2672, 1, 6628186, 0.999999094776, 3, 6624417, 0.999999396174, , , ,
|
|
Packit Service |
0e769b |
# 2736, 1, 6628187, 0.999999245646, 1, 6624418, 0.99999954713, , , ,
|
|
Packit Service |
0e769b |
# 2768, 2, 6628189, 0.999999547388, 1, 6624419, 0.999999698087, , , ,
|
|
Packit Service |
0e769b |
# 2800, , , , 1, 6624420, 0.999999849043, , , ,
|
|
Packit Service |
0e769b |
# 2832, 1, 6628190, 0.999999698259, , , , , , ,
|
|
Packit Service |
0e769b |
# 4192, 1, 6628191, 0.999999849129, , , , , , ,
|
|
Packit Service |
0e769b |
# 5792, , , , 1, 6624421, 1.0, , , ,
|
|
Packit Service |
0e769b |
# 10304, 1, 6628192, 1.0, , , , , , ,
|
|
Packit Service |
0e769b |
#
|
|
Packit Service |
0e769b |
# The first line says that you had one read IO with 25ns clat,
|
|
Packit Service |
0e769b |
# the cumulative number of read IOs at or below 25ns is 1, and
|
|
Packit Service |
0e769b |
# 25ns is the 0.00001509th percentile for read latency
|
|
Packit Service |
0e769b |
#
|
|
Packit Service |
0e769b |
# The job had 2 write IOs complete in 2544ns,
|
|
Packit Service |
0e769b |
# 6624404 write IOs completed in 2544ns or less,
|
|
Packit Service |
0e769b |
# and this represents the 99.99974th percentile for write latency
|
|
Packit Service |
0e769b |
#
|
|
Packit Service |
0e769b |
# The last line says that one read IO had 10304ns clat,
|
|
Packit Service |
0e769b |
# 6628192 read IOs had 10304ns or shorter clat, and
|
|
Packit Service |
0e769b |
# 10304ns is the 100th percentile for read latency
|
|
Packit Service |
0e769b |
#
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
from __future__ import absolute_import
|
|
Packit Service |
0e769b |
from __future__ import print_function
|
|
Packit Service |
0e769b |
import os
|
|
Packit Service |
0e769b |
import json
|
|
Packit Service |
0e769b |
import argparse
|
|
Packit Service |
0e769b |
import six
|
|
Packit Service |
0e769b |
from six.moves import range
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
def parse_args():
|
|
Packit Service |
0e769b |
parser = argparse.ArgumentParser()
|
|
Packit Service |
0e769b |
parser.add_argument('source',
|
|
Packit Service |
0e769b |
help='fio json+ output file containing completion '
|
|
Packit Service |
0e769b |
'latency data')
|
|
Packit Service |
0e769b |
parser.add_argument('dest',
|
|
Packit Service |
0e769b |
help='destination file stub for latency data in CSV '
|
|
Packit Service |
0e769b |
'format. job number will be appended to filename')
|
|
Packit Service |
0e769b |
args = parser.parse_args()
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
return args
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
def percentile(idx, run_total):
|
|
Packit Service |
0e769b |
total = run_total[len(run_total)-1]
|
|
Packit Service |
0e769b |
if total == 0:
|
|
Packit Service |
0e769b |
return 0
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
return float(run_total[idx]) / total
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
def more_lines(indices, bins):
|
|
Packit Service |
0e769b |
for key, value in six.iteritems(indices):
|
|
Packit Service |
0e769b |
if value < len(bins[key]):
|
|
Packit Service |
0e769b |
return True
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
return False
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
def main():
|
|
Packit Service |
0e769b |
args = parse_args()
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
with open(args.source, 'r') as source:
|
|
Packit Service |
0e769b |
jsondata = json.loads(source.read())
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
for jobnum in range(0, len(jsondata['jobs'])):
|
|
Packit Service |
0e769b |
bins = {}
|
|
Packit Service |
0e769b |
run_total = {}
|
|
Packit Service |
0e769b |
ddir_set = set(['read', 'write', 'trim'])
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
prev_ddir = None
|
|
Packit Service |
0e769b |
for ddir in ddir_set:
|
|
Packit Service |
0e769b |
if 'bins' in jsondata['jobs'][jobnum][ddir]['clat_ns']:
|
|
Packit Service |
0e769b |
bins_loc = 'clat_ns'
|
|
Packit Service |
0e769b |
elif 'bins' in jsondata['jobs'][jobnum][ddir]['lat_ns']:
|
|
Packit Service |
0e769b |
bins_loc = 'lat_ns'
|
|
Packit Service |
0e769b |
else:
|
|
Packit Service |
0e769b |
raise RuntimeError("Latency bins not found. "
|
|
Packit Service |
0e769b |
"Are you sure you are using json+ output?")
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
bins[ddir] = [[int(key), value] for key, value in
|
|
Packit Service |
0e769b |
six.iteritems(jsondata['jobs'][jobnum][ddir][bins_loc]
|
|
Packit Service |
0e769b |
['bins'])]
|
|
Packit Service |
0e769b |
bins[ddir] = sorted(bins[ddir], key=lambda bin: bin[0])
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
run_total[ddir] = [0 for x in range(0, len(bins[ddir]))]
|
|
Packit Service |
0e769b |
if len(bins[ddir]) > 0:
|
|
Packit Service |
0e769b |
run_total[ddir][0] = bins[ddir][0][1]
|
|
Packit Service |
0e769b |
for x in range(1, len(bins[ddir])):
|
|
Packit Service |
0e769b |
run_total[ddir][x] = run_total[ddir][x-1] + \
|
|
Packit Service |
0e769b |
bins[ddir][x][1]
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
stub, ext = os.path.splitext(args.dest)
|
|
Packit Service |
0e769b |
outfile = stub + '_job' + str(jobnum) + ext
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
with open(outfile, 'w') as output:
|
|
Packit Service |
0e769b |
output.write("{0}ec, ".format(bins_loc))
|
|
Packit Service |
0e769b |
ddir_list = list(ddir_set)
|
|
Packit Service |
0e769b |
for ddir in ddir_list:
|
|
Packit Service |
0e769b |
output.write("{0}_count, {0}_cumulative, {0}_percentile, ".
|
|
Packit Service |
0e769b |
format(ddir))
|
|
Packit Service |
0e769b |
output.write("\n")
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
#
|
|
Packit Service |
0e769b |
# Have a counter for each ddir
|
|
Packit Service |
0e769b |
# In each round, pick the shortest remaining duration
|
|
Packit Service |
0e769b |
# and output a line with any values for that duration
|
|
Packit Service |
0e769b |
#
|
|
Packit Service |
0e769b |
indices = {x: 0 for x in ddir_list}
|
|
Packit Service |
0e769b |
while more_lines(indices, bins):
|
|
Packit Service |
0e769b |
min_lat = 17112760320
|
|
Packit Service |
0e769b |
for ddir in ddir_list:
|
|
Packit Service |
0e769b |
if indices[ddir] < len(bins[ddir]):
|
|
Packit Service |
0e769b |
min_lat = min(bins[ddir][indices[ddir]][0], min_lat)
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
output.write("{0}, ".format(min_lat))
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
for ddir in ddir_list:
|
|
Packit Service |
0e769b |
if indices[ddir] < len(bins[ddir]) and \
|
|
Packit Service |
0e769b |
min_lat == bins[ddir][indices[ddir]][0]:
|
|
Packit Service |
0e769b |
count = bins[ddir][indices[ddir]][1]
|
|
Packit Service |
0e769b |
cumulative = run_total[ddir][indices[ddir]]
|
|
Packit Service |
0e769b |
ptile = percentile(indices[ddir], run_total[ddir])
|
|
Packit Service |
0e769b |
output.write("{0}, {1}, {2}, ".format(count,
|
|
Packit Service |
0e769b |
cumulative, ptile))
|
|
Packit Service |
0e769b |
indices[ddir] += 1
|
|
Packit Service |
0e769b |
else:
|
|
Packit Service |
0e769b |
output.write(", , , ")
|
|
Packit Service |
0e769b |
output.write("\n")
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
print("{0} generated".format(outfile))
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
|
|
Packit Service |
0e769b |
if __name__ == '__main__':
|
|
Packit Service |
0e769b |
main()
|