grayhook
/
1d_anal
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

many

Browse Source
remotes/origin/master
Сергей Маринкевич 4 years ago
parent 12f15a24e5
commit 98e162dfb0
1 changed files with 131 additions and 35 deletions
Show Stats Download Patch File Download Diff File

166
freq_anal.py
Unescape Escape View File

@ -3,6 +3,7 @@
import sys
import os
from configparser import ConfigParser
from configparser import MissingSectionHeaderError
import datetime
import texttable
import statistics
@ -11,12 +12,17 @@ from matplotlib import colors
from matplotlib.ticker import PercentFormatter
VERBOSE = False
DATALOG_PATH = None
OUTPUT_PATH = "./"
FILTER_BY_STDDEVS = 1
DRAW_AVG_PLOT = 0
BOLD_LINES_ON_PLOT = False
__CONTEXT = {
"VERBOSE": False,
"PARSE_CONFIG": False,
"DATALOG_PATH": None,
"OUTPUT_PATH": "./",
"FILTER_BY_STDDEVS": 1,
"DRAW_AVG_PLOT": 0,
"DRAW_STDDEV_PLOT": False,
"BOLD_LINES_ON_PLOT": False,
"VALUE_MULTI": 1000000000,
}
def std_dev(values):
@ -24,17 +30,20 @@ def std_dev(values):
squares_sum = sum([(value - avg)**2 for value in values])
return (squares_sum / (len(values) - 1))**(0.5)
def calc_stats(values):
twentyciles = statistics.quantiles(values, n=20)
return {
"Meta": {
"Generated_from": DATALOG_PATH,
"Generated_at": datetime.datetime.now().strftime("%Y.%m.%d"),
"DRAW_AVG_PLOT": DRAW_AVG_PLOT,
"FILTER_BY_STDDEVS": FILTER_BY_STDDEVS,
"BOLD_LINES_ON_PLOT": BOLD_LINES_ON_PLOT,
},
"Main": {
def calc_avg(values):
avg_plot_buf = []
avg_plot = []
for val in values:
avg_plot_buf.append(val)
if len(avg_plot_buf) > DRAW_AVG_PLOT:
avg_plot_buf.pop(0)
avg_plot.append(statistics.mean(avg_plot_buf))
return avg_plot
def __calc_stats(values):
return {
"MIN": min(values),
"MAX": max(values),
"SPAN": max(values) - min(values),
@ -42,7 +51,19 @@ def calc_stats(values):
"MEDIAN": statistics.median(values),
"MODE": statistics.mode(values),
"STDDEV": std_dev(values),
}, "Percentiles": {
}
def calc_stats(values):
twentyciles = statistics.quantiles(values, n=20)
stats = {
"Meta": {
"Generated_from": DATALOG_PATH,
"Generated_at": datetime.datetime.now().strftime("%Y.%m.%d"),
"Number_of_measurements": len(values),
},
"Params": __CONTEXT,
"Main": __calc_stats(values),
"Percentiles": {
"5%": twentyciles[0],
"10%": twentyciles[1],
"25%": twentyciles[4],
@ -53,6 +74,11 @@ def calc_stats(values):
}
}
if DRAW_AVG_PLOT:
stats.update({"Average": __calc_stats(calc_avg(values))})
return stats
def val_to_text(value):
units = "Sec"
# sec to msec
@ -74,6 +100,20 @@ def val_to_text(value):
return f"{value:+.3f} {units}"
def text_to_val(text):
if text.isdigit():
return int(text)
multi_map = {
"n": 1000000000,
"u": 1000000,
"m": 1000,
"s": 1,
}
return multi_map[text]
def init_table():
table = texttable.Texttable()
table.set_deco(table.HEADER | table.VLINES | table.BORDER)
@ -124,6 +164,10 @@ def do_statistics():
stats = calc_stats(measurements)
print_stats(stats["Main"], "Filtered")
print_stats(stats["Average"], "Average(filtered)")
else:
print_stats(stats["Average"], "Average(non-filtered)")
print_stats(stats["Percentiles"], "Percentiles")
return stats, measurements
@ -132,16 +176,25 @@ def do_plot(stats, measurements):
# Simple plot
linewidth = 1 if BOLD_LINES_ON_PLOT else 0.2
plt.figure(figsize=(15, 5))
plt.plot(range(len(measurements)), [val * 1000000000 for val in measurements], linewidth=linewidth)
if DRAW_STDDEV_PLOT:
stddev = stats["Main"]["STDDEV"] * VALUE_MULTI
mean = stats["Main"]["MEDIAN"] * VALUE_MULTI
edges = [
([mean + 1 * stddev, mean - 1 * stddev], "dotted", "grey"),
([mean + 2 * stddev, mean - 2 * stddev], "dashed", "grey"),
([mean + 3 * stddev, mean - 3 * stddev], "dashdot", "red"),
]
for lines, style, color in edges:
for line in lines:
plt.axhline(y=line, color=color, linewidth=linewidth, linestyle=style)
plt.plot(range(len(measurements)), [val * VALUE_MULTI for val in measurements], linewidth=linewidth)
if DRAW_AVG_PLOT:
avg_plot_buf = []
avg_plot = []
for val in measurements:
avg_plot_buf.append(val)
if len(avg_plot_buf) > DRAW_AVG_PLOT:
avg_plot_buf.pop(0)
avg_plot.append(statistics.mean(avg_plot_buf) * 1000000000)
avg_plot = [val * VALUE_MULTI for val in calc_avg(measurements)]
plt.plot(range(DRAW_AVG_PLOT//2, len(avg_plot)+DRAW_AVG_PLOT//2), avg_plot, linewidth=linewidth)
plt.grid(axis='both')
plt.title('Generated from ' + DATALOG_PATH)
plt.xlabel('Time, readings')
@ -150,7 +203,7 @@ def do_plot(stats, measurements):
# Simple scatter
plt.figure(figsize=(15, 5))
plt.scatter(range(len(measurements)), [val * 1000000000 for val in measurements], s=2)
plt.scatter(range(len(measurements)), [val * VALUE_MULTI for val in measurements], s=2)
plt.grid(axis='both')
plt.title('Generated from ' + DATALOG_PATH)
plt.xlabel('Time, readings')
@ -162,7 +215,7 @@ def do_plot(stats, measurements):
plt.title('Generated from ' + DATALOG_PATH)
plt.xlabel('Diff, nanoseconds')
plt.ylabel('Proportion, %')
N, bins, patches = axs.hist([val * 1000000000 for val in measurements], bins=21)
N, bins, patches = axs.hist([val * VALUE_MULTI for val in measurements], bins=21)
axs.yaxis.set_major_formatter(PercentFormatter(xmax=len(measurements)))
plt.savefig(os.path.join(OUTPUT_PATH, "histogram.png"), dpi=300)
@ -178,10 +231,8 @@ def eat_param(param, type_class, i):
param()
return 0
g = globals()
if type_class == bool:
g.update({param: True})
__CONTEXT.update({param: True})
return 0
try:
@ -190,7 +241,7 @@ def eat_param(param, type_class, i):
key = sys.argv[i]
raise IndexError(f"{err};\nYou trying to specify parameter with key {key} but haven't place it!")
g.update({param: val})
__CONTEXT.update({param: val})
return 1
@ -201,20 +252,27 @@ def show_help(exitcode=0):
print(f"Usage: {sys.argv[0]} [-hv] -f FILE -o DIRECTORY [-F VALUE]")
print("\t-h, --help\t— show this message")
print("\t-f FILE \t— give a input csv-file")
print("\t-p FILE \t— give a file that contains parameters(as showed in [Params] of stats.txt)")
print("\t-o DIRECTORY \t— give a output directory")
print("\t-F VALUE \t— give a number of STDDEVs to use in filter (`0`(default) means do not filter)")
print("\t-F VALUE \t— give a number of STDDEVs to use in filter (`0` means do not filter, default is `1`)")
print("\t-A VALUE \t— draw avg plot for given count of measurements (`0`(default) means do not draw)")
print("\t-M VALUE \t— multiply measurements to make it one of: `n`(default), `u`, `m`, " + \
"`s`(means `1`),\n\t\t\t or any numeric multiplicator.")
print("\t-b\t\t— bold lines on a plot")
print("\t-s\t\t— draw STDDEV lines on a plot")
print("\t-v\t\t— verbose (show digits in scientific and very long float)")
sys.exit(exitcode)
ARG_MAP = {
"-v": (bool, "VERBOSE"),
"-b": (bool, "BOLD_LINES_ON_PLOT"),
"-s": (bool, "DRAW_STDDEV_PLOT"),
"-f": (str, "DATALOG_PATH"),
"-o": (str, "OUTPUT_PATH"),
"-F": (int, "FILTER_BY_STDDEVS"),
"-A": (int, "DRAW_AVG_PLOT"),
"-M": (text_to_val, "VALUE_MULTI"),
"-p": (str, "PARSE_CONFIG"),
"--help": (None, show_help),
"-h": (None, show_help),
}
@ -226,8 +284,44 @@ def eat_args():
type_class, param = ARG_MAP.get(arg, (eat_unknown, arg))
i += eat_param(param, type_class, i)
def main():
g = globals()
g.update(__CONTEXT)
def startup():
eat_args()
if PARSE_CONFIG:
config = ConfigParser()
try:
config.read(PARSE_CONFIG)
except configparser.MissingSectionHeaderError as err:
print(err)
config_string = "[Params]\n"
with open(PARSE_CONFIG, 'r') as config_file:
config_string += [line for line in config_file.readlines()]
config.read_string(config_string)
try:
for key, val in config["Params"].items():
key = key.upper()
for arg in ARG_MAP.values():
if key == arg[1]:
if arg[0] == bool:
val = True if val == 'True' else False
break
val = arg[0](val)
break
__CONTEXT.update({key: val})
except KeyError as err:
print(err)
print("No [Params] section?")
sys.exit(-3)
g = globals()
g.update(__CONTEXT)
# Do it again to make CLI to prior
eat_args()
if not DATALOG_PATH:
print("Gimme input data!")
show_help(-1)
@ -238,6 +332,9 @@ def main():
print("Gimme directory as output path!")
show_help(-2)
def main():
startup()
stats, measurements = do_statistics()
with open(os.path.join(OUTPUT_PATH, "filtered.csv"), 'w') as output_csv:
@ -249,7 +346,6 @@ def main():
with open(os.path.join(OUTPUT_PATH, "stats.txt"), 'w') as output:
parser.write(output)
do_plot(stats, measurements)
if __name__ == '__main__':

Write Preview
Loading…
Cancel
Save