-
Notifications
You must be signed in to change notification settings - Fork 6
/
plot_benchmark.py
139 lines (132 loc) · 4.78 KB
/
plot_benchmark.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
import json
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import argparse
ALGORITMS = ['cbs', 'cbs_disjoint']
THEME = 'darkgrid'
def plot_time_area(data, time_limit):
"""
Plot cpu time of the be
"""
keys = list(data.keys())
start,finish = int(keys[0]), int(keys[-1])+2
sns.set_theme(style=THEME)
x_axis = [i for i in range(start,finish,2)]
y_axis = []
for alg in ALGORITMS:
d = []
for i in range(start,finish,2):
sub = data[str(i)][alg]['cpu_time']
sub = [time_limit if x == -1 else x for x in sub]
d.append(sub)
y_axis.append(d)
# d is a list of lists
std_of_mean = [] # standard error of the mean
for alg in [0,1]:
std_of_mean.append([])
for i in range(len(y_axis[alg])):
std_of_mean[alg].append(np.std(y_axis[alg][i], ddof=1) / np.sqrt(np.size(y_axis[alg][i])))
cbs = np.array([np.mean(x) for x in y_axis[0]])
cbs_disjoint = np.array([np.mean(x) for x in y_axis[1]])
data = pd.DataFrame(
data={
'agents': x_axis,
'cbs': cbs,
'cbs_disjoint': cbs_disjoint
}
)
plt = sns.lineplot(data=data.set_index('agents'), linewidth=2.5)
plt.set(xlabel='Number of agents', ylabel='CPU time (s)')
plt.axes.set_xticks(np.arange(start, finish, 2.0))
# CBS area
lower = cbs - std_of_mean[0]; upper = cbs + std_of_mean[0]
plt.plot(x_axis, lower, color='tab:blue', alpha=0.2)
plt.plot(x_axis, upper, color='tab:blue', alpha=0.2)
plt.fill_between(x_axis, lower, upper, color='tab:blue', alpha=0.2)
# CBS disjoint area
lower = cbs_disjoint - std_of_mean[1]; upper = cbs_disjoint + std_of_mean[1]
plt.plot(x_axis, lower, color='tab:orange', alpha=0.2)
plt.plot(x_axis, upper, color='tab:orange', alpha=0.2)
plt.fill_between(x_axis, lower, upper, color='tab:orange', alpha=0.2)
plt.set(title='CPU time with standard error of the mean')
def plot_success_rate(data, time_limit):
"""
Plot the success rate of the benchmark.
"""
keys = list(data.keys())
start,finish = int(keys[0]), int(keys[-1])+1
max_tests = len(data[str(start)]['cbs']['cpu_time'])
sns.set_theme(style=THEME)
x_axis = [i for i in range(start,finish,2)]
d = {}
for alg in ALGORITMS:
d[alg] = []
for agents in data:
success = 0
for sample in data[agents][alg]['cpu_time']:
if(sample != -1):
success += 1
d[alg].append(success)
data = pd.DataFrame(
data={
'agents': x_axis,
'cbs': d['cbs'],
'cbs_disjoint': d['cbs_disjoint']
}
)
plt = sns.lineplot(data=data.set_index('agents'), linewidth=2.5)
plt.set(xlabel='Number of agents', ylabel='Problems solved (target={})'.format(max_tests))
plt.axes.set_xticks(np.arange(start, finish + 1, 2.0))
plt.set(title='Success rate with time limit of {} minutes'.format(time_limit))
def plot_expanded_nodes(data, default_nodes= 1000):
"""
Plot cpu time of the be
"""
keys = list(data.keys())
start,finish = int(keys[0]), int(keys[-1])+2
sns.set_theme(style=THEME)
x_axis = [i for i in range(start,finish,2)]
y_axis = []
for alg in ALGORITMS:
d = []
for i in range(start,finish,2):
sub = data[str(i)][alg]['expanded']
sub = [default_nodes if x == -1 else x for x in sub]
d.append(np.mean(sub))
y_axis.append(d)
data = pd.DataFrame(
data={
'agents': x_axis,
'cbs': y_axis[0],
'cbs_disjoint': y_axis[1]
}
)
plt = sns.lineplot(data=data.set_index('agents'), linewidth=2.5)
plt.set(xlabel='Number of agents', ylabel='Expanded nodes')
plt.axes.set_xticks(np.arange(start, finish, 2.0))
plt.set(title='Mean of expanded nodes')
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Runs various MAPF algorithms')
parser.add_argument('--plot', type=str,
help='Select the plot to be generated, based on benchmark data {random, success}')
args=parser.parse_args()
if args.plot == 'random':
# plot based on many random instances
json_file = open('benchmark/result.json')
data = json.load(json_file)
plot_success_rate(data, 2)
plt.show()
plot_time_area(data, 60*2)
plt.show()
plot_expanded_nodes(data, 500)
plt.show()
if args.plot == 'success':
# plot based of one map with different instances
json_file = open('benchmark/result_success.json')
data = json.load(json_file)
plot_success_rate(data,5)
plt.show()
plot_time_area(data, 60*5)
plt.show()