Figure 2 fixes (#25)

* flip signs on alpha

* add new plots

* added new normalized metrics

* updated data and figs

* fix failing test

* moved measures to new module and added measures

* added unit tests

* get precision and recall as well

* added unit tests

cm.py

@@ -27,7 +27,7 @@
 n = 50
 kmin = 2
 kmax = n - 1
-alpha_list = np.linspace(1.5, 4, n_a)
+alpha_list = np.linspace(-4, -1.5, n_a)
 beta_list = np.linspace(0.0, 1.0, n_b)
 rho0 = 1.0
 gamma = 0.1

collect_clustered_network.py

@@ -55,12 +55,39 @@ def get_metrics(f, dir, c_dict, b_dict, s_dict, r_dict):
     A = np.array(data["A"])
     samples = np.array(data["samples"])
 
+    rho = density(A)
+
     ps = posterior_similarity(samples, A)
-    sps = samplewise_posterior_similarity(samples, A)
+    fs = f_score(samples, A)
+    fs_norm_random = f_score(samples, A, normalize=True, rho_guess=0.5)
+    fs_norm_density = f_score(samples, A, normalize=True, rho_guess=rho)
     fc = fraction_of_correct_entries(samples, A)
+    fc_norm_random = fraction_of_correct_entries(
+        samples, A, normalize=True, rho_guess=0.5
+    )
+    fc_norm_density = fraction_of_correct_entries(
+        samples, A, normalize=True, rho_guess=rho
+    )
+    pr = precision(samples, A)
+    re = recall(samples, A)
+
     print((i, j, k, l), flush=True)
 
-    return i, j, k, l, ps, sps, fc
+    return (
+        i,
+        j,
+        k,
+        l,
+        ps,
+        fs,
+        fs_norm_random,
+        fs_norm_density,
+        fc,
+        fc_norm_random,
+        fc_norm_density,
+        pr,
+        re,
+    )
 
 
 # get number of available cores
@@ -79,26 +106,58 @@ def get_metrics(f, dir, c_dict, b_dict, s_dict, r_dict):
 n_r = len(r_dict)
 
 ps = np.zeros((n_c, n_b, n_s, n_r))
-sps = np.zeros((n_c, n_b, n_s, n_r))
+fs = np.zeros((n_c, n_b, n_s, n_r))
+fs_norm_random = np.zeros((n_c, n_b, n_s, n_r))
+fs_norm_density = np.zeros((n_c, n_b, n_s, n_r))
 fce = np.zeros((n_c, n_b, n_s, n_r))
+fce_norm_random = np.zeros((n_c, n_b, n_s, n_r))
+fce_norm_density = np.zeros((n_c, n_b, n_s, n_r))
+pr = np.zeros((n_c, n_b, n_s, n_r))
+re = np.zeros((n_c, n_b, n_s, n_r))
 
 arglist = []
 for f in os.listdir(data_dir):
     arglist.append((f, data_dir, c_dict, b_dict, s_dict, r_dict))
 
 data = Parallel(n_jobs=n_processes)(delayed(get_metrics)(*arg) for arg in arglist)
 
-for i, j, k, l, pos_sim, s_pos_sim, frac_corr in data:
-    ps[i, j, k, l] = pos_sim
-    sps[i, j, k, l] = s_pos_sim
-    fce[i, j, k, l] = frac_corr
+for (
+    i,
+    j,
+    k,
+    l,
+    metric1,
+    metric2,
+    metric3,
+    metric4,
+    metric5,
+    metric6,
+    metric7,
+    metric8,
+    metric9,
+) in data:
+    ps[i, j, k, l] = metric1
+    fs[i, j, k, l] = metric2
+    fs_norm_random[i, j, k, l] = metric3
+    fs_norm_density[i, j, k, l] = metric4
+    fce[i, j, k, l] = metric5
+    fce_norm_random[i, j, k, l] = metric6
+    fce_norm_density[i, j, k, l] = metric7
+    pr[i, j, k, l] = metric8
+    re[i, j, k, l] = metric9
 
 data = {}
 data["beta"] = list(b_dict)
 data["size"] = list(s_dict)
-data["sps"] = sps.tolist()
 data["ps"] = ps.tolist()
+data["fs"] = fs.tolist()
+data["fs-norm-random"] = fs_norm_random.tolist()
+data["fs-norm-density"] = fs_norm_density.tolist()
 data["fce"] = fce.tolist()
+data["fce-norm-random"] = fce_norm_random.tolist()
+data["fce-norm-density"] = fce_norm_density.tolist()
+data["precision"] = pr.tolist()
+data["recall"] = re.tolist()
 datastring = json.dumps(data)
 
 with open("Data/clustered_network.json", "w") as output_file:

collect_cm.py

@@ -55,12 +55,40 @@ def get_metrics(f, dir, c_dict, b_dict, a_dict, r_dict):
     A = np.array(data["A"])
     samples = np.array(data["samples"])
 
+    rho = density(A)
+
     ps = posterior_similarity(samples, A)
-    sps = samplewise_posterior_similarity(samples, A)
+    fs = f_score(samples, A)
+    fs_norm_random = f_score(samples, A, normalize=True, rho_guess=0.5)
+    fs_norm_density = f_score(samples, A, normalize=True, rho_guess=rho)
     fc = fraction_of_correct_entries(samples, A)
+    fc_norm_random = fraction_of_correct_entries(
+        samples, A, normalize=True, rho_guess=0.5
+    )
+    fc_norm_density = fraction_of_correct_entries(
+        samples, A, normalize=True, rho_guess=rho
+    )
+
+    pr = precision(samples, A)
+    re = recall(samples, A)
+
     print((i, j, k, l), flush=True)
 
-    return i, j, k, l, ps, sps, fc
+    return (
+        i,
+        j,
+        k,
+        l,
+        ps,
+        fs,
+        fs_norm_random,
+        fs_norm_density,
+        fc,
+        fc_norm_random,
+        fc_norm_density,
+        pr,
+        re,
+    )
 
 
 # get number of available cores
@@ -74,26 +102,58 @@ def get_metrics(f, dir, c_dict, b_dict, a_dict, r_dict):
 n_r = len(r_dict)
 
 ps = np.zeros((n_c, n_b, n_a, n_r))
-sps = np.zeros((n_c, n_b, n_a, n_r))
+fs = np.zeros((n_c, n_b, n_a, n_r))
+fs_norm_random = np.zeros((n_c, n_b, n_a, n_r))
+fs_norm_density = np.zeros((n_c, n_b, n_a, n_r))
 fce = np.zeros((n_c, n_b, n_a, n_r))
+fce_norm_random = np.zeros((n_c, n_b, n_a, n_r))
+fce_norm_density = np.zeros((n_c, n_b, n_a, n_r))
+pr = np.zeros((n_c, n_b, n_a, n_r))
+re = np.zeros((n_c, n_b, n_a, n_r))
 
 arglist = []
 for f in os.listdir(data_dir):
     arglist.append((f, data_dir, c_dict, b_dict, a_dict, r_dict))
 
 data = Parallel(n_jobs=n_processes)(delayed(get_metrics)(*arg) for arg in arglist)
 
-for i, j, k, l, pos_sim, s_pos_sim, frac_corr in data:
-    ps[i, j, k, l] = pos_sim
-    sps[i, j, k, l] = s_pos_sim
-    fce[i, j, k, l] = frac_corr
+for (
+    i,
+    j,
+    k,
+    l,
+    metric1,
+    metric2,
+    metric3,
+    metric4,
+    metric5,
+    metric6,
+    metric7,
+    metric8,
+    metric9,
+) in data:
+    ps[i, j, k, l] = metric1
+    fs[i, j, k, l] = metric2
+    fs_norm_random[i, j, k, l] = metric3
+    fs_norm_density[i, j, k, l] = metric4
+    fce[i, j, k, l] = metric5
+    fce_norm_random[i, j, k, l] = metric6
+    fce_norm_density[i, j, k, l] = metric7
+    pr[i, j, k, l] = metric8
+    re[i, j, k, l] = metric9
 
 data = {}
 data["beta"] = list(b_dict)
 data["alpha"] = list(a_dict)
-data["sps"] = sps.tolist()
 data["ps"] = ps.tolist()
+data["fs"] = fs.tolist()
+data["fs-norm-random"] = fs_norm_random.tolist()
+data["fs-norm-density"] = fs_norm_density.tolist()
 data["fce"] = fce.tolist()
+data["fce-norm-random"] = fce_norm_random.tolist()
+data["fce-norm-density"] = fce_norm_density.tolist()
+data["precision"] = pr.tolist()
+data["recall"] = re.tolist()
 datastring = json.dumps(data)
 
 with open("Data/cm.json", "w") as output_file:

collect_erdos-renyi.py

@@ -55,12 +55,40 @@ def get_metrics(f, dir, c_dict, b_dict, p_dict, r_dict):
     A = np.array(data["A"])
     samples = np.array(data["samples"])
 
+    rho = density(A)
+
     ps = posterior_similarity(samples, A)
-    sps = samplewise_posterior_similarity(samples, A)
+    fs = f_score(samples, A)
+    fs_norm_random = f_score(samples, A, normalize=True, rho_guess=0.5)
+    fs_norm_density = f_score(samples, A, normalize=True, rho_guess=rho)
     fc = fraction_of_correct_entries(samples, A)
+    fc_norm_random = fraction_of_correct_entries(
+        samples, A, normalize=True, rho_guess=0.5
+    )
+    fc_norm_density = fraction_of_correct_entries(
+        samples, A, normalize=True, rho_guess=rho
+    )
+
+    pr = precision(samples, A)
+    re = recall(samples, A)
+
     print((i, j, k, l), flush=True)
 
-    return i, j, k, l, ps, sps, fc
+    return (
+        i,
+        j,
+        k,
+        l,
+        ps,
+        fs,
+        fs_norm_random,
+        fs_norm_density,
+        fc,
+        fc_norm_random,
+        fc_norm_density,
+        pr,
+        re,
+    )
 
 
 # get number of available cores
@@ -74,26 +102,58 @@ def get_metrics(f, dir, c_dict, b_dict, p_dict, r_dict):
 n_r = len(r_dict)
 
 ps = np.zeros((n_c, n_b, n_p, n_r))
-sps = np.zeros((n_c, n_b, n_p, n_r))
+fs = np.zeros((n_c, n_b, n_p, n_r))
+fs_norm_random = np.zeros((n_c, n_b, n_p, n_r))
+fs_norm_density = np.zeros((n_c, n_b, n_p, n_r))
 fce = np.zeros((n_c, n_b, n_p, n_r))
+fce_norm_random = np.zeros((n_c, n_b, n_p, n_r))
+fce_norm_density = np.zeros((n_c, n_b, n_p, n_r))
+pr = np.zeros((n_c, n_b, n_p, n_r))
+re = np.zeros((n_c, n_b, n_p, n_r))
 
 arglist = []
 for f in os.listdir(data_dir):
     arglist.append((f, data_dir, c_dict, b_dict, p_dict, r_dict))
 
 data = Parallel(n_jobs=n_processes)(delayed(get_metrics)(*arg) for arg in arglist)
 
-for i, j, k, l, pos_sim, s_pos_sim, frac_corr in data:
-    ps[i, j, k, l] = pos_sim
-    sps[i, j, k, l] = s_pos_sim
-    fce[i, j, k, l] = frac_corr
+for (
+    i,
+    j,
+    k,
+    l,
+    metric1,
+    metric2,
+    metric3,
+    metric4,
+    metric5,
+    metric6,
+    metric7,
+    metric8,
+    metric9,
+) in data:
+    ps[i, j, k, l] = metric1
+    fs[i, j, k, l] = metric2
+    fs_norm_random[i, j, k, l] = metric3
+    fs_norm_density[i, j, k, l] = metric4
+    fce[i, j, k, l] = metric5
+    fce_norm_random[i, j, k, l] = metric6
+    fce_norm_density[i, j, k, l] = metric7
+    pr[i, j, k, l] = metric8
+    re[i, j, k, l] = metric9
 
 data = {}
 data["beta"] = list(b_dict)
 data["p"] = list(p_dict)
-data["sps"] = sps.tolist()
 data["ps"] = ps.tolist()
+data["fs"] = fs.tolist()
+data["fs-norm-random"] = fs_norm_random.tolist()
+data["fs-norm-density"] = fs_norm_density.tolist()
 data["fce"] = fce.tolist()
+data["fce-norm-random"] = fce_norm_random.tolist()
+data["fce-norm-density"] = fce_norm_density.tolist()
+data["precision"] = pr.tolist()
+data["recall"] = re.tolist()
 datastring = json.dumps(data)
 
 with open("Data/erdos-renyi.json", "w") as output_file: