Add auroc to scripts (#28)

* added auroc measure to results. Fixed import.

* fixed issues with the measures

* rearranged subplot orders

* updated data

* updated figures

* update figures

collect_clustered_network.py

@@ -58,6 +58,7 @@ def get_metrics(f, dir, c_dict, b_dict, s_dict, r_dict):
     rho = density(A)
 
     ps = posterior_similarity(samples, A)
+    sps = samplewise_posterior_similarity(samples, A)
     fs = f_score(samples, A)
     fs_norm_random = f_score(samples, A, normalize=True, rho_guess=0.5)
     fs_norm_density = f_score(samples, A, normalize=True, rho_guess=rho)
@@ -70,6 +71,7 @@ def get_metrics(f, dir, c_dict, b_dict, s_dict, r_dict):
     )
     pr = precision(samples, A)
     re = recall(samples, A)
+    ar = auroc(samples, A)
 
     print((i, j, k, l), flush=True)
 
@@ -87,6 +89,8 @@ def get_metrics(f, dir, c_dict, b_dict, s_dict, r_dict):
         fc_norm_density,
         pr,
         re,
+        sps,
+        ar,
     )
 
 
@@ -114,6 +118,8 @@ def get_metrics(f, dir, c_dict, b_dict, s_dict, r_dict):
 fce_norm_density = np.zeros((n_c, n_b, n_s, n_r))
 pr = np.zeros((n_c, n_b, n_s, n_r))
 re = np.zeros((n_c, n_b, n_s, n_r))
+sps = np.zeros((n_c, n_b, n_s, n_r))
+ar = np.zeros((n_c, n_b, n_s, n_r))
 
 arglist = []
 for f in os.listdir(data_dir):
@@ -135,6 +141,8 @@ def get_metrics(f, dir, c_dict, b_dict, s_dict, r_dict):
     metric7,
     metric8,
     metric9,
+    metric10,
+    metric11,
 ) in data:
     ps[i, j, k, l] = metric1
     fs[i, j, k, l] = metric2
@@ -145,6 +153,8 @@ def get_metrics(f, dir, c_dict, b_dict, s_dict, r_dict):
     fce_norm_density[i, j, k, l] = metric7
     pr[i, j, k, l] = metric8
     re[i, j, k, l] = metric9
+    sps[i, j, k, l] = metric10
+    ar[i, j, k, l] = metric11
 
 data = {}
 data["beta"] = list(b_dict)
@@ -158,6 +168,8 @@ def get_metrics(f, dir, c_dict, b_dict, s_dict, r_dict):
 data["fce-norm-density"] = fce_norm_density.tolist()
 data["precision"] = pr.tolist()
 data["recall"] = re.tolist()
+data["sps"] = sps.tolist()
+data["auroc"] = ar.tolist()
 datastring = json.dumps(data)
 
 with open("Data/clustered_network.json", "w") as output_file:

collect_cm.py

@@ -58,6 +58,7 @@ def get_metrics(f, dir, c_dict, b_dict, a_dict, r_dict):
     rho = density(A)
 
     ps = posterior_similarity(samples, A)
+    sps = samplewise_posterior_similarity(samples, A)
     fs = f_score(samples, A)
     fs_norm_random = f_score(samples, A, normalize=True, rho_guess=0.5)
     fs_norm_density = f_score(samples, A, normalize=True, rho_guess=rho)
@@ -68,9 +69,9 @@ def get_metrics(f, dir, c_dict, b_dict, a_dict, r_dict):
     fc_norm_density = fraction_of_correct_entries(
         samples, A, normalize=True, rho_guess=rho
     )
-
     pr = precision(samples, A)
     re = recall(samples, A)
+    ar = auroc(samples, A)
 
     print((i, j, k, l), flush=True)
 
@@ -88,6 +89,8 @@ def get_metrics(f, dir, c_dict, b_dict, a_dict, r_dict):
         fc_norm_density,
         pr,
         re,
+        sps,
+        ar,
     )
 
 
@@ -110,6 +113,8 @@ def get_metrics(f, dir, c_dict, b_dict, a_dict, r_dict):
 fce_norm_density = np.zeros((n_c, n_b, n_a, n_r))
 pr = np.zeros((n_c, n_b, n_a, n_r))
 re = np.zeros((n_c, n_b, n_a, n_r))
+sps = np.zeros((n_c, n_b, n_a, n_r))
+ar = np.zeros((n_c, n_b, n_a, n_r))
 
 arglist = []
 for f in os.listdir(data_dir):
@@ -131,6 +136,8 @@ def get_metrics(f, dir, c_dict, b_dict, a_dict, r_dict):
     metric7,
     metric8,
     metric9,
+    metric10,
+    metric11,
 ) in data:
     ps[i, j, k, l] = metric1
     fs[i, j, k, l] = metric2
@@ -141,6 +148,8 @@ def get_metrics(f, dir, c_dict, b_dict, a_dict, r_dict):
     fce_norm_density[i, j, k, l] = metric7
     pr[i, j, k, l] = metric8
     re[i, j, k, l] = metric9
+    sps[i, j, k, l] = metric10
+    ar[i, j, k, l] = metric11
 
 data = {}
 data["beta"] = list(b_dict)
@@ -154,6 +163,8 @@ def get_metrics(f, dir, c_dict, b_dict, a_dict, r_dict):
 data["fce-norm-density"] = fce_norm_density.tolist()
 data["precision"] = pr.tolist()
 data["recall"] = re.tolist()
+data["sps"] = sps.tolist()
+data["auroc"] = ar.tolist()
 datastring = json.dumps(data)
 
 with open("Data/cm.json", "w") as output_file:

collect_erdos-renyi.py

@@ -58,6 +58,7 @@ def get_metrics(f, dir, c_dict, b_dict, p_dict, r_dict):
     rho = density(A)
 
     ps = posterior_similarity(samples, A)
+    sps = samplewise_posterior_similarity(samples, A)
     fs = f_score(samples, A)
     fs_norm_random = f_score(samples, A, normalize=True, rho_guess=0.5)
     fs_norm_density = f_score(samples, A, normalize=True, rho_guess=rho)
@@ -68,9 +69,9 @@ def get_metrics(f, dir, c_dict, b_dict, p_dict, r_dict):
     fc_norm_density = fraction_of_correct_entries(
         samples, A, normalize=True, rho_guess=rho
     )
-
     pr = precision(samples, A)
     re = recall(samples, A)
+    ar = auroc(samples, A)
 
     print((i, j, k, l), flush=True)
 
@@ -88,6 +89,8 @@ def get_metrics(f, dir, c_dict, b_dict, p_dict, r_dict):
         fc_norm_density,
         pr,
         re,
+        sps,
+        ar,
     )
 
 
@@ -110,6 +113,8 @@ def get_metrics(f, dir, c_dict, b_dict, p_dict, r_dict):
 fce_norm_density = np.zeros((n_c, n_b, n_p, n_r))
 pr = np.zeros((n_c, n_b, n_p, n_r))
 re = np.zeros((n_c, n_b, n_p, n_r))
+sps = np.zeros((n_c, n_b, n_p, n_r))
+ar = np.zeros((n_c, n_b, n_p, n_r))
 
 arglist = []
 for f in os.listdir(data_dir):
@@ -131,6 +136,8 @@ def get_metrics(f, dir, c_dict, b_dict, p_dict, r_dict):
     metric7,
     metric8,
     metric9,
+    metric10,
+    metric11,
 ) in data:
     ps[i, j, k, l] = metric1
     fs[i, j, k, l] = metric2
@@ -141,6 +148,8 @@ def get_metrics(f, dir, c_dict, b_dict, p_dict, r_dict):
     fce_norm_density[i, j, k, l] = metric7
     pr[i, j, k, l] = metric8
     re[i, j, k, l] = metric9
+    sps[i, j, k, l] = metric10
+    ar[i, j, k, l] = metric11
 
 data = {}
 data["beta"] = list(b_dict)
@@ -154,6 +163,8 @@ def get_metrics(f, dir, c_dict, b_dict, p_dict, r_dict):
 data["fce-norm-density"] = fce_norm_density.tolist()
 data["precision"] = pr.tolist()
 data["recall"] = re.tolist()
+data["sps"] = sps.tolist()
+data["auroc"] = ar.tolist()
 datastring = json.dumps(data)
 
 with open("Data/erdos-renyi.json", "w") as output_file:

collect_sbm.py

@@ -58,6 +58,7 @@ def get_metrics(f, dir, c_dict, b_dict, e_dict, r_dict):
     rho = density(A)
 
     ps = posterior_similarity(samples, A)
+    sps = samplewise_posterior_similarity(samples, A)
     fs = f_score(samples, A)
     fs_norm_random = f_score(samples, A, normalize=True, rho_guess=0.5)
     fs_norm_density = f_score(samples, A, normalize=True, rho_guess=rho)
@@ -68,9 +69,9 @@ def get_metrics(f, dir, c_dict, b_dict, e_dict, r_dict):
     fc_norm_density = fraction_of_correct_entries(
         samples, A, normalize=True, rho_guess=rho
     )
-
     pr = precision(samples, A)
     re = recall(samples, A)
+    ar = auroc(samples, A)
 
     print((i, j, k, l), flush=True)
 
@@ -88,6 +89,8 @@ def get_metrics(f, dir, c_dict, b_dict, e_dict, r_dict):
         fc_norm_density,
         pr,
         re,
+        sps,
+        ar,
     )
 
 
@@ -110,6 +113,8 @@ def get_metrics(f, dir, c_dict, b_dict, e_dict, r_dict):
 fce_norm_density = np.zeros((n_c, n_b, n_e, n_r))
 pr = np.zeros((n_c, n_b, n_e, n_r))
 re = np.zeros((n_c, n_b, n_e, n_r))
+sps = np.zeros((n_c, n_b, n_e, n_r))
+ar = np.zeros((n_c, n_b, n_e, n_r))
 
 arglist = []
 for f in os.listdir(data_dir):
@@ -131,6 +136,8 @@ def get_metrics(f, dir, c_dict, b_dict, e_dict, r_dict):
     metric7,
     metric8,
     metric9,
+    metric10,
+    metric11,
 ) in data:
     ps[i, j, k, l] = metric1
     fs[i, j, k, l] = metric2
@@ -141,6 +148,8 @@ def get_metrics(f, dir, c_dict, b_dict, e_dict, r_dict):
     fce_norm_density[i, j, k, l] = metric7
     pr[i, j, k, l] = metric8
     re[i, j, k, l] = metric9
+    sps[i, j, k, l] = metric10
+    ar[i, j, k, l] = metric11
 
 data = {}
 data["beta"] = list(b_dict)
@@ -154,6 +163,8 @@ def get_metrics(f, dir, c_dict, b_dict, e_dict, r_dict):
 data["fce-norm-density"] = fce_norm_density.tolist()
 data["precision"] = pr.tolist()
 data["recall"] = re.tolist()
+data["sps"] = sps.tolist()
+data["auroc"] = ar.tolist()
 datastring = json.dumps(data)
 
 with open("Data/sbm.json", "w") as output_file:

collect_watts-strogatz.py

@@ -58,6 +58,7 @@ def get_metrics(f, dir, c_dict, b_dict, p_dict, r_dict):
     rho = density(A)
 
     ps = posterior_similarity(samples, A)
+    sps = samplewise_posterior_similarity(samples, A)
     fs = f_score(samples, A)
     fs_norm_random = f_score(samples, A, normalize=True, rho_guess=0.5)
     fs_norm_density = f_score(samples, A, normalize=True, rho_guess=rho)
@@ -68,9 +69,9 @@ def get_metrics(f, dir, c_dict, b_dict, p_dict, r_dict):
     fc_norm_density = fraction_of_correct_entries(
         samples, A, normalize=True, rho_guess=rho
     )
-
     pr = precision(samples, A)
     re = recall(samples, A)
+    ar = auroc(samples, A)
 
     print((i, j, k, l), flush=True)
 
@@ -88,6 +89,8 @@ def get_metrics(f, dir, c_dict, b_dict, p_dict, r_dict):
         fc_norm_density,
         pr,
         re,
+        sps,
+        ar,
     )
 
 
@@ -110,6 +113,8 @@ def get_metrics(f, dir, c_dict, b_dict, p_dict, r_dict):
 fce_norm_density = np.zeros((n_c, n_b, n_p, n_r))
 pr = np.zeros((n_c, n_b, n_p, n_r))
 re = np.zeros((n_c, n_b, n_p, n_r))
+sps = np.zeros((n_c, n_b, n_p, n_r))
+ar = np.zeros((n_c, n_b, n_p, n_r))
 
 arglist = []
 for f in os.listdir(data_dir):
@@ -131,6 +136,8 @@ def get_metrics(f, dir, c_dict, b_dict, p_dict, r_dict):
     metric7,
     metric8,
     metric9,
+    metric10,
+    metric11,
 ) in data:
     ps[i, j, k, l] = metric1
     fs[i, j, k, l] = metric2
@@ -141,6 +148,8 @@ def get_metrics(f, dir, c_dict, b_dict, p_dict, r_dict):
     fce_norm_density[i, j, k, l] = metric7
     pr[i, j, k, l] = metric8
     re[i, j, k, l] = metric9
+    sps[i, j, k, l] = metric10
+    ar[i, j, k, l] = metric11
 
 data = {}
 data["beta"] = list(b_dict)
@@ -154,6 +163,8 @@ def get_metrics(f, dir, c_dict, b_dict, p_dict, r_dict):
 data["fce-norm-density"] = fce_norm_density.tolist()
 data["precision"] = pr.tolist()
 data["recall"] = re.tolist()
+data["sps"] = sps.tolist()
+data["auroc"] = ar.tolist()
 datastring = json.dumps(data)
 
 with open("Data/watts-strogatz.json", "w") as output_file:

lcs/measures.py

@@ -1,5 +1,6 @@
 import numpy as np
 from scipy.special import binom
+from sklearn.metrics import roc_auc_score
 
 
 def posterior_similarity(samples, A):
@@ -28,11 +29,20 @@ def samplewise_posterior_similarity(samples, A):
 def f_score(samples, A, normalize=False, rho_guess=0.5):
     p = precision(samples, A)
     r = recall(samples, A)
-    f = 2 * p * r / (p + r)
+
+    if np.isnan(p) or np.isnan(r):
+        f = np.nan
+    else:
+        f = 2 * p * r / (p + r)
+
     if normalize:
         rho = density(A)
         # https://stats.stackexchange.com/questions/390200/what-is-the-baseline-of-the-f1-score-for-a-binary-classifier
-        f_random = 2 * rho * rho_guess / (rho + rho_guess)
+        if rho + rho_guess > 0:
+            f_random = 2 * rho * rho_guess / (rho + rho_guess)
+        else:
+            f_random = 0
+
         return f / f_random
     else:
         return f
@@ -42,14 +52,20 @@ def precision(samples, A):
     Q = samples.mean(axis=0)
     tp = np.sum(Q * A)
     fp = np.sum(Q * (1 - A))
-    return tp / (tp + fp)
+    if tp + fp > 0:
+        return tp / (tp + fp)
+    else:
+        return np.nan
 
 
 def recall(samples, A):
     Q = samples.mean(axis=0)
     tp = np.sum(Q * A)
     fn = np.sum((1 - Q) * A)
-    return tp / (tp + fn)
+    if tp + fn > 0:
+        return tp / (tp + fn)
+    else:
+        return np.nan
 
 
 def fraction_of_correct_entries(samples, A, normalize=False, rho_guess=0.5):
@@ -78,8 +94,11 @@ def hamming_distance(A1, A2):
     return np.sum(np.abs(A1 - A2)) / 2
 
 
-def auroc(samples,A):
+def auroc(samples, A):
+    n = A.shape[0]
     Q = samples.mean(axis=0)
-    A = A.flatten()
-    Q = Q.flatten()
-    return roc_auc_score(A,Q)
+    y_true = A[np.tril_indices(n, -1)]
+    y_score = Q[np.tril_indices(n, -1)]
+    if len(np.unique(y_true)) == 1:
+        return np.nan
+    return roc_auc_score(y_true, y_score)