From 93192f2ce3763737e7fb148ad680312a4d8d0859 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Santanch=C3=A8?= Date: Tue, 2 Jan 2024 14:27:38 -0300 Subject: [PATCH 1/4] fix (arc): adjust in wrong formula --- src/adonisjs/public/editor/annotate/js/metrics.js | 4 ++-- src/adonisjs/public/editor/annotate/metrics/test-metrics.js | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/adonisjs/public/editor/annotate/js/metrics.js b/src/adonisjs/public/editor/annotate/js/metrics.js index a8b430bd..7ff7d81f 100644 --- a/src/adonisjs/public/editor/annotate/js/metrics.js +++ b/src/adonisjs/public/editor/annotate/js/metrics.js @@ -107,7 +107,7 @@ class AnnotationMetrics { let er = 0 // expected number of category repetitions for (const cat in nc) er += nc[cat] * nc[cat] - er = er / n - 1 + er = er / (n - 1) const rr = r / (n - 1) // ratio of repetition @@ -125,7 +125,7 @@ class AnnotationMetrics { present(JSON.stringify(sortedL)) present('--- c = ' + c) present('--- ni') - present(nc) + present(JSON.stringify(nc)) present('--- r = ' + r) present('--- max = ' + max) present('--- E(r) = ' + Math.round(er * 100) / 100) diff --git a/src/adonisjs/public/editor/annotate/metrics/test-metrics.js b/src/adonisjs/public/editor/annotate/metrics/test-metrics.js index 4d745ff4..481e0952 100644 --- a/src/adonisjs/public/editor/annotate/metrics/test-metrics.js +++ b/src/adonisjs/public/editor/annotate/metrics/test-metrics.js @@ -31,4 +31,7 @@ present(AnnotationMetrics.i._clusteringFreeRecall( present(AnnotationMetrics.i._clusteringFreeRecall( [[2, 71], [2, 96], [3, 98], [2, 98], [5, 98], [5, 130], [5, 135], [3, 140], [5, 180]], present)) present(AnnotationMetrics.i._clusteringFreeRecall( - [[2, 71], [2, 96], [3, 98], [2, 98], [5, 98], [7,98], [5, 130], [5, 135], [3, 140], [5, 180]], present)) \ No newline at end of file + [[2, 71], [2, 96], [3, 98], [2, 98], [5, 98], [7,98], [5, 130], [5, 135], [3, 140], [5, 180]], present)) + present(AnnotationMetrics.i._clusteringFreeRecall( + [[3, 1], [3, 2], [3, 3]], present + )) \ No newline at end of file From 5d344b624ae0cd2f6c53b3a69c3c1478c30a99c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Santanch=C3=A8?= Date: Wed, 3 Jan 2024 08:31:39 -0300 Subject: [PATCH 2/4] fix (arc): return back to the original (correct) --- .../public/editor/annotate/js/metrics.js | 18 +++++++++++++++++- .../editor/annotate/metrics/test-metrics.js | 18 +++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/adonisjs/public/editor/annotate/js/metrics.js b/src/adonisjs/public/editor/annotate/js/metrics.js index 7ff7d81f..88d03c22 100644 --- a/src/adonisjs/public/editor/annotate/js/metrics.js +++ b/src/adonisjs/public/editor/annotate/js/metrics.js @@ -107,7 +107,7 @@ class AnnotationMetrics { let er = 0 // expected number of category repetitions for (const cat in nc) er += nc[cat] * nc[cat] - er = er / (n - 1) + er = er / n - 1 const rr = r / (n - 1) // ratio of repetition @@ -117,6 +117,22 @@ class AnnotationMetrics { const arc = (r - er) / (max - er) // adjusted ratio of clustering + console.log('\n\n=== Clustering Free Recall ===') + console.log(JSON.stringify(categoriesOrder)) + console.log('--- n = ' + n) + console.log('--- sorted by position') + console.log(JSON.stringify(sortedL)) + console.log('--- c = ' + c) + console.log('--- ni') + console.log(JSON.stringify(nc)) + console.log('--- r = ' + r) + console.log('--- max = ' + max) + console.log('--- E(r) = ' + Math.round(er * 100) / 100) + console.log('--- RR = ' + Math.round(rr * 100) / 100) + console.log('--- MRR = ' + Math.round(mrr * 100) / 100) + console.log('--- DS = ' + Math.round(ds * 100) / 100) + console.log('--- ARC = ' + (Math.round(arc * 100) / 100)) + if (present != null) { present('\n\n=== Clustering Free Recall ===') present(JSON.stringify(categoriesOrder)) diff --git a/src/adonisjs/public/editor/annotate/metrics/test-metrics.js b/src/adonisjs/public/editor/annotate/metrics/test-metrics.js index 481e0952..b198ff3b 100644 --- a/src/adonisjs/public/editor/annotate/metrics/test-metrics.js +++ b/src/adonisjs/public/editor/annotate/metrics/test-metrics.js @@ -32,6 +32,18 @@ present(AnnotationMetrics.i._clusteringFreeRecall( [[2, 71], [2, 96], [3, 98], [2, 98], [5, 98], [5, 130], [5, 135], [3, 140], [5, 180]], present)) present(AnnotationMetrics.i._clusteringFreeRecall( [[2, 71], [2, 96], [3, 98], [2, 98], [5, 98], [7,98], [5, 130], [5, 135], [3, 140], [5, 180]], present)) - present(AnnotationMetrics.i._clusteringFreeRecall( - [[3, 1], [3, 2], [3, 3]], present - )) \ No newline at end of file +present(AnnotationMetrics.i._clusteringFreeRecall( + [[2, 1], [5, 2], [2, 3], [3, 4], [1, 5], [5, 6]], present +)) +present(AnnotationMetrics.i._clusteringFreeRecall( + [[1, 1], [1, 2], [1, 3], [2, 4]], present +)) +present(AnnotationMetrics.i._clusteringFreeRecall( + [[7,1],[3,2],[7,3],[3,4],[2,5],[2,6]], present +)) +present(AnnotationMetrics.i._clusteringFreeRecall( + [[1,1],[1,2],[5,3],[2,4],[8,5],[1,6],[1,7]], present +)) +present(AnnotationMetrics.i._clusteringFreeRecall( + [[5,1], [5,2], [5,3]], present +)) \ No newline at end of file From 245edaa7cac969d9f96b467fe0845763f19ac5d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Santanch=C3=A8?= Date: Thu, 4 Jan 2024 08:14:35 -0300 Subject: [PATCH 3/4] fix (annotator): bug in overlapping annotations --- src/adonisjs/public/editor/annotate/js/annotator.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/adonisjs/public/editor/annotate/js/annotator.js b/src/adonisjs/public/editor/annotate/js/annotator.js index 9c121757..a0cad73e 100644 --- a/src/adonisjs/public/editor/annotate/js/annotator.js +++ b/src/adonisjs/public/editor/annotate/js/annotator.js @@ -211,7 +211,11 @@ class Annotator { for (const r of a.fragments) ranges.push({fragment: r, annot: [a]}) } - ranges.sort((a, b) => a.fragment.start - b.fragment.start) + ranges.sort((a, b) => + (a.fragment.start == b.fragment.start) + ? b.fragment.size - a.fragment.size // bigger first + : a.fragment.start - b.fragment.start + ) for (let r = 0; r < ranges.length - 1; r++) { if ((ranges[r].fragment.start == ranges[r + 1].fragment.start) && (ranges[r].fragment.size == ranges[r + 1].fragment.size)) { From c31d6cc80cea9680dea26e6e9529532657b49727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Santanch=C3=A8?= Date: Thu, 4 Jan 2024 08:49:40 -0300 Subject: [PATCH 4/4] feat (arc): adjust in ARC calculus for overlapping --- src/adonisjs/public/editor/annotate/js/annotator.js | 3 ++- src/adonisjs/public/editor/annotate/js/metrics.js | 11 +++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/adonisjs/public/editor/annotate/js/annotator.js b/src/adonisjs/public/editor/annotate/js/annotator.js index a0cad73e..bd084f2e 100644 --- a/src/adonisjs/public/editor/annotate/js/annotator.js +++ b/src/adonisjs/public/editor/annotate/js/annotator.js @@ -544,7 +544,8 @@ class Annotator { } const ctcategories = Object.keys(catIndex).length - const clustering = Math.round(AnnotationMetrics.i._clusteringFreeRecall(catOrder)*100) / 100 + const cfr = AnnotationMetrics.i._clusteringFreeRecall(catOrder) + const clustering = (isNaN) ? cfr : Math.round(cfr * 100) / 100 if (isAnnotations) { document.querySelector('#memory-scores').innerHTML = diff --git a/src/adonisjs/public/editor/annotate/js/metrics.js b/src/adonisjs/public/editor/annotate/js/metrics.js index 88d03c22..6dc412f0 100644 --- a/src/adonisjs/public/editor/annotate/js/metrics.js +++ b/src/adonisjs/public/editor/annotate/js/metrics.js @@ -87,7 +87,9 @@ class AnnotationMetrics { else nc[cat]++ let nextPos = i + 1 - while (nextPos < sortedL.length && sortedL[nextPos][1] === sortedL[i][1]) + // find next position of the same category or neighbor start + while (nextPos < sortedL.length && + sortedL[nextPos][0] != cat && sortedL[nextPos][1] === sortedL[i][1]) nextPos++ if (nextPos < sortedL.length) { let sp = nextPos @@ -115,7 +117,8 @@ class AnnotationMetrics { const ds = r - er // deviation score - const arc = (r - er) / (max - er) // adjusted ratio of clustering + const arc = + (max - er == 0) ? '0/0' : (r - er) / (max - er) // adjusted ratio of clustering console.log('\n\n=== Clustering Free Recall ===') console.log(JSON.stringify(categoriesOrder)) @@ -131,7 +134,7 @@ class AnnotationMetrics { console.log('--- RR = ' + Math.round(rr * 100) / 100) console.log('--- MRR = ' + Math.round(mrr * 100) / 100) console.log('--- DS = ' + Math.round(ds * 100) / 100) - console.log('--- ARC = ' + (Math.round(arc * 100) / 100)) + console.log('--- ARC = ' + ((isNaN(arc)) ? arc : Math.round(arc * 100) / 100)) if (present != null) { present('\n\n=== Clustering Free Recall ===') @@ -148,7 +151,7 @@ class AnnotationMetrics { present('--- RR = ' + Math.round(rr * 100) / 100) present('--- MRR = ' + Math.round(mrr * 100) / 100) present('--- DS = ' + Math.round(ds * 100) / 100) - present('--- ARC = ' + Math.round(arc * 100) / 100) + present('--- ARC = ' + ((isNaN(arc)) ? arc : Math.round(arc * 100) / 100)) } return arc