From 2b5a042adb1491277ba0963bff0f4ce284fae2fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Santanch=C3=A8?= Date: Sat, 20 Jan 2024 13:15:13 -0300 Subject: [PATCH 1/2] feat (report/annotations): report with score --- .../public/editor/annotate/js/metrics.js | 16 ------- .../public/report/js/report-annotations.js | 42 ++++++++++++------- 2 files changed, 27 insertions(+), 31 deletions(-) diff --git a/src/adonisjs/public/editor/annotate/js/metrics.js b/src/adonisjs/public/editor/annotate/js/metrics.js index f043e53c..a1eb3cce 100644 --- a/src/adonisjs/public/editor/annotate/js/metrics.js +++ b/src/adonisjs/public/editor/annotate/js/metrics.js @@ -120,22 +120,6 @@ class AnnotationMetrics { const arc = (max - er == 0) ? ('' + (r - er)) + ('/' + (max - er)) : (r - er) / (max - er) // adjusted ratio of clustering - console.log('\n\n=== Clustering Free Recall ===') - console.log(JSON.stringify(categoriesOrder)) - console.log('--- n = ' + n) - console.log('--- sorted by position') - console.log(JSON.stringify(sortedL)) - console.log('--- c = ' + c) - console.log('--- ni') - console.log(JSON.stringify(nc)) - console.log('--- r = ' + r) - console.log('--- max = ' + max) - console.log('--- E(r) = ' + Math.round(er * 100) / 100) - console.log('--- RR = ' + Math.round(rr * 100) / 100) - console.log('--- MRR = ' + Math.round(mrr * 100) / 100) - console.log('--- DS = ' + Math.round(ds * 100) / 100) - console.log('--- ARC = ' + ((isNaN(arc)) ? arc : Math.round(arc * 100) / 100)) - if (present != null) { present('\n\n=== Clustering Free Recall ===') present(JSON.stringify(categoriesOrder)) diff --git a/src/adonisjs/public/report/js/report-annotations.js b/src/adonisjs/public/report/js/report-annotations.js index 6e98e9b2..4b7a396a 100644 --- a/src/adonisjs/public/report/js/report-annotations.js +++ b/src/adonisjs/public/report/js/report-annotations.js @@ -130,22 +130,13 @@ class ReportManager { const clustering = AnnotationMetrics.i._clusteringFreeRecall(catOrder) - let o1csv = '' - let sep = '' - for (const g of selfOrder.groups) { - o1csv += sep + ReportManager.catList[g[0]-1] + ':' + g[2] - sep = '; ' - } + const o1csv = this._groupsToCSV(selfOrder.groups) + + const o2csv = this._groupsToCSV(selfOrder.ordered) - let o2csv = '' - sep = '' const ordered = {} - for (const g of selfOrder.ordered) { + for (const g of selfOrder.ordered) ordered[ReportManager.catList[g[0]-1]] = g[2] - o2csv += sep + ReportManager.catList[g[0]-1] + ':' + g[2] - sep = '; ' - } - let countCat = '' for (const c of ReportManager.catList) countCat += ',' + (ordered[c] ? ordered[c] : 0) @@ -158,6 +149,16 @@ class ReportManager { `${(ctideas == 0) ? 0 : selfOrder.score / ctideas},${clustering}${countCat},"${o1csv}","${o2csv}"` } + _groupsToCSV (groups) { + let g2csv = '' + let sep = '' + for (const g of groups) { + g2csv += sep + ReportManager.catList[g[0]-1] + ':' + g[2] + sep = '; ' + } + return g2csv + } + async _downloadAnalysis () { const tprefix = document.querySelector('#tprefix').value @@ -242,6 +243,7 @@ class ReportManager { // expand annotations (one class per token) let expanded = [] let t = 0 + const catOrder = [] // to calculate metrics while (t < tokens.length) { const tk = tokens[t] const tCats = Object.keys(tk[3]) @@ -263,8 +265,10 @@ class ReportManager { blocks.push(bl) } if (multiple) { - for (const bl of blocks) + for (const bl of blocks) { expanded = expanded.concat(bl) + catOrder.push([ReportManager.catList.indexOf(bl[0][4])+1, bl[0][1]]) + } } else { // select the longest blocks const selected = [] @@ -276,15 +280,23 @@ class ReportManager { // select a random among biggest const sel = selected[Math.floor(Math.random() * selected.length)] expanded = expanded.concat(sel) + catOrder.push([ReportManager.catList.indexOf(sel[0][4])+1, sel[0][1]]) } } t = last + 1 } + + const selfOrder = AnnotationMetrics.i._selfOrderCount(catOrder) return { doc_id: caseId, text: tt.text, - labels: expanded + labels: expanded, + self_order_score: selfOrder.score, + self_order_score_normalized: (annotations.length == 0) ? 0 : selfOrder.score / annotations.length, + clustering: AnnotationMetrics.i._clusteringFreeRecall(catOrder), + self_order_groups: this._groupsToCSV(selfOrder.groups), + self_order_ordered: this._groupsToCSV(selfOrder.ordered) } } From 7d348c9fb4593d6899251fef9823cd07e6c3910d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Santanch=C3=A8?= Date: Sat, 20 Jan 2024 13:35:02 -0300 Subject: [PATCH 2/2] feat (report/annotations): full report with score --- .../public/report/annotations/index.html | 2 +- .../public/report/js/report-annotations.js | 35 +++++++++++++------ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/adonisjs/public/report/annotations/index.html b/src/adonisjs/public/report/annotations/index.html index a2540daa..b0c5a5f9 100644 --- a/src/adonisjs/public/report/annotations/index.html +++ b/src/adonisjs/public/report/annotations/index.html @@ -104,7 +104,7 @@ - + diff --git a/src/adonisjs/public/report/js/report-annotations.js b/src/adonisjs/public/report/js/report-annotations.js index 4b7a396a..c883127c 100644 --- a/src/adonisjs/public/report/js/report-annotations.js +++ b/src/adonisjs/public/report/js/report-annotations.js @@ -3,7 +3,7 @@ class ReportManager { MessageBus.i.subscribe('report/download', this._downloadAnalysis.bind(this)) MessageBus.i.subscribe('report/bio/single', this._downloadBIO.bind(this)) MessageBus.i.subscribe('report/bio/multiple', this._downloadBIO.bind(this)) - MessageBus.i.subscribe('report/json', this._downloadJSON.bind(this)) + MessageBus.i.subscribe('report/full', this._downloadFull.bind(this)) this._roomId = new URL(document.location).searchParams.get('roomid') } @@ -143,10 +143,18 @@ class ReportManager { const ctcategories = Object.keys(catIndex).length - return `${ctcategories},${ctright},${ctinfright},${ctideas},${ctrightencap},${ctinfrightencap},${ctwrong},${ctwrongencap},` + - `${ctcategories * ctideas},${(ctideas == 0) ? 0 : ctright / ctideas},${(ctideas == 0) ? 0 : ctinfright / ctideas},` + - `${(ctideas == 0) ? 0 : (ctrightencap + ctwrongencap) / ctideas},${selfOrder.score},` + - `${(ctideas == 0) ? 0 : selfOrder.score / ctideas},${clustering}${countCat},"${o1csv}","${o2csv}"` + const ctordernorm = (ctideas == 0) ? 0 : selfOrder.score / ctideas + + return {self_order_score: selfOrder.score, + self_order_score_normalized: ctordernorm, + clustering: clustering, + self_order_groups: o1csv, + self_order_ordered: o2csv, + csv: `${ctcategories},${ctright},${ctinfright},${ctideas},${ctrightencap},${ctinfrightencap},${ctwrong},${ctwrongencap},` + + `${ctcategories * ctideas},${(ctideas == 0) ? 0 : ctright / ctideas},${(ctideas == 0) ? 0 : ctinfright / ctideas},` + + `${(ctideas == 0) ? 0 : (ctrightencap + ctwrongencap) / ctideas},${selfOrder.score},` + + `${ctordernorm},${clustering}${countCat},"${o1csv}","${o2csv}"` + } } _groupsToCSV (groups) { @@ -184,7 +192,7 @@ class ReportManager { table += '"' + c.title + '","' + c.id + '",' const ant = await this._loadAnnotations(c.id) - const metrics = this._calculateMetrics(ant.annotations) + const metrics = this._calculateMetrics(ant.annotations).csv table += `"${ant.organization}","${ant.score}","${ant.year}",` + metrics + '\n' @@ -401,7 +409,7 @@ class ReportManager { * Export JSON */ - async _buildJSON (caseId, annotations) { + async _buildFull (caseId, annotations) { const text = await this._loadCaseText(caseId) const annComp = [] @@ -418,14 +426,21 @@ class ReportManager { ]) } + const metrics = this._calculateMetrics(annotations) + return { doc_id: caseId, text: text, - annotations: annComp + annotations: annComp, + self_order_score: metrics.self_order_score, + self_order_score_normalized: metrics.self_order_score_normalized, + clustering: metrics.clustering, + self_order_groups: metrics.self_order_groups, + self_order_ordered: metrics.self_order_ordered } } - async _downloadJSON () { + async _downloadFull () { const tprefix = document.querySelector('#tprefix').value const cases = await this._requestCases() @@ -434,7 +449,7 @@ class ReportManager { if (cases != null) { for (const c of cases.message) { const ant = await this._loadAnnotations(c.id) - const annJson = await this._buildJSON(c.id, ant.annotations) + const annJson = await this._buildFull(c.id, ant.annotations) table += JSON.stringify(annJson) + '\n' }