Merge pull request #476 from harena-lab/development

Development

src/adonisjs/public/editor/annotate/js/metrics.js

@@ -120,22 +120,6 @@ class AnnotationMetrics {
     const arc = 
       (max - er == 0) ? ('' + (r - er)) + ('/' + (max - er)) : (r - er) / (max - er)  // adjusted ratio of clustering
 
-    console.log('\n\n=== Clustering Free Recall ===')
-    console.log(JSON.stringify(categoriesOrder))
-    console.log('--- n = ' + n)
-    console.log('--- sorted by position')
-    console.log(JSON.stringify(sortedL))
-    console.log('--- c = ' + c)
-    console.log('--- ni')
-    console.log(JSON.stringify(nc))
-    console.log('--- r = ' + r)
-    console.log('--- max = ' + max)
-    console.log('--- E(r) = ' + Math.round(er * 100) / 100)
-    console.log('--- RR = ' + Math.round(rr * 100) / 100)
-    console.log('--- MRR = ' + Math.round(mrr * 100) / 100)
-    console.log('--- DS = ' + Math.round(ds * 100) / 100)
-    console.log('--- ARC = ' + ((isNaN(arc)) ? arc : Math.round(arc * 100) / 100))
-
     if (present != null) {
       present('\n\n=== Clustering Free Recall ===')
       present(JSON.stringify(categoriesOrder))

src/adonisjs/public/report/annotations/index.html

@@ -104,7 +104,7 @@
               <dcc-rest id="harena-room-load-case" bind="harena-room-load-case"
                         subscribe="case/source/get"></dcc-rest>
               <dcc-button label="Export Analysis" xstyle="in" topic="report/download"></dcc-button>
-              <dcc-button label="Export Full Annotations" xstyle="in" topic="report/json"></dcc-button>
+              <dcc-button label="Export Full Annotations" xstyle="in" topic="report/full"></dcc-button>
               <dcc-button label="Export BIO Single" xstyle="in" topic="report/bio/single"></dcc-button>
               <dcc-button label="Export BIO Multiple" xstyle="in" topic="report/bio/multiple"></dcc-button>
             </div>

src/adonisjs/public/report/js/report-annotations.js

@@ -3,7 +3,7 @@ class ReportManager {
     MessageBus.i.subscribe('report/download', this._downloadAnalysis.bind(this))
     MessageBus.i.subscribe('report/bio/single', this._downloadBIO.bind(this))
     MessageBus.i.subscribe('report/bio/multiple', this._downloadBIO.bind(this))
-    MessageBus.i.subscribe('report/json', this._downloadJSON.bind(this))
+    MessageBus.i.subscribe('report/full', this._downloadFull.bind(this))
     this._roomId = new URL(document.location).searchParams.get('roomid')
   }
 
@@ -130,32 +130,41 @@ class ReportManager {
 
     const clustering = AnnotationMetrics.i._clusteringFreeRecall(catOrder)
 
-    let o1csv = ''
-    let sep = ''
-    for (const g of selfOrder.groups) {
-      o1csv += sep + ReportManager.catList[g[0]-1] + ':' + g[2]
-      sep = '; '
-    }
+    const o1csv = this._groupsToCSV(selfOrder.groups)
+
+    const o2csv = this._groupsToCSV(selfOrder.ordered)
 
-    let o2csv = ''
-    sep = ''
     const ordered = {}
-    for (const g of selfOrder.ordered) {
+    for (const g of selfOrder.ordered)
       ordered[ReportManager.catList[g[0]-1]] = g[2]
-      o2csv += sep + ReportManager.catList[g[0]-1] + ':' + g[2]
-      sep = '; '
-    }
-
     let countCat = ''
     for (const c of ReportManager.catList)
       countCat += ',' + (ordered[c] ? ordered[c] : 0)
 
     const ctcategories = Object.keys(catIndex).length
 
-    return `${ctcategories},${ctright},${ctinfright},${ctideas},${ctrightencap},${ctinfrightencap},${ctwrong},${ctwrongencap},` +
-           `${ctcategories * ctideas},${(ctideas == 0) ? 0 : ctright / ctideas},${(ctideas == 0) ? 0 : ctinfright / ctideas},` +
-           `${(ctideas == 0) ? 0 : (ctrightencap + ctwrongencap) / ctideas},${selfOrder.score},` +
-           `${(ctideas == 0) ? 0 : selfOrder.score / ctideas},${clustering}${countCat},"${o1csv}","${o2csv}"`
+    const ctordernorm = (ctideas == 0) ? 0 : selfOrder.score / ctideas
+
+    return {self_order_score: selfOrder.score,
+            self_order_score_normalized: ctordernorm,
+            clustering: clustering,
+            self_order_groups: o1csv,
+            self_order_ordered: o2csv,
+            csv: `${ctcategories},${ctright},${ctinfright},${ctideas},${ctrightencap},${ctinfrightencap},${ctwrong},${ctwrongencap},` +
+                 `${ctcategories * ctideas},${(ctideas == 0) ? 0 : ctright / ctideas},${(ctideas == 0) ? 0 : ctinfright / ctideas},` +
+                 `${(ctideas == 0) ? 0 : (ctrightencap + ctwrongencap) / ctideas},${selfOrder.score},` +
+                 `${ctordernorm},${clustering}${countCat},"${o1csv}","${o2csv}"`
+           }
+  }
+
+  _groupsToCSV (groups) {
+    let g2csv = ''
+    let sep = ''
+    for (const g of groups) {
+      g2csv += sep + ReportManager.catList[g[0]-1] + ':' + g[2]
+      sep = '; '
+    }
+    return g2csv
   }
 
   async _downloadAnalysis () {
@@ -183,7 +192,7 @@ class ReportManager {
         table += '"' + c.title + '","' + c.id + '",'
 
         const ant = await this._loadAnnotations(c.id)
-        const metrics = this._calculateMetrics(ant.annotations)
+        const metrics = this._calculateMetrics(ant.annotations).csv
 
         table += `"${ant.organization}","${ant.score}","${ant.year}",` + 
                  metrics + '\n'
@@ -242,6 +251,7 @@ class ReportManager {
     // expand annotations (one class per token)
     let expanded = []
     let t = 0
+    const catOrder = []  // to calculate metrics
     while (t < tokens.length) {
       const tk = tokens[t]
       const tCats = Object.keys(tk[3])
@@ -263,8 +273,10 @@ class ReportManager {
           blocks.push(bl)
         }
         if (multiple) {
-          for (const bl of blocks)
+          for (const bl of blocks) {
             expanded = expanded.concat(bl)
+            catOrder.push([ReportManager.catList.indexOf(bl[0][4])+1, bl[0][1]])
+          }
         } else {
           // select the longest blocks
           const selected = []
@@ -276,15 +288,23 @@ class ReportManager {
           // select a random among biggest
           const sel = selected[Math.floor(Math.random() * selected.length)]
           expanded = expanded.concat(sel)
+          catOrder.push([ReportManager.catList.indexOf(sel[0][4])+1, sel[0][1]])
         }
       }
       t = last + 1
     }
+
+    const selfOrder = AnnotationMetrics.i._selfOrderCount(catOrder)
 
     return {
       doc_id: caseId,
       text: tt.text,
-      labels: expanded
+      labels: expanded,
+      self_order_score: selfOrder.score,
+      self_order_score_normalized: (annotations.length == 0) ? 0 : selfOrder.score / annotations.length,
+      clustering: AnnotationMetrics.i._clusteringFreeRecall(catOrder),
+      self_order_groups: this._groupsToCSV(selfOrder.groups),
+      self_order_ordered: this._groupsToCSV(selfOrder.ordered)
     }
   }
 
@@ -389,7 +409,7 @@ class ReportManager {
    * Export JSON
   */
 
-  async _buildJSON (caseId, annotations) {
+  async _buildFull (caseId, annotations) {
     const text = await this._loadCaseText(caseId)
     const annComp = []
 
@@ -406,14 +426,21 @@ class ReportManager {
       ])
     }
 
+    const metrics = this._calculateMetrics(annotations)
+
     return {
       doc_id: caseId,
       text: text,
-      annotations: annComp
+      annotations: annComp,
+      self_order_score: metrics.self_order_score,
+      self_order_score_normalized: metrics.self_order_score_normalized,
+      clustering: metrics.clustering,
+      self_order_groups: metrics.self_order_groups,
+      self_order_ordered: metrics.self_order_ordered
     }
   }
 
-  async _downloadJSON () {
+  async _downloadFull () {
     const tprefix = document.querySelector('#tprefix').value
 
     const cases = await this._requestCases()
@@ -422,7 +449,7 @@ class ReportManager {
     if (cases != null) {
       for (const c of cases.message) {
         const ant = await this._loadAnnotations(c.id)
-        const annJson = await this._buildJSON(c.id, ant.annotations)
+        const annJson = await this._buildFull(c.id, ant.annotations)
         table += JSON.stringify(annJson) + '\n'
       }