Merge pull request #470 from harena-lab/development

Development
harena-lab · Jan 4, 2024 · ceb7b87 · ceb7b87
2 parents 183fe49 + 60cf63e
commit ceb7b87
Show file tree

Hide file tree

Showing 3 changed files with 165 additions and 105 deletions.
diff --git a/src/adonisjs/public/editor/annotate/js/metrics.js b/src/adonisjs/public/editor/annotate/js/metrics.js
@@ -118,7 +118,7 @@ class AnnotationMetrics {
     const ds = r - er  // deviation score
 
     const arc = 
-      (max - er == 0) ? '0/0' : (r - er) / (max - er)  // adjusted ratio of clustering
+      (max - er == 0) ? ('' + (r - er)) + ('/' + (max - er)) : (r - er) / (max - er)  // adjusted ratio of clustering
 
     console.log('\n\n=== Clustering Free Recall ===')
     console.log(JSON.stringify(categoriesOrder))

diff --git a/src/adonisjs/public/report/annotations/index.html b/src/adonisjs/public/report/annotations/index.html
@@ -105,6 +105,7 @@
                         subscribe="case/source/get"></dcc-rest>
               <dcc-button label="Export Analysis" xstyle="in" topic="report/download"></dcc-button>
               <dcc-button label="Export BILOU" xstyle="in" topic="report/bilou"></dcc-button>
+              <dcc-button label="Export JSON" xstyle="in" topic="report/json"></dcc-button>
             </div>
 
          </div>

diff --git a/src/adonisjs/public/report/js/report-annotations.js b/src/adonisjs/public/report/js/report-annotations.js
@@ -2,6 +2,7 @@ class ReportManager {
   start () {
     MessageBus.i.subscribe('report/download', this._downloadAnalysis.bind(this))
     MessageBus.i.subscribe('report/bilou', this._downloadBILOU.bind(this))
+    MessageBus.i.subscribe('report/json', this._downloadJSON.bind(this))
     this._roomId = new URL(document.location).searchParams.get('roomid')
   }
 
@@ -78,6 +79,123 @@ class ReportManager {
     return result
   }
 
+  _download (table) {
+    const element = document.createElement('a')
+    element.setAttribute('href',
+      'data:text/plain;charset=utf-8,' + encodeURIComponent(table))
+    element.setAttribute('download', 'annotations.csv')
+    element.style.display = 'none'
+    document.body.appendChild(element)
+    element.click()
+    document.body.removeChild(element)
+  }
+
+  /*
+   * Export Analysis
+  */
+
+  _calculateMetrics (annotations) {
+    let ctideas = 0, ctright = 0, ctinfright = 0
+    let ctwrong = 0, ctrightencap = 0, ctinfrightencap = 0, ctwrongencap = 0
+    const catIndex = {}
+    const catOrder = []
+
+    for (const an of annotations) {
+      ctideas++
+      let start = -1
+      for (const f of an.fragments) {
+        start = (start == -1) ? f.start : start
+      }
+      if (an.categories.includes('encapsulated'))
+        if (an.categories.includes('wrong'))
+          ctwrongencap++
+        else {
+          ctinfrightencap++
+          if (an.categories.includes('right'))
+            ctrightencap++
+        }
+      if (!an.categories.includes('wrong'))
+        ctinfright++
+      for (const c of an.categories) {
+        if (c == 'right') ctright++
+        else if (c == 'wrong') ctwrong++
+        if (ReportManager.catList.includes(c)) {
+          catIndex[c] = c
+          catOrder.push([ReportManager.catList.indexOf(c)+1, start])
+        }
+      }
+    }
+    const selfOrder = AnnotationMetrics.i._selfOrderCount(catOrder)
+
+    const clustering = AnnotationMetrics.i._clusteringFreeRecall(catOrder)
+
+    let o1csv = ''
+    let sep = ''
+    for (const g of selfOrder.groups) {
+      o1csv += sep + ReportManager.catList[g[0]-1] + ':' + g[2]
+      sep = '; '
+    }
+
+    let o2csv = ''
+    sep = ''
+    const ordered = {}
+    for (const g of selfOrder.ordered) {
+      ordered[ReportManager.catList[g[0]-1]] = g[2]
+      o2csv += sep + ReportManager.catList[g[0]-1] + ':' + g[2]
+      sep = '; '
+    }
+
+    let countCat = ''
+    for (const c of ReportManager.catList)
+      countCat += ',' + (ordered[c] ? ordered[c] : 0)
+
+    const ctcategories = Object.keys(catIndex).length
+
+    return `${ctcategories},${ctright},${ctinfright},${ctideas},${ctrightencap},${ctinfrightencap},${ctwrong},${ctwrongencap},` +
+           `${ctcategories * ctideas},${(ctideas == 0) ? 0 : ctright / ctideas},${(ctideas == 0) ? 0 : ctinfright / ctideas},` +
+           `${(ctideas == 0) ? 0 : (ctrightencap + ctwrongencap) / ctideas},${selfOrder.score},` +
+           `${(ctideas == 0) ? 0 : selfOrder.score / ctideas},${clustering}${countCat},"${o1csv}","${o2csv}"`
+  }
+
+  async _downloadAnalysis () {
+    const tprefix = document.querySelector('#tprefix').value
+
+    const cases = await this._requestCases()
+
+    if (cases != null) {
+      let table = '"user id","annotation id","organization level","global score","student year",' +
+        '"used categories","right","right (inferred)","total ideas","right encapsulated",' +
+        '"right encapsulated (inferred)","wrong","wrong encapsulated","coverage score",' +
+        '"accuracy score","accuracy score (inferred)","encapsulated score","self order score",' +
+        '"normalized self order score","clustering in free recall"'
+
+      for (const m in ReportManager.catList)
+        table += ',"' + ReportManager.catList[m] + '"'
+
+      table += ',"self order groups","self order ordered"\n'
+
+      for (const c of cases.message) {
+        // remove prefix from title
+        if (c.title.startsWith(tprefix))
+          c.title = c.title.replace(tprefix, '')
+
+        table += '"' + c.title + '","' + c.id + '",'
+
+        const ant = await this._loadAnnotations(c.id)
+        const metrics = this._calculateMetrics(ant.annotations)
+
+        table += `"${ant.organization}","${ant.score}","${ant.year}",` + 
+                 metrics + '\n'
+      }
+
+      this._download(table)
+    }
+  }
+
+  /*
+   * Export BILOU
+  */
+
   async _buildBILOU (caseId, annotations) {
     const tt = await this._tokenize(caseId)
     const tokens = tt.tokens
@@ -141,21 +259,17 @@ class ReportManager {
     }
   }
 
-  async _tokenize (caseId) {
-
-    const tokens = []
-
-    let result = {}
-
+  async _loadCaseText (caseId) {
     const cs = await MessageBus.i.request('case/source/get',
                                           {room_id: this._roomId, case_id: caseId})
 
+    let text = null
     if (cs != null && cs.message != null) {
       const compiled =
       await Translator.instance.compileMarkdown(
         cs.message.id, cs.message.source)
 
-      let text = ''
+      text = ''
       for (const knot in compiled.knots) {
         const mkHTML =
           await Translator.instance.generateHTML(compiled.knots[knot])
@@ -165,7 +279,18 @@ class ReportManager {
       text = text.replace(/<[^>]+>/gm,'')
                  .replace(/<\/[^>]+>/gm, '')
                  .replace(/^[\r\n][\r\n]?/, '')
+    }
+
+    return text
+  }
+
+  async _tokenize (caseId) {
+    const tokens = []
+    let result = {}
+
+    const text = await this._loadCaseText(caseId)
 
+    if (text != null) {
       let c = 0
       let tk = ''
       let tks = -1
@@ -213,75 +338,11 @@ class ReportManager {
       ranges.push([f.start, f.start + f.size - 1])
   }
 
-  _calculateMetrics (annotations) {
-    let ctideas = 0, ctright = 0, ctinfright = 0
-    let ctwrong = 0, ctrightencap = 0, ctinfrightencap = 0, ctwrongencap = 0
-    const catIndex = {}
-    const catOrder = []
-
-    for (const an of annotations) {
-      ctideas++
-      let start = -1
-      for (const f of an.fragments) {
-        start = (start == -1) ? f.start : start
-      }
-      if (an.categories.includes('encapsulated'))
-        if (an.categories.includes('wrong'))
-          ctwrongencap++
-        else {
-          ctinfrightencap++
-          if (an.categories.includes('right'))
-            ctrightencap++
-        }
-      if (!an.categories.includes('wrong'))
-        ctinfright++
-      for (const c of an.categories) {
-        if (c == 'right') ctright++
-        else if (c == 'wrong') ctwrong++
-        if (ReportManager.catList.includes(c)) {
-          catIndex[c] = c
-          catOrder.push([ReportManager.catList.indexOf(c)+1, start])
-        }
-      }
-    }
-    const selfOrder = AnnotationMetrics.i._selfOrderCount(catOrder)
-
-    const clustering = AnnotationMetrics.i._clusteringFreeRecall(catOrder)
-
-    let o1csv = ''
-    let sep = ''
-    for (const g of selfOrder.groups) {
-      o1csv += sep + ReportManager.catList[g[0]-1] + ':' + g[2]
-      sep = '; '
-    }
-
-    let o2csv = ''
-    sep = ''
-    const ordered = {}
-    for (const g of selfOrder.ordered) {
-      ordered[ReportManager.catList[g[0]-1]] = g[2]
-      o2csv += sep + ReportManager.catList[g[0]-1] + ':' + g[2]
-      sep = '; '
-    }
-
-    let countCat = ''
-    for (const c of ReportManager.catList)
-      countCat += ',' + (ordered[c] ? ordered[c] : 0)
-
-    const ctcategories = Object.keys(catIndex).length
-
-    return `${ctcategories},${ctright},${ctinfright},${ctideas},${ctrightencap},${ctinfrightencap},${ctwrong},${ctwrongencap},` +
-           `${ctcategories * ctideas},${(ctideas == 0) ? 0 : ctright / ctideas},${(ctideas == 0) ? 0 : ctinfright / ctideas},` +
-           `${(ctideas == 0) ? 0 : (ctrightencap + ctwrongencap) / ctideas},${selfOrder.score},` +
-           `${(ctideas == 0) ? 0 : selfOrder.score / ctideas},${clustering}${countCat},"${o1csv}","${o2csv}"`
-  }
-
   async _downloadBILOU () {
     const tprefix = document.querySelector('#tprefix').value
 
     const cases = await this._requestCases()
 
-    console.log(cases)
     let table = ''
 
     if (cases != null) {
@@ -295,51 +356,49 @@ class ReportManager {
     }
   }
 
-  async _downloadAnalysis () {
-    const tprefix = document.querySelector('#tprefix').value
+  /*
+   * Export JSON
+  */
 
-    const cases = await this._requestCases()
-
-    if (cases != null) {
-      let table = '"user id","annotation id","organization level","global score","student year",' +
-        '"used categories","right","right (inferred)","total ideas","right encapsulated",' +
-        '"right encapsulated (inferred)","wrong","wrong encapsulated","coverage score",' +
-        '"accuracy score","accuracy score (inferred)","encapsulated score","self order score",' +
-        '"normalized self order score","clustering in free recall"'
+  async _buildJSON (caseId, annotations) {
+    const text = await this._loadCaseText(caseId)
+    const annComp = []
 
-      for (const m in ReportManager.catList)
-        table += ',"' + ReportManager.catList[m] + '"'
+    for (const an of annotations) {
+      for (let f = 0; f < an.fragments.length; f++) {
+        annComp.push([
+          an.fragments[f].fragment,
+          an.fragments[f].start,
+          an.fragments[f].start + an.fragments[f].size - 1,
+          (an.fragments.length == 1) ? 'U' : ((f == 0) ? 'B' : ((f+1 < an.fragments.length) ? 'I' : 'L')),
+          an.categories
+        ])
+      }
+    }
 
-      table += ',"self order groups","self order ordered"\n'
+    return {
+      doc_id: caseId,
+      text: text,
+      annotations: annComp
+    }
+  }
 
-      for (const c of cases.message) {
-        // remove prefix from title
-        if (c.title.startsWith(tprefix))
-          c.title = c.title.replace(tprefix, '')
+  async _downloadJSON () {
+    const tprefix = document.querySelector('#tprefix').value
 
-        table += '"' + c.title + '","' + c.id + '",'
+    const cases = await this._requestCases()
 
+    let table = ''
+    if (cases != null) {
+      for (const c of cases.message) {
         const ant = await this._loadAnnotations(c.id)
-        const metrics = this._calculateMetrics(ant.annotations)
-
-        table += `"${ant.organization}","${ant.score}","${ant.year}",` + 
-                 metrics + '\n'
+        const annJson = await this._buildJSON(c.id, ant.annotations)
+        table += JSON.stringify(annJson) + '\n'
       }
 
       this._download(table)
     }
   }
-
-  _download (table) {
-    const element = document.createElement('a')
-    element.setAttribute('href',
-      'data:text/plain;charset=utf-8,' + encodeURIComponent(table))
-    element.setAttribute('download', 'annotations.csv')
-    element.style.display = 'none'
-    document.body.appendChild(element)
-    element.click()
-    document.body.removeChild(element)
-  }
 }
 
 (function () {