Change main stat preprocessing from bandpass to blud/dilate/invert/th…

…reshold, also add an e2e test (#2580)

Co-authored-by: frzyc <frzyc@users.noreply.github.com>

apps/frontend-e2e/src/e2e/app.cy.ts

@@ -1,13 +1,35 @@
-import { getGreeting } from '../support/app.po'
-
+/* eslint-disable cypress/no-unnecessary-waiting */
 describe('frontend', () => {
-  beforeEach(() => cy.visit('/'))
+  it('should test artifacts images to check if main stat is being parsed correctly', () => {
+    cy.visit('/#/artifacts', { timeout: 600000 })
+    cy.get('.MuiModal-root [data-testid="CloseIcon"]').click()
+    cy.contains('button', 'Add New Artifact').click()
+    cy.contains('button', 'Add Artifact').as('addArtifact')
+    cy.contains('label', 'Upload Screenshot').selectFile('assets/em_155.png')
+    cy.wait(5000)
+    cy.contains('button', 'Add Artifact', { timeout: 60000 }).should(
+      'not.be.disabled'
+    )
+    cy.get(
+      '[data-testid="artifact-editor"] [data-testid="main-stat"] button'
+    ).should('have.text', 'Elemental Mastery')
+    cy.get(
+      '[data-testid="artifact-editor"] [data-testid="main-stat"] p'
+    ).should('have.text', '155')
 
-  it('should display welcome message', () => {
-    // Custom command example, see `../support/commands.ts` file
-    cy.login('my-email@something.com', 'myPassword')
+    cy.contains('button', 'Clear').click()
+    cy.wait(500)
 
-    // Function helper example, see `../support/app.po.ts` file
-    getGreeting().contains('Welcome frontend')
+    cy.contains('label', 'Upload Screenshot').selectFile('assets/def_p_8_7.png')
+    cy.wait(5000)
+    cy.contains('button', 'Add Artifact', { timeout: 60000 }).should(
+      'not.be.disabled'
+    )
+    cy.get(
+      '[data-testid="artifact-editor"] [data-testid="main-stat"] button'
+    ).should('have.text', 'DEF%')
+    cy.get(
+      '[data-testid="artifact-editor"] [data-testid="main-stat"] p'
+    ).should('have.text', '8.7%')
   })
 })

libs/common/img-util/src/processing.ts

@@ -125,3 +125,286 @@ export function findHistogramRange(
   if (b > length - 1) b = length - 1
   return [a, b]
 }
+
+/**
+ * This function performs standard image preprocessing. Standard blur,
+ * dilate, invert and threshold is done consecutively.
+ * @param pixelData imagedata
+ * @returns a preprocessed cloned image
+ */
+export function preprocessImage(pixelData: ImageData) {
+  const imageClone = Uint8ClampedArray.from(pixelData.data)
+  blurARGB(imageClone, pixelData.width, pixelData.height, 0.5)
+  dilate(imageClone, pixelData.width)
+  invertColors(imageClone)
+  thresholdFilter(imageClone, 0.4)
+  return new ImageData(imageClone, pixelData.width, pixelData.height)
+}
+
+// from https://github.com/processing/p5.js/blob/main/src/image/filters.js
+function thresholdFilter(pixels: Uint8ClampedArray, level: number) {
+  if (level === undefined) {
+    level = 0.5
+  }
+  const thresh = Math.floor(level * 255)
+  for (let i = 0; i < pixels.length; i += 4) {
+    const r = pixels[i]
+    const g = pixels[i + 1]
+    const b = pixels[i + 2]
+    const gray = 0.2126 * r + 0.7152 * g + 0.0722 * b
+    let val
+    if (gray >= thresh) {
+      val = 255
+    } else {
+      val = 0
+    }
+    pixels[i] = pixels[i + 1] = pixels[i + 2] = val
+  }
+}
+// from https://css-tricks.com/manipulating-pixels-using-canvas/
+function invertColors(pixels: Uint8ClampedArray) {
+  for (let i = 0; i < pixels.length; i += 4) {
+    pixels[i] = pixels[i] ^ 255 // Invert Red
+    pixels[i + 1] = pixels[i + 1] ^ 255 // Invert Green
+    pixels[i + 2] = pixels[i + 2] ^ 255 // Invert Blue
+  }
+}
+
+// internal kernel stuff for the gaussian blur filter
+let blurRadius: number
+let blurKernelSize: number
+let blurKernel: Int32Array
+let blurMult: Array<Int32Array>
+
+// from https://github.com/processing/p5.js/blob/main/src/image/filters.js
+function buildBlurKernel(r: number) {
+  let radius = (r * 3.5) | 0
+  radius = radius < 1 ? 1 : radius < 248 ? radius : 248
+
+  if (blurRadius !== radius) {
+    blurRadius = radius
+    blurKernelSize = (1 + blurRadius) << 1
+    blurKernel = new Int32Array(blurKernelSize)
+    blurMult = new Array(blurKernelSize)
+    for (let l = 0; l < blurKernelSize; l++) {
+      blurMult[l] = new Int32Array(256)
+    }
+
+    let bki: number
+    let bm, bmi: Int32Array
+
+    for (let i = 1, radiusi = radius - 1; i < radius; i++) {
+      blurKernel[radius + i] = blurKernel[radiusi] = bki = radiusi * radiusi
+      bm = blurMult[radius + i]
+      bmi = blurMult[radiusi--]
+      for (let j = 0; j < 256; j++) {
+        bm[j] = bmi[j] = bki * j
+      }
+    }
+    const bk = (blurKernel[radius] = radius * radius)
+    bm = blurMult[radius]
+
+    for (let k = 0; k < 256; k++) {
+      bm[k] = bk * k
+    }
+  }
+}
+
+// from https://github.com/processing/p5.js/blob/main/src/image/filters.js
+function blurARGB(
+  pixels: Uint8ClampedArray,
+  width: number,
+  height: number,
+  radius: number
+) {
+  const numPackedPixels = width * height
+  const argb = new Int32Array(numPackedPixels)
+  for (let j = 0; j < numPackedPixels; j++) {
+    argb[j] = getARGB(pixels, j)
+  }
+  let sum, cr, cg, cb, ca
+  let read, ri, ym, ymi, bk0
+  const a2 = new Int32Array(numPackedPixels)
+  const r2 = new Int32Array(numPackedPixels)
+  const g2 = new Int32Array(numPackedPixels)
+  const b2 = new Int32Array(numPackedPixels)
+  let yi = 0
+  buildBlurKernel(radius)
+  let x, y, i
+  let bm
+  for (y = 0; y < height; y++) {
+    for (x = 0; x < width; x++) {
+      cb = cg = cr = ca = sum = 0
+      read = x - blurRadius
+      if (read < 0) {
+        bk0 = -read
+        read = 0
+      } else {
+        if (read >= width) {
+          break
+        }
+        bk0 = 0
+      }
+      for (i = bk0; i < blurKernelSize; i++) {
+        if (read >= width) {
+          break
+        }
+        const c = argb[read + yi]
+        bm = blurMult[i]
+        ca += bm[(c & -16777216) >>> 24]
+        cr += bm[(c & 16711680) >> 16]
+        cg += bm[(c & 65280) >> 8]
+        cb += bm[c & 255]
+        sum += blurKernel[i]
+        read++
+      }
+      ri = yi + x
+      a2[ri] = ca / sum
+      r2[ri] = cr / sum
+      g2[ri] = cg / sum
+      b2[ri] = cb / sum
+    }
+    yi += width
+  }
+  yi = 0
+  ym = -blurRadius
+  ymi = ym * width
+  for (y = 0; y < height; y++) {
+    for (x = 0; x < width; x++) {
+      cb = cg = cr = ca = sum = 0
+      if (ym < 0) {
+        bk0 = ri = -ym
+        read = x
+      } else {
+        if (ym >= height) {
+          break
+        }
+        bk0 = 0
+        ri = ym
+        read = x + ymi
+      }
+      for (i = bk0; i < blurKernelSize; i++) {
+        if (ri >= height) {
+          break
+        }
+        bm = blurMult[i]
+        ca += bm[a2[read]]
+        cr += bm[r2[read]]
+        cg += bm[g2[read]]
+        cb += bm[b2[read]]
+        sum += blurKernel[i]
+        ri++
+        read += width
+      }
+      argb[x + yi] =
+        ((ca / sum) << 24) | ((cr / sum) << 16) | ((cg / sum) << 8) | (cb / sum)
+    }
+    yi += width
+    ymi += width
+    ym++
+  }
+  setPixels(pixels, argb)
+}
+
+function getARGB(data: Uint8ClampedArray, i: number) {
+  const offset = i * 4
+  return (
+    ((data[offset + 3] << 24) & 0xff000000) |
+    ((data[offset] << 16) & 0x00ff0000) |
+    ((data[offset + 1] << 8) & 0x0000ff00) |
+    (data[offset + 2] & 0x000000ff)
+  )
+}
+
+function setPixels(pixels: Uint8ClampedArray, data: Int32Array) {
+  let offset = 0
+  for (let i = 0, al = pixels.length; i < al; i++) {
+    offset = i * 4
+    pixels[offset + 0] = (data[i] & 0x00ff0000) >>> 16
+    pixels[offset + 1] = (data[i] & 0x0000ff00) >>> 8
+    pixels[offset + 2] = data[i] & 0x000000ff
+    pixels[offset + 3] = (data[i] & 0xff000000) >>> 24
+  }
+}
+
+// from https://github.com/processing/p5.js/blob/main/src/image/filters.js
+function dilate(pixels: Uint8ClampedArray, width: number) {
+  let currIdx = 0
+  const maxIdx = pixels.length ? pixels.length / 4 : 0
+  const out = new Int32Array(maxIdx)
+  let currRowIdx, maxRowIdx, colOrig, colOut, currLum
+
+  let idxRight, idxLeft, idxUp, idxDown
+  let colRight, colLeft, colUp, colDown
+  let lumRight, lumLeft, lumUp, lumDown
+
+  while (currIdx < maxIdx) {
+    currRowIdx = currIdx
+    maxRowIdx = currIdx + width
+    while (currIdx < maxRowIdx) {
+      colOrig = colOut = getARGB(pixels, currIdx)
+      idxLeft = currIdx - 1
+      idxRight = currIdx + 1
+      idxUp = currIdx - width
+      idxDown = currIdx + width
+
+      if (idxLeft < currRowIdx) {
+        idxLeft = currIdx
+      }
+      if (idxRight >= maxRowIdx) {
+        idxRight = currIdx
+      }
+      if (idxUp < 0) {
+        idxUp = 0
+      }
+      if (idxDown >= maxIdx) {
+        idxDown = currIdx
+      }
+      colUp = getARGB(pixels, idxUp)
+      colLeft = getARGB(pixels, idxLeft)
+      colDown = getARGB(pixels, idxDown)
+      colRight = getARGB(pixels, idxRight)
+
+      //compute luminance
+      currLum =
+        77 * ((colOrig >> 16) & 0xff) +
+        151 * ((colOrig >> 8) & 0xff) +
+        28 * (colOrig & 0xff)
+      lumLeft =
+        77 * ((colLeft >> 16) & 0xff) +
+        151 * ((colLeft >> 8) & 0xff) +
+        28 * (colLeft & 0xff)
+      lumRight =
+        77 * ((colRight >> 16) & 0xff) +
+        151 * ((colRight >> 8) & 0xff) +
+        28 * (colRight & 0xff)
+      lumUp =
+        77 * ((colUp >> 16) & 0xff) +
+        151 * ((colUp >> 8) & 0xff) +
+        28 * (colUp & 0xff)
+      lumDown =
+        77 * ((colDown >> 16) & 0xff) +
+        151 * ((colDown >> 8) & 0xff) +
+        28 * (colDown & 0xff)
+
+      if (lumLeft > currLum) {
+        colOut = colLeft
+        currLum = lumLeft
+      }
+      if (lumRight > currLum) {
+        colOut = colRight
+        currLum = lumRight
+      }
+      if (lumUp > currLum) {
+        colOut = colUp
+        currLum = lumUp
+      }
+      if (lumDown > currLum) {
+        colOut = colDown
+        currLum = lumDown
+      }
+      out[currIdx++] = colOut
+    }
+  }
+  setPixels(pixels, out)
+}

libs/gi/art-scanner/src/lib/processImg.ts

@@ -15,6 +15,7 @@ import {
   histogramContAnalysis,
   imageDataToCanvas,
   lighterColor,
+  preprocessImage,
   urlToImageData,
 } from '@genshin-optimizer/common/img-util'
 import {
@@ -239,12 +240,7 @@ export async function processEntry(
     debugImgs['substatsCardCropped'] = canvas.toDataURL()
   }
 
-  const bwHeader = bandPass(
-    headerCropped,
-    { r: 140, g: 140, b: 140 },
-    { r: 255, g: 255, b: 255 },
-    'bw'
-  )
+  const bwHeader = preprocessImage(headerCropped)
   const bwGreenText = bandPass(
     greenTextCropped,
     { r: 30, g: 100, b: 30 },

libs/gi/ui/src/components/artifact/editor/index.tsx

@@ -428,7 +428,7 @@ export function ArtifactEditor({
 
   const removeId = (artifactIdToEdit !== 'new' && artifactIdToEdit) || old?.id
   return (
-    <ModalWrapper open={show} onClose={onClose}>
+    <ModalWrapper open={show} onClose={onClose} data-testid="artifact-editor">
       <CardThemed>
         <UploadExplainationModal
           modalShow={modalShow}
@@ -551,7 +551,12 @@ export function ArtifactEditor({
               </Box>
 
               {/* main stat */}
-              <Box component="div" display="flex" gap={1}>
+              <Box
+                component="div"
+                display="flex"
+                gap={1}
+                data-testid="main-stat"
+              >
                 <DropdownButton
                   startIcon={
                     artifact?.mainStatKey ? (