Process MLText OCR lineframe array using openCV

Tangerine-Community · May 17, 2023 · 169007e · 169007e
1 parent 63c957b
commit 169007e
Show file tree

Hide file tree

Showing 2 changed files with 80 additions and 2 deletions.
diff --git a/.jshintrc b/.jshintrc
diff --git a/tangy-form/input/tangy-scan-image.js b/tangy-form/input/tangy-scan-image.js
@@ -27,6 +27,7 @@ class TangyScanImage extends LitElement {
     super.connectedCallback()
     console.log("connected...")
     this.video = null;
+    this.scannedImage = null;
     this.canvas = null;
     this.context = null;
     this.t = {
@@ -54,6 +55,15 @@ class TangyScanImage extends LitElement {
         }
 
       </style>
+      <div id="imageDisplay">
+<!--        <video width="640" height="480" autoplay id="video"></video>-->
+        <!--            <canvas id="canvas" style='display:none'></canvas>-->
+<!--        <canvas id="canvas" width="640" height="480"></canvas>-->
+        <canvas id="canvas"></canvas>
+        <!-- <img src="${this.value}" style='display:none' id="photoCaptureImage"/> -->
+        <img src="${this.value}" id="scannedImage" />
+<!--        <div id="centeredText" class="centered">[[t.saving]]</div>-->
+      </div>
       <paper-button id="retake-button" @click="${this.scanImage}"><iron-icon icon="camera-enhance"></iron-icon> ${this.t.retake} </paper-button>
     `
   }
@@ -64,19 +74,87 @@ class TangyScanImage extends LitElement {
     } else {
       this.front = true
     }
+    // this.video = this.shadowRoot.querySelector('video');
+    this.scannedImage = this.shadowRoot.querySelector('#scannedImage');
+    this.canvas = this.shadowRoot.querySelector('canvas');
     this.scanImage();
   }
 
   scanImage() {
     const onSuccess = (imageData) => {
-      const onSuccess = (recognizedText) => {
+      const onSuccess = async (recognizedText) => {
+        const sleep = (milliseconds) => new Promise((res) => setTimeout(() => res(true), milliseconds))
         //var element = document.getElementById('pp');
         //element.innerHTML=recognizedText.blocks.blocktext;
         //Use above two lines to show recognizedText in html
         console.log(recognizedText);
         // alert(recognizedText.blocks.blocktext);
         const lines = recognizedText.lines.linetext;
-        // this.value = JSON.stringify(lines);
+        const baseImage = new Image()
+        baseImage.src = imageData
+        // baseImage.onload = () => sendCanvas(canvas, baseImage);
+        await baseImage.decode();
+        this.canvas.width = baseImage.width;
+        this.canvas.height = baseImage.height;
+
+        let ctx = this.canvas.getContext('2d');
+        ctx.drawImage(baseImage, 0, 0, baseImage.width, baseImage.height);
+
+        // this.scannedImage.src = imageData;
+        const image = cv.imread(baseImage); // Load the image using OpenCV
+        console.log('image width: ' + image.cols + '\n' +
+            'image height: ' + image.rows + '\n' +
+            'image size: ' + image.size().width + '*' + image.size().height + '\n' +
+            'image depth: ' + image.depth() + '\n' +
+            'image channels ' + image.channels() + '\n' +
+            'image type: ' + image.type() + '\n');
+        // const blockframe = recognizedText.blocks.blockframe
+        const itemframe = recognizedText.lines.lineframe
+        for (let i = 0; i < itemframe.length; i++) {
+          const item = itemframe[i]
+          // const [x, y, width, height] = [xCoordinate, yCoordinate, boundingBoxWidth, boundingBoxHeight]; // Extract the coordinates
+          let [x, y, width, height] = [item.x, item.y, item.width, item.height]; // Extract the coordinates
+          if (height > image.height) {
+            console.error("height > image.height: " + height)
+            height = image.height
+          }
+          if (width > image.width) {
+            console.error("width > image.width: " + width)
+            width = image.width
+          }
+          let itemNumber = i+1
+          console.log("Item: " + itemNumber + " Creating Rect using dimensions: x:" + x + " y: " + y  + " width: " + width + " height: " + height)
+          // Extract the region of interest from the image
+          const rect = new cv.Rect(x, y, width, height)
+          try {
+            const roi = image.roi(rect);
+            // Convert the OpenCV image to grayscale for processing
+            const gray = new cv.Mat();
+            cv.cvtColor(roi, gray, cv.COLOR_BGR2GRAY);
+            // Perform image processing and circle detection with OpenCV
+            const circles = new cv.Mat();
+            // cv.HoughCircles(gray, circles, cv.HOUGH_GRADIENT, 1, 20, 50, 30, 0, 0);
+            cv.HoughCircles(gray, circles, cv.HOUGH_GRADIENT, 1,
+                gray.rows/16,  // change this value to detect circles with different distances to each other
+                100, 30, 1, 30 // change the last two parameters
+                // (min_radius & max_radius) to detect larger circles
+            );
+            // cv.HoughCircles(src, circles, cv.HOUGH_GRADIENT, 1, 35, 70, 20, 0, 0);
+            // Analyze the circle detection results
+            const hasFilledCircle = circles.size() > 0;
+
+            cv.imshow(this.canvas, roi);
+            await sleep(1000)
+            // Clean up resources
+            // roi.delete();
+            // gray.delete();
+            // circles.delete();
+            // Use the 'hasFilledCircle' variable as needed for further processing or display
+            console.log(hasFilledCircle);
+          } catch (e) {
+            console.log("error: " + e)
+          }
+        }
         this.dispatchEvent(new CustomEvent('TANGY_SCAN_IMAGE_VALUE', {
           bubbles: true,
           composed: true,