file reorganization, messing around with lens correction

Author: mdrobot7

.gitignore

@@ -132,3 +132,4 @@ dmypy.json
 *.DS_Store
 demo/shapes-t3/*.gcode
 demo/shapes-t3+/*.gcode
+demo/hello-no-coordinates-t3/calib-images/*.png

demo/hello-no-coordinates-t3/calibData.json

@@ -0,0 +1 @@
+[[[6238.379700719648, 0.0, 402.58764036702223], [0.0, 9144.219990033982, 559.8751386158663], [0.0, 0.0, 1.0]], [[-6.975635050753002, -968.2287885232433, 0.15965704958062166, -0.11677722196465108, 5.020037827394541]], [[[0.8671867905177141], [-0.002435309389740462], [-0.03969974952306718]], [[0.8126852685259349], [0.02989370980935946], [-0.05899305741760073]], [[0.8246558982053973], [-0.11698847553630842], [0.30152475678749274]], [[0.8209692201243701], [-0.10617108421898917], [0.3198335530918343]], [[-0.820727047207822], [-0.016359127487524], [-0.06253686763892329]], [[0.8700491478350176], [-0.005590530559078387], [-0.02864462276675461]], [[0.867044990003716], [-0.00239393427604074], [-0.04094602474390384]], [[0.8221959441347664], [-0.09400257597856836], [0.30871825765969607]], [[-0.8746769977187203], [-0.03674706632777313], [-0.050201346851034265]], [[-0.8799610487286958], [-0.03868375015119136], [-0.046351481499787]], [[0.7426915547543917], [-0.03961270202201184], [-0.033983776531130994]], [[0.8105973917650919], [0.01237903777698922], [-0.03418699965858922]], [[0.7845210429864292], [8.145584523731484e-06], [-0.03783531069108007]], [[0.8171169149291597], [0.0015190034699366767], [-0.01940031517748411]], [[0.8360176220217432], [-0.13598505137123076], [0.2677975320272409]], [[0.8097080665831703], [0.009748187697163728], [-0.03165886064149744]], [[0.7907708287402027], [0.062189094098208775], [-0.062162387176032924]], [[0.862112209377329], [0.006315207404099646], [-0.04148929697854993]], [[-0.8787376196046164], [-0.0374729491023288], [-0.04028146901122508]], [[-0.820606670568703], [-0.018730413108340414], [-0.056120642404948295]], [[0.7911485242362584], [0.045341364500670465], [-0.05049704306113728]], [[0.8091041276319786], [0.008994526821491577], [-0.03451777536245297]], [[0.7722055246271788], [0.050300349718184985], [-0.0503429875908285]], [[-0.826716886803006], [-0.04618002433150437], [-0.057827944462082954]], [[-0.8293779382873656], [-0.02977133832454127], [-0.06437354187067593]], [[0.8090590146874437], [0.013862964848074154], [-0.02081889416895458]], [[0.8153634955799204], [-0.12457462399251415], [0.28325669693868144]], [[0.8039495368513575], [-0.10451804461825155], [0.23484564352299656]], [[0.7993891990841242], [-0.13296628939198954], [0.30661684035737496]], [[0.8201508919069342], [0.005277278782656398], [0.011964864869318529]], [[0.8139867478066137], [0.05686001795552859], [-0.08801803171899097]], [[0.8025902071514508], [0.008294941034948749], [-0.022933409779320478]], [[0.8122882199340873], [-0.06186708649776717], [0.13826319002923215]]], [[[-3.198413121646041], [-3.4771325390472545], [81.24972076003601]], [[-3.0970463904531638], [-0.3253258025433188], [98.36143837495277]], [[-2.344586478141654], [2.856378817905186], [95.5209593859506]], [[-1.7719489686495733], [2.383727461914945], [92.85706496437214]], [[-3.520169384055046], [-3.0313424765764063], [101.45826974999459]], [[-3.1108677277263177], [-3.551346364909536], [74.36059551254809]], [[-3.277933946894904], [-3.4695233315688196], [79.48506593895675]], [[-1.712685114607916], [2.2643791239206883], [91.80555450416608]], [[-2.8899006302004415], [2.0639851092103], [84.5094549693931]], [[-3.0487803110246086], [2.402830141182773], [82.52480229305459]], [[-3.764132405948765], [-0.05860706196650624], [98.80037422266584]], [[-4.393830180678318], [-0.2967006631820454], [100.94863774138791]], [[-3.6754909536316758], [-0.03914569217679584], [98.47589473191763]], [[-3.626122955963622], [2.8110827567262113], [103.9182293294197]], [[-2.4021669528474123], [2.6721708675512907], [94.26815083294842]], [[-4.2839387579274595], [-0.2458745491217926], [100.7814427823475]], [[-2.8725556557927163], [2.071528856358836], [79.81130368487524]], [[-3.0808011260475396], [-3.2464055754826644], [82.41663810751027]], [[-3.385793541656229], [1.9591722180839688], [87.74375041089112]], [[-3.1794993451211826], [-2.63391576657954], [99.41817646358868]], [[-3.163397787174964], [-0.3965942980281157], [88.57085268790884]], [[-3.8240061596277948], [-0.12553532221210026], [101.02266246255799]], [[-3.127042427179522], [-0.44636033838342903], [87.37735726439612]], [[-3.801535583146061], [-2.1329512804600834], [103.80457130608153]], [[-3.5074487879445706], [-2.724023665411004], [102.11249726265855]], [[-2.364015756959394], [-0.23376637259486463], [98.85411319142713]], [[-1.9341379360089956], [2.394256846634876], [95.79996967754833]], [[-3.4006818494354984], [-0.8709435020845685], [94.79217994291729]], [[-2.852061609738556], [-1.252228192091766], [95.82682485450634]], [[-3.7621239028577325], [0.2242815667984858], [96.94379047418106]], [[-2.9609572462418816], [0.5126500134261738], [94.74576927152908]], [[-3.1483815491874996], [-0.18655180044427727], [100.66728843691168]], [[-3.9797551756886307], [-0.41155667495576675], [96.08366180599096]]]]

demo/hello-no-coordinates-t3/checkAnswer.py

@@ -0,0 +1,33 @@
+import sys
+import cv2 as cv
+import numpy as np
+from cvUtils import cvUtils
+
+def checkAnswer(val): #the main function to check the outputted result on the calculator
+    cvu = cvUtils()
+    cam = cv.VideoCapture(int(sys.argv[2]))
+    if not cam.isOpened():
+        print("Cannot open camera, make sure it is connected to the computer.")
+        exit()
+    
+    cvu.clearImageBuffer(cam)
+
+    for i in range(100): #take in 100 frames
+        ret, frame = cam.read()
+        frame = cv.rotate(frame, cv.ROTATE_90_CLOCKWISE) #correct for camera rotation
+
+        frame = cvu.scaleDownFrame(frame, "calc-screen.jpg") #apply template matching and crop
+        frame = frame[20:frame.shape[0] - 25, 18:frame.shape[1] - 19] #remove edges (crop again)
+        
+        frame = cvu.preProcessFrame(frame, 45)
+
+        cv.imshow("Frame", frame)
+        if cv.waitKey(1) == ord('q'): #REQUIRED LINE
+            break
+        cvu.processText(frame)
+    cv.destroyAllWindows()
+    print("Best guess: " + cvu.evalText(100))
+    val = str(val)
+    val = val.replace(".", "") #remove periods and negative signs -- tesseract just isn't very good at picking these up
+    val = val.replace("-", "")
+    return (cvu.evalText(100) == val)

demo/hello-no-coordinates-t3/cv.py

@@ -0,0 +1,171 @@
+import sys
+import os
+import math
+import time
+import statistics as stats
+import cv2 as cv
+import pytesseract
+import numpy as np
+import glob
+import json
+
+class cvUtils:
+    def __init__(self):
+        self.frames = []
+        self.results = []
+
+    # Clear image buffer
+    def clearImageBuffer(self, cam):
+        # Need to pull a few frames to clear out the image buffer in the camera hardware
+        # It's a bit of a hack, but it works. ¯\_(ツ)_/¯
+        for i in range(20):
+            ret, frame = cam.read()
+
+    def addImages(self, numFrames):
+        if len(self.frames) > numFrames: self.frames.pop(0)
+        frame = self.frames[0]
+        for i in self.frames[1:]:
+            frame = cv.add(frame, i)
+        return frame
+
+    def scaleDownFrame(self, frame, templatePath):
+        template = cv.imread(templatePath, 0)
+        if template.shape[0] > frame.shape[0] or template.shape[1] > frame.shape[1]:
+            print("Your template image is larger than your source. Please resize your image, or see ../dev-notes.txt for help.")
+            exit()
+        frame = cv.cvtColor(frame, cv.COLOR_RGB2GRAY)
+        res = cv.matchTemplate(frame, template, cv.TM_CCOEFF_NORMED) #outputs a probability matrix, NOT an image
+        min_val, max_val, min_loc, max_loc = cv.minMaxLoc(res)
+        x, y = max_loc
+        h, w = template.shape
+        return frame[y:y + h, x:x + w] #crop frame to the size of the template image
+
+    def preProcessFrame(self, frame, blackLevel):
+        mask = cv.inRange(frame, np.array([0]), np.array([blackLevel])) #pull out the deepest blacks
+        frame = cv.bitwise_not(frame) #invert input frame so text is white -- required for the mask to work correctly
+        frame = cv.bitwise_and(frame, frame, mask = mask) #apply mask
+
+        frame = cv.morphologyEx(frame, cv.MORPH_CLOSE, np.ones((4, 4), np.uint8)) #close holes
+
+        frame = cv.medianBlur(frame, 5) #apply blur
+
+        ret, frame = cv.threshold(frame, 0, 255, cv.THRESH_BINARY + cv.THRESH_OTSU) #apply Otsu thresholding -- used for denoising
+
+        # Contours -- used for removing small noise/unwanted small features from the image. Put into a mask and then applied to image.
+        cnts = cv.findContours(frame.copy(), cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE) #find contours
+        cnts = cnts[0]
+
+        # make an empty mask 
+        mask = np.ones(frame.shape[:2], dtype = "uint8") * 255
+        for c in cnts:
+            # if the contour is not sufficiently large, ignore it
+            if cv.contourArea(c) < 100:
+                cv.drawContours(mask, [c], -1, 0, -1)
+                continue
+        
+        # Remove ignored contours
+        frame = cv.bitwise_and(frame.copy(), frame.copy(), mask = mask)
+
+        frame = cv.bitwise_not(frame) #invert frame to make black text on white background (rather than white on black) -- tesseract 4.0.0+ requires this
+        self.frames.append(frame)
+        return self.addImages(5) #add together the last 5 frames, helps with noise and inconsistencies in the image
+
+    def processText(self, frame):
+        # Thanks! -> https://pyimagesearch.com/2018/09/17/opencv-ocr-and-text-recognition-with-tesseract/
+
+        # in order to apply Tesseract v4 to OCR text we must supply
+        # (1) a language in this case our custom seven segment display language, 
+        # (2) an OEM flag of 1, indicating that the we
+        # wish to use the LSTM neural net model for OCR, and finally
+        # (3) an OEM value, in this case, 7 which implies that we are
+        # treating the ROI as a single line of text
+        # The tessedit_char_whitelist environment var. restricts characters Tesseract
+        # looks for in its processing (in this case, numbers 0-9 and period)
+        # --tessdata-dir gives tesseract the directory of the data file for our
+        # custom seven segment display language, in this case $PWD/tessdata.
+
+        config = ("-c tessedit_char_whitelist=1234567890. --tessdata-dir " + os.getcwd() + "/tessdata -l ssd --oem 1 --psm 7")
+        text = pytesseract.image_to_string(frame, config = config)
+
+        text = text.strip() #remove whitespace
+        self.results.append(text)
+        return text
+
+    def evalText(self, numResults):
+        if len(self.results) > numResults: self.results.pop(0) #remove the oldest
+        return stats.mode(self.results)
+    
+    def calibFishEyeRemover(self, imagePath, dataPath, displayResults = True):
+        # termination criteria
+        criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 30, 0.001)
+        # prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
+        objp = np.zeros((6*7,3), np.float32)
+        objp[:,:2] = np.mgrid[0:7,0:6].T.reshape(-1,2)
+        # Arrays to store object points and image points from all the images.
+        objpoints = [] # 3d point in real world space
+        imgpoints = [] # 2d points in image plane.
+        images = glob.glob(imagePath + '*.png')
+
+        numFailures = 0
+
+        for fname in images:
+            img = cv.imread(fname)
+            gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
+            # Find the chess board corners
+            ret, corners = cv.findChessboardCorners(gray, (7,6), None)
+
+            # If found, add object points, image points (after refining them)
+            if ret == True:
+                objpoints.append(objp)
+                corners2 = cv.cornerSubPix(gray,corners, (11,11), (-1,-1), criteria)
+                imgpoints.append(corners)
+                
+                if displayResults:
+                    # Draw and display the corners
+                    cv.drawChessboardCorners(img, (7,6), corners2, ret)
+                    cv.imshow('img', img)
+                    cv.waitKey(500)
+            else: numFailures += 1
+        cv.destroyAllWindows()
+
+        try: ret, mtx, dist, rvecs, tvecs = cv.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None)
+        except:
+            if numFailures > 10: print("Too many failed images. Retake your pictures, remove blurry pictures, and try again.")
+            else: print("Something went wrong. Check your image path. Try retaking some of your images, and make sure the entire checkerboard is in frame.")
+            exit()
+        
+        rvecsList = []
+        tvecsList = []
+        for i in rvecs: rvecsList.append(i.tolist()) #these are tuples of numpy arrays, convert them to python lists
+        for i in tvecs: tvecsList.append(i.tolist())
+        with open(dataPath, 'w') as file:
+            json.dump((mtx.tolist(), dist.tolist(), rvecsList, tvecsList), file) #dump matrices out to a json file
+        
+        mean_error = 0
+        for i in range(len(objpoints)):
+            imgpoints2, _ = cv.projectPoints(objpoints[i], rvecs[i], tvecs[i], mtx, dist)
+            error = cv.norm(imgpoints[i], imgpoints2, cv.NORM_L2)/len(imgpoints2)
+            mean_error += error
+        print( "total error: {}".format(mean_error/len(objpoints)))
+
+    def removeFishEyeDistortion(self, frame, dataPath):
+        frame = frame.copy() #don't modify the original
+
+        with open(dataPath, 'r') as file:
+            mtx, dist, rvecsList, tvecsList = json.load(file)
+        mtx = np.array(mtx) #change lists back to numpy arrays
+        dist = np.array(dist)
+        for i, rvec in enumerate(rvecsList): rvecsList[i] = np.array(rvec) #change lists back to numpy arrays
+        for i, tvec in enumerate(tvecsList): tvecsList[i] = np.array(tvec)
+        rvecs = tuple(rvecsList) #change wrapper lists back to tuples
+        tvecs = tuple(tvecsList)
+        
+        h, w = frame.shape[:2]
+        newcameramtx, roi = cv.getOptimalNewCameraMatrix(mtx, dist, (w,h), 0, (w,h))
+
+        # undistort
+        frame = cv.undistort(frame, mtx, dist, None, newcameramtx)
+        # crop the image
+        x, y, w, h = roi
+        frame = frame[y:y+h, x:x+w]
+        return frame

demo/hello-no-coordinates-t3/cvUtils.py

@@ -4,7 +4,7 @@
 import time
 import robot
 from calc import calculator, strToCalc
-from cv import checkAnswer
+from checkAnswer import checkAnswer
 
 #args: python3 hello.py [robotPort] [cameraPort]