diff --git a/Real Time Sudoku Solver/Dataset/README.md b/Real Time Sudoku Solver/Dataset/README.md new file mode 100644 index 000000000..9f79c02c0 --- /dev/null +++ b/Real Time Sudoku Solver/Dataset/README.md @@ -0,0 +1,6 @@ +# Real Time Sudoku Solver + +1. In the Models Folder, Run the `sudoku testing.ipynb` file. You need not run other files. This file itself will do all the work for you. +2. Make sure your webcam is connected and all the requirements is satisfied. +3. As soon as the window opens, show your sudoku image (as shown in the video demonstration). +4. Your sudoku is solved! diff --git a/Real Time Sudoku Solver/Images/1.jpg b/Real Time Sudoku Solver/Images/1.jpg new file mode 100644 index 000000000..fb31ea9f8 Binary files /dev/null and b/Real Time Sudoku Solver/Images/1.jpg differ diff --git a/Real Time Sudoku Solver/Images/2.png b/Real Time Sudoku Solver/Images/2.png new file mode 100644 index 000000000..552e1e869 Binary files /dev/null and b/Real Time Sudoku Solver/Images/2.png differ diff --git a/Real Time Sudoku Solver/Images/3.png b/Real Time Sudoku Solver/Images/3.png new file mode 100644 index 000000000..2e14df605 Binary files /dev/null and b/Real Time Sudoku Solver/Images/3.png differ diff --git a/Real Time Sudoku Solver/Images/4.png b/Real Time Sudoku Solver/Images/4.png new file mode 100644 index 000000000..3be84da72 Binary files /dev/null and b/Real Time Sudoku Solver/Images/4.png differ diff --git a/Real Time Sudoku Solver/Images/5.png b/Real Time Sudoku Solver/Images/5.png new file mode 100644 index 000000000..6d3ee5d7d Binary files /dev/null and b/Real Time Sudoku Solver/Images/5.png differ diff --git a/Real Time Sudoku Solver/Images/6.png b/Real Time Sudoku Solver/Images/6.png new file mode 100644 index 000000000..ccd9e49f6 Binary files /dev/null and b/Real Time Sudoku Solver/Images/6.png differ diff --git a/Real Time Sudoku Solver/Model/RealTimeSudokuSolver.ipynb b/Real Time Sudoku Solver/Model/RealTimeSudokuSolver.ipynb new file mode 100644 index 000000000..a3ba3f449 --- /dev/null +++ b/Real Time Sudoku Solver/Model/RealTimeSudokuSolver.ipynb @@ -0,0 +1,455 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "ename": "ModuleNotFoundError", + "evalue": "No module named 'sudokuSolver'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 3\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mscipy\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mndimage\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 4\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mmath\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0msudokuSolver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mipynb\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[1;33m*\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 6\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 7\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'sudokuSolver'" + ] + } + ], + "source": [ + "import cv2\n", + "import numpy as np\n", + "from scipy import ndimage\n", + "import math\n", + "import sudokuSolver \n", + "import copy\n", + "\n", + "# recognize_sudoku_and_solve function does following steps: Convert colored image to gray scale, blurring, thresholding, \n", + "# find contours, get biggest contour, get corners, get perspective transform.\n", + "\n", + "def recognize_sudoku_and_solve(image, model, old_sudoku):\n", + "\n", + " clone_image = np.copy(image) # deep copy to use later\n", + "\n", + " # Convert to a gray image, blur that gray image for easier detection\n", + " # and apply adaptiveThreshold\n", + " gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n", + " blur = cv2.GaussianBlur(gray, (5,5), 0)\n", + " thresh = cv2.adaptiveThreshold(blur, 255, 1, 1, 11, 2)\n", + "\n", + " # Find all contours\n", + " contours, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)\n", + " \n", + " # Extract countour with biggest area, assuming the Sudoku board is the BIGGEST contour\n", + " max_area = 0\n", + " biggest_contour = None\n", + " for c in contours:\n", + " area = cv2.contourArea(c)\n", + " if area > max_area:\n", + " max_area = area\n", + " biggest_contour = c\n", + "\n", + " if biggest_contour is None: # If no sudoku\n", + " return image\n", + "\n", + " # Get 4 corners of the biggest contour\n", + " corners = get_corners_from_contours(biggest_contour, 4)\n", + "\n", + " if corners is None: # If no sudoku\n", + " return image\n", + "\n", + " # Now we have 4 corners, locate the top left, top right, bottom left and bottom right corners\n", + " rect = np.zeros((4, 2), dtype = \"float32\")\n", + " corners = corners.reshape(4,2)\n", + "\n", + " # Find top left (sum of coordinates is the smallest)\n", + " sum = 10000\n", + " index = 0\n", + " for i in range(4):\n", + " if(corners[i][0]+corners[i][1] < sum):\n", + " sum = corners[i][0]+corners[i][1]\n", + " index = i\n", + " rect[0] = corners[index]\n", + " corners = np.delete(corners, index, 0)\n", + "\n", + " # Find bottom right (sum of coordinates is the biggest)\n", + " sum = 0\n", + " for i in range(3):\n", + " if(corners[i][0]+corners[i][1] > sum):\n", + " sum = corners[i][0]+corners[i][1]\n", + " index = i\n", + " rect[2] = corners[index]\n", + " corners = np.delete(corners, index, 0)\n", + "\n", + " # Find top right (Only 2 points left, should be easy\n", + " if(corners[0][0] > corners[1][0]):\n", + " rect[1] = corners[0]\n", + " rect[3] = corners[1]\n", + " \n", + " else:\n", + " rect[1] = corners[1]\n", + " rect[3] = corners[0]\n", + "\n", + " rect = rect.reshape(4,2)\n", + "\n", + "\n", + " # After having found 4 corners A B C D, check if ABCD is approximately square\n", + " # A------B\n", + " # | |\n", + " # | |\n", + " # D------C\n", + "\n", + " A = rect[0]\n", + " B = rect[1]\n", + " C = rect[2]\n", + " D = rect[3]\n", + " \n", + " # 1st condition: If all 4 angles are not approximately 90 degrees (with tolerance = epsAngle), stop\n", + " AB = B - A # 4 vectors AB AD BC DC\n", + " AD = D - A\n", + " BC = C - B\n", + " DC = C - D\n", + " eps_angle = 20\n", + " if not (approx_90_degrees(angle_between(AB,AD), eps_angle) and approx_90_degrees(angle_between(AB,BC), eps_angle)\n", + " and approx_90_degrees(angle_between(BC,DC), eps_angle) and approx_90_degrees(angle_between(AD,DC), eps_angle)):\n", + " return image\n", + " \n", + " # 2nd condition: The Lengths of AB, AD, BC, DC have to be approximately equal\n", + " # => Longest and shortest sides have to be approximately equal\n", + " eps_scale = 1.2 # Longest cannot be longer than epsScale * shortest\n", + " if(side_lengths_are_too_different(A, B, C, D, eps_scale)):\n", + " return image\n", + " \n", + " # Now we are sure ABCD correspond to 4 corners of a Sudoku board\n", + "\n", + " # the width of our Sudoku board\n", + " (tl, tr, br, bl) = rect\n", + " width_A = np.sqrt(((br[0] - bl[0]) ** 2) + ((br[1] - bl[1]) ** 2))\n", + " width_B = np.sqrt(((tr[0] - tl[0]) ** 2) + ((tr[1] - tl[1]) ** 2))\n", + "\n", + " # the height of our Sudoku board\n", + " height_A = np.sqrt(((tr[0] - br[0]) ** 2) + ((tr[1] - br[1]) ** 2))\n", + " height_B = np.sqrt(((tl[0] - bl[0]) ** 2) + ((tl[1] - bl[1]) ** 2))\n", + "\n", + " # take the maximum of the width and height values to reach\n", + " # our final dimensions\n", + " max_width = max(int(width_A), int(width_B))\n", + " max_height = max(int(height_A), int(height_B))\n", + "\n", + " # construct our destination points which will be used to\n", + " # map the screen to a top-down, \"birds eye\" view\n", + " dst = np.array([\n", + " [0, 0],\n", + " [max_width - 1, 0],\n", + " [max_width - 1, max_height - 1],\n", + " [0, max_height - 1]], dtype = \"float32\")\n", + "\n", + " # calculate the perspective transform matrix and warp\n", + " # the perspective to grab the screen\n", + " perspective_transformed_matrix = cv2.getPerspectiveTransform(rect, dst)\n", + " warp = cv2.warpPerspective(image, perspective_transformed_matrix, (max_width, max_height))\n", + " orginal_warp = np.copy(warp)\n", + "\n", + " # At this point, warp contains ONLY the chopped Sudoku board\n", + " # Do some image processing to get ready for recognizing digits\n", + " warp = cv2.cvtColor(warp,cv2.COLOR_BGR2GRAY)\n", + " warp = cv2.GaussianBlur(warp, (5,5), 0)\n", + " warp = cv2.adaptiveThreshold(warp, 255, 1, 1, 11, 2)\n", + " warp = cv2.bitwise_not(warp)\n", + " _, warp = cv2.threshold(warp, 150, 255, cv2.THRESH_BINARY)\n", + "\n", + " # Init grid to store Sudoku Board digits\n", + " SIZE = 9\n", + " grid = []\n", + " for i in range(SIZE):\n", + " row = []\n", + " for j in range(SIZE):\n", + " row.append(0)\n", + " grid.append(row)\n", + "\n", + " height = warp.shape[0] // 9\n", + " width = warp.shape[1] // 9\n", + "\n", + " offset_width = math.floor(width / 10) # Offset is used to get rid of the boundaries\n", + " offset_height = math.floor(height / 10)\n", + " # Divide the Sudoku board into 9x9 square:\n", + " for i in range(SIZE):\n", + " for j in range(SIZE):\n", + "\n", + " # Crop with offset (We don't want to include the boundaries)\n", + " crop_image = warp[height*i+offset_height:height*(i+1)-offset_height, width*j+offset_width:width*(j+1)-offset_width] \n", + " \n", + " # There are still some boundary lines left though\n", + " # => Remove all black lines near the edges\n", + " # ratio = 0.6 => If 60% pixels are black, remove\n", + " # Notice as soon as we reach a line which is not a black line, the while loop stops\n", + " ratio = 0.6 \n", + " # Top\n", + " while np.sum(crop_image[0]) <= (1-ratio) * crop_image.shape[1] * 255:\n", + " crop_image = crop_image[1:]\n", + " # Bottom\n", + " while np.sum(crop_image[:,-1]) <= (1-ratio) * crop_image.shape[1] * 255:\n", + " crop_image = np.delete(crop_image, -1, 1)\n", + " # Left\n", + " while np.sum(crop_image[:,0]) <= (1-ratio) * crop_image.shape[0] * 255:\n", + " crop_image = np.delete(crop_image, 0, 1)\n", + " # Right\n", + " while np.sum(crop_image[-1]) <= (1-ratio) * crop_image.shape[0] * 255:\n", + " crop_image = crop_image[:-1] \n", + "\n", + " # Take the largestConnectedComponent (The digit), and remove all noises\n", + " crop_image = cv2.bitwise_not(crop_image)\n", + " crop_image = largest_connected_component(crop_image)\n", + " \n", + " # Resize\n", + " digit_pic_size = 28\n", + " crop_image = cv2.resize(crop_image, (digit_pic_size,digit_pic_size))\n", + "\n", + " # If this is a white cell, set grid[i][j] to 0 and continue on the next image:\n", + "\n", + " # Criteria 1 for detecting white cell:\n", + " # Has too little black pixels\n", + " if crop_image.sum() >= digit_pic_size**2*255 - digit_pic_size * 1 * 255:\n", + " grid[i][j] == 0\n", + " continue # Move on if we have a white cell\n", + "\n", + " # Criteria 2 for detecting white cell\n", + " # Huge white area in the center\n", + " center_width = crop_image.shape[1] // 2\n", + " center_height = crop_image.shape[0] // 2\n", + " x_start = center_height // 2\n", + " x_end = center_height // 2 + center_height\n", + " y_start = center_width // 2\n", + " y_end = center_width // 2 + center_width\n", + " center_region = crop_image[x_start:x_end, y_start:y_end]\n", + " \n", + " if center_region.sum() >= center_width * center_height * 255 - 255:\n", + " grid[i][j] = 0\n", + " continue # Move on if we have a white cell\n", + " \n", + " # Now we are quite certain that this crop_image contains a number\n", + "\n", + " # Store the number of rows and cols\n", + " rows, cols = crop_image.shape\n", + "\n", + " # Apply Binary Threshold to make digits more clear\n", + " _, crop_image = cv2.threshold(crop_image, 200, 255, cv2.THRESH_BINARY) \n", + " crop_image = crop_image.astype(np.uint8)\n", + "\n", + " # Centralize the image according to center of mass\n", + " crop_image = cv2.bitwise_not(crop_image)\n", + " shift_x, shift_y = get_best_shift(crop_image)\n", + " shifted = shift(crop_image,shift_x,shift_y)\n", + " crop_image = shifted\n", + "\n", + " crop_image = cv2.bitwise_not(crop_image)\n", + " \n", + " # Up to this point crop_image is good and clean!\n", + " #cv2.imshow(str(i)+str(j), crop_image)\n", + "\n", + " # Convert to proper format to recognize\n", + " crop_image = prepare(crop_image)\n", + "\n", + " # Recognize digits\n", + " prediction = model.predict([crop_image]) # model is trained by digitRecognition.py\n", + " grid[i][j] = np.argmax(prediction[0]) + 1 # 1 2 3 4 5 6 7 8 9 starts from 0, so add 1\n", + "\n", + " user_grid = copy.deepcopy(grid)\n", + "\n", + " # Solve sudoku after we have recognizing each digits of the Sudoku board:\n", + "\n", + " # If this is the same board as last camera frame\n", + " # Phewww, print the same solution. No need to solve it again\n", + " if (not old_sudoku is None) and two_matrices_are_equal(old_sudoku, grid, 9, 9):\n", + " if(sudokuSolver.all_board_non_zero(grid)):\n", + " orginal_warp = write_solution_on_image(orginal_warp, old_sudoku, user_grid)\n", + " # If this is a different board\n", + " else:\n", + " sudokuSolver.solve_sudoku(grid) # Solve it\n", + " if(sudokuSolver.all_board_non_zero(grid)): # If we got a solution\n", + " orginal_warp = write_solution_on_image(orginal_warp, grid, user_grid)\n", + " old_sudoku = copy.deepcopy(grid) # Keep the old solution\n", + "\n", + " # Apply inverse perspective transform and paste the solutions on top of the orginal image\n", + " result_sudoku = cv2.warpPerspective(orginal_warp, perspective_transformed_matrix, (image.shape[1], image.shape[0])\n", + " , flags=cv2.WARP_INVERSE_MAP)\n", + " result = np.where(result_sudoku.sum(axis=-1,keepdims=True)!=0, result_sudoku, image)\n", + "\n", + " return result\n", + "\n", + "# Write solution on \"image\"\n", + "def write_solution_on_image(image, grid, user_grid):\n", + " # Write grid on image\n", + " SIZE = 9\n", + " width = image.shape[1] // 9\n", + " height = image.shape[0] // 9\n", + " for i in range(SIZE):\n", + " for j in range(SIZE):\n", + " if(user_grid[i][j] != 0): # If user fill this cell\n", + " continue # Move on\n", + " text = str(grid[i][j])\n", + " off_set_x = width // 15\n", + " off_set_y = height // 15\n", + " font = cv2.FONT_HERSHEY_SIMPLEX\n", + " (text_height, text_width), baseLine = cv2.getTextSize(text, font, fontScale=1, thickness=3)\n", + " marginX = math.floor(width / 7)\n", + " marginY = math.floor(height / 7)\n", + " \n", + " font_scale = 0.6 * min(width, height) / max(text_height, text_width)\n", + " text_height *= font_scale\n", + " text_width *= font_scale\n", + " bottom_left_corner_x = width*j + math.floor((width - text_width) / 2) + off_set_x\n", + " bottom_left_corner_y = height*(i+1) - math.floor((height - text_height) / 2) + off_set_y\n", + " image = cv2.putText(image, text, (bottom_left_corner_x, bottom_left_corner_y), \n", + " font, font_scale, (0,0,255), thickness=3, lineType=cv2.LINE_AA)\n", + " return image\n", + "\n", + "# Compare every single elements of 2 matrices and return if all corresponding entries are equal\n", + "def two_matrices_are_equal(matrix_1, matrix_2, row, col):\n", + " for i in range(row):\n", + " for j in range(col):\n", + " if matrix_1[i][j] != matrix_2[i][j]:\n", + " return False\n", + " return True\n", + "\n", + "# This function is used as the first criteria for detecting whether \n", + "# the contour is a Sudoku board or not: Length of Sides CANNOT be too different (sudoku board is square)\n", + "# Return if the longest size is > the shortest size * eps_scale\n", + "def side_lengths_are_too_different(A, B, C, D, eps_scale):\n", + " AB = math.sqrt((A[0]-B[0])**2 + (A[1]-B[1])**2)\n", + " AD = math.sqrt((A[0]-D[0])**2 + (A[1]-D[1])**2)\n", + " BC = math.sqrt((B[0]-C[0])**2 + (B[1]-C[1])**2)\n", + " CD = math.sqrt((C[0]-D[0])**2 + (C[1]-D[1])**2)\n", + " shortest = min(AB, AD, BC, CD)\n", + " longest = max(AB, AD, BC, CD)\n", + " return longest > eps_scale * shortest\n", + "\n", + "# This function is used as the second criteria for detecting whether\n", + "# the contour is a Sudoku board or not: All 4 angles has to be approximately 90 degree\n", + "# Approximately 90 degress with tolerance epsilon\n", + "def approx_90_degrees(angle, epsilon):\n", + " return abs(angle - 90) < epsilon\n", + "\n", + "# This function is used for seperating the digit from noise in \"crop_image\"\n", + "# The Sudoku board will be chopped into 9x9 small square image,\n", + "# each of those image is a \"crop_image\"\n", + "def largest_connected_component(image):\n", + "\n", + " image = image.astype('uint8')\n", + " nb_components, output, stats, centroids = cv2.connectedComponentsWithStats(image, connectivity=8)\n", + " sizes = stats[:, -1]\n", + "\n", + " if(len(sizes) <= 1):\n", + " blank_image = np.zeros(image.shape)\n", + " blank_image.fill(255)\n", + " return blank_image\n", + "\n", + " max_label = 1\n", + " # Start from component 1 (not 0) because we want to leave out the background\n", + " max_size = sizes[1] \n", + "\n", + " for i in range(2, nb_components):\n", + " if sizes[i] > max_size:\n", + " max_label = i\n", + " max_size = sizes[i]\n", + "\n", + " img2 = np.zeros(output.shape)\n", + " img2.fill(255)\n", + " img2[output == max_label] = 0\n", + " return img2\n", + "\n", + "# Return the angle between 2 vectors in degrees\n", + "def angle_between(vector_1, vector_2):\n", + " unit_vector_1 = vector_1 / np.linalg.norm(vector_1)\n", + " unit_vector2 = vector_2 / np.linalg.norm(vector_2)\n", + " dot_droduct = np.dot(unit_vector_1, unit_vector2)\n", + " angle = np.arccos(dot_droduct)\n", + " return angle * 57.2958 # Convert to degree\n", + "\n", + "# Calculate how to centralize the image using its center of mass\n", + "def get_best_shift(img):\n", + " cy, cx = ndimage.measurements.center_of_mass(img)\n", + " rows, cols = img.shape\n", + " shiftx = np.round(cols/2.0-cx).astype(int)\n", + " shifty = np.round(rows/2.0-cy).astype(int)\n", + " return shiftx, shifty\n", + "\n", + "# Shift the image using what get_best_shift returns\n", + "def shift(img,sx,sy):\n", + " rows,cols = img.shape\n", + " M = np.float32([[1,0,sx],[0,1,sy]])\n", + " shifted = cv2.warpAffine(img,M,(cols,rows))\n", + " return shifted\n", + "\n", + "# Get 4 corners from contour.\n", + "# These 4 corners will be the corners of the Sudoku board\n", + "def get_corners_from_contours(contours, corner_amount=4, max_iter=200):\n", + "\n", + " coefficient = 1\n", + " while max_iter > 0 and coefficient >= 0:\n", + " max_iter = max_iter - 1\n", + "\n", + " epsilon = coefficient * cv2.arcLength(contours, True)\n", + "\n", + " poly_approx = cv2.approxPolyDP(contours, epsilon, True)\n", + " hull = cv2.convexHull(poly_approx)\n", + " if len(hull) == corner_amount:\n", + " return hull\n", + " else:\n", + " if len(hull) > corner_amount:\n", + " coefficient += .01\n", + " else:\n", + " coefficient -= .01\n", + " return None\n", + "\n", + "# Prepare and normalize the image to get ready for digit recognition\n", + "def prepare(img_array):\n", + " new_array = img_array.reshape(-1, 28, 28, 1)\n", + " new_array = new_array.astype('float32')\n", + " new_array /= 255\n", + " return new_array\n", + "\n", + "def showImage(img, name, width, height):\n", + " new_image = np.copy(img)\n", + " new_image = cv2.resize(new_image, (width, height))\n", + " cv2.imshow(name, new_image)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Real Time Sudoku Solver/Model/digitRecognition.h5 b/Real Time Sudoku Solver/Model/digitRecognition.h5 new file mode 100644 index 000000000..999205fae Binary files /dev/null and b/Real Time Sudoku Solver/Model/digitRecognition.h5 differ diff --git a/Real Time Sudoku Solver/Model/sudoku testing.ipynb b/Real Time Sudoku Solver/Model/sudoku testing.ipynb new file mode 100644 index 000000000..51144b170 --- /dev/null +++ b/Real Time Sudoku Solver/Model/sudoku testing.ipynb @@ -0,0 +1,143 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using TensorFlow backend.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "importing Jupyter notebook from sudokuSolver.ipynb\n" + ] + } + ], + "source": [ + "import cv2\n", + "import numpy as np\n", + "import keras\n", + "from tensorflow.keras.models import Sequential\n", + "from tensorflow.keras.layers import Dense, Dropout, Flatten\n", + "from tensorflow.keras.layers import Conv2D, MaxPooling2D\n", + "from tensorflow.keras import backend as K\n", + "\n", + "import import_ipynb\n", + "%run RealTimeSudokuSolver.ipynb\n", + "%run sudokuSolver.ipynb\n", + "# --->import RealTimeSudokuSolve\n", + "from scipy import ndimage\n", + "import math\n", + "# --->import sudokuSolver\n", + "import copy\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# This is the main file. Just run all cells in this file\n", + "# you need not run other files. This file itself will do all the work for you.\n", + "\n", + "# I have trained the CNN model and saved the architecture in digitRecognition.h5 file.\n", + "\n", + "def showImage(img, name, width, height):\n", + " new_image = np.copy(img)\n", + " new_image = cv2.resize(new_image, (width, height))\n", + " cv2.imshow(name, new_image)\n", + "\n", + "# Load and set up Camera\n", + "cap = cv2.VideoCapture(0, cv2.CAP_DSHOW)\n", + "cap.set(3, 1280) # HD Camera\n", + "cap.set(4, 720)\n", + "\n", + "# Loading model (Load weights and configuration seperately to speed up model and predictions)\n", + "input_shape = (28, 28, 1)\n", + "num_classes = 9\n", + "model = Sequential()\n", + "model.add(Conv2D(32, kernel_size=(3, 3),\n", + " activation='relu',\n", + " input_shape=input_shape))\n", + "model.add(Conv2D(64, (3, 3), activation='relu'))\n", + "model.add(MaxPooling2D(pool_size=(2, 2)))\n", + "model.add(Dropout(0.25))\n", + "model.add(Flatten())\n", + "model.add(Dense(128, activation='relu'))\n", + "model.add(Dropout(0.5))\n", + "model.add(Dense(num_classes, activation='softmax'))\n", + "\n", + "# Load weights from pre-trained model. This model is trained in digitRecognition.py\n", + "model.load_weights(\"digitRecognition.h5\") \n", + "\n", + "old_sudoku = None # Used to compare new sudoku or old sudoku\n", + "while(True):\n", + " ret, frame = cap.read() # Read the frame\n", + " if ret == True:\n", + " \n", + " # RealTimeSudokuSolver.recognize_sudoku_and_solve\n", + " sudoku_frame = recognize_sudoku_and_solve(frame, model, old_sudoku) \n", + " showImage(sudoku_frame, \"Real Time Sudoku Solver\", 1066, 600)\n", + " if cv2.waitKey(1) == ord('q'): # Hit q if you want to stop the camera\n", + " break\n", + " else:\n", + " break\n", + "\n", + "cap.release()\n", + "cv2.destroyAllWindows()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Real Time Sudoku Solver/Model/sudokuSolver.ipynb b/Real Time Sudoku Solver/Model/sudokuSolver.ipynb new file mode 100644 index 000000000..7a44f60de --- /dev/null +++ b/Real Time Sudoku Solver/Model/sudokuSolver.ipynb @@ -0,0 +1,149 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "class EntryData:\n", + " def __init__(self, r, c, n):\n", + " self.row = r\n", + " self.col = c\n", + " self.choices = n\n", + "\n", + " def set_data(self, r, c, n):\n", + " self.row = r\n", + " self.col = c\n", + " self.choices = n\n", + " \n", + "# Solve Sudoku using Best-first search\n", + "def solve_sudoku(matrix):\n", + " cont = [True]\n", + " # See if it is even possible to have a solution\n", + " for i in range(9):\n", + " for j in range(9):\n", + " if not can_be_correct(matrix, i, j): # If it is not possible, stop\n", + " return\n", + " sudoku_helper(matrix, cont) # Otherwise try to solve the Sudoku puzzle\n", + "\n", + "# Helper function - The heart of Best First Search\n", + "def sudoku_helper(matrix, cont):\n", + " if not cont[0]: # Stopping point 1\n", + " return\n", + "\n", + " # Find the best entry (The one with the least possibilities)\n", + " best_candidate = EntryData(-1, -1, 100)\n", + " for i in range(9):\n", + " for j in range(9):\n", + " if matrix[i][j] == 0: # If it is unfilled\n", + " num_choices = count_choices(matrix, i, j)\n", + " if best_candidate.choices > num_choices:\n", + " best_candidate.set_data(i, j, num_choices)\n", + "\n", + " # If didn't find any choices, it means...\n", + " if best_candidate.choices == 100: # Has filled all board, Best-First Search done! Note, whether we have a solution or not depends on whether all Board is non-zero\n", + " cont[0] = False # Set the flag so that the rest of the recursive calls can stop at \"stopping points\"\n", + " return\n", + "\n", + " row = best_candidate.row\n", + " col = best_candidate.col\n", + "\n", + " # If found the best candidate, try to fill 1-9\n", + " for j in range(1, 10):\n", + " if not cont[0]: # Stopping point 2\n", + " return\n", + "\n", + " matrix[row][col] = j\n", + "\n", + " if can_be_correct(matrix, row, col):\n", + " sudoku_helper(matrix, cont)\n", + "\n", + " if not cont[0]: # Stopping point 3\n", + " return\n", + " matrix[row][col] = 0 # Backtrack, mark the current cell empty again\n", + " \n", + "\n", + "# Count the number of choices haven't been used\n", + "def count_choices(matrix, i, j):\n", + " can_pick = [True,True,True,True,True,True,True,True,True,True]; # From 0 to 9 - drop 0\n", + " \n", + " # Check row\n", + " for k in range(9):\n", + " can_pick[matrix[i][k]] = False\n", + "\n", + " # Check col\n", + " for k in range(9):\n", + " can_pick[matrix[k][j]] = False;\n", + "\n", + " # Check 3x3 square\n", + " r = i // 3\n", + " c = j // 3\n", + " for row in range(r*3, r*3+3):\n", + " for col in range(c*3, c*3+3):\n", + " can_pick[matrix[row][col]] = False\n", + "\n", + " # Count\n", + " count = 0\n", + " for k in range(1, 10): # 1 to 9\n", + " if can_pick[k]:\n", + " count += 1\n", + "\n", + " return count\n", + "\n", + "# Return true if the current cell doesn't create any violation\n", + "def can_be_correct(matrix, row, col):\n", + " \n", + " # Check row\n", + " for c in range(9):\n", + " if matrix[row][col] != 0 and col != c and matrix[row][col] == matrix[row][c]:\n", + " return False\n", + "\n", + " # Check column\n", + " for r in range(9):\n", + " if matrix[row][col] != 0 and row != r and matrix[row][col] == matrix[r][col]:\n", + " return False\n", + "\n", + " # Check 3x3 square\n", + " r = row // 3\n", + " c = col // 3\n", + " for i in range(r*3, r*3+3):\n", + " for j in range(c*3, c*3+3):\n", + " if row != i and col != j and matrix[i][j] != 0 and matrix[i][j] == matrix[row][col]:\n", + " return False\n", + " \n", + " return True\n", + "\n", + "# Return true if the whole board has been occupied by some non-zero number\n", + "# If this happens, the current board is the solution to the original Sudoku\n", + "def all_board_non_zero(matrix):\n", + " for i in range(9):\n", + " for j in range(9):\n", + " if matrix[i][j] == 0:\n", + " return False\n", + " return True" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Real Time Sudoku Solver/README.md b/Real Time Sudoku Solver/README.md new file mode 100644 index 000000000..4dc899efc --- /dev/null +++ b/Real Time Sudoku Solver/README.md @@ -0,0 +1,42 @@ +# Real Time Sudoku Solver + +## ๐ŸŽฏ **Goal** +This project aims to develop an application that can solve standard Sudoku puzzles in real time using image processing and machine learning techniques, specifically Convolutional Neural Networks (CNN). The application captures video to identify the Sudoku board, solve the puzzle, and writes the solution on the board itself. + +## ๐Ÿงต **Dataset** +The application starts by capturing video to identify the Sudoku board in real time. It processes the captured image to detect and recognize the numbers on the board, solves the puzzle, and overlays the solution back onto the original board. + +If you want to try training the Convolution Network on your computer, you will need to download Chars74K Dataset http://www.ee.surrey.ac.uk/CVSSP/demos/chars74k/, takes images 1-9 (We only need 1-9) and put them in folders "1", "2", ..., "9" respectively in the same directory with where you put all my Python files. After that, just run digitRecognition.py + +## ๐Ÿงพ **Description** +This project leverages image processing and machine learning to create a real-time Sudoku solver. The core functionality involves using a Convolutional Neural Network (CNN) to recognize the digits on the Sudoku board from a live video feed. The recognized digits are then processed to solve the Sudoku puzzle, and the solution is displayed directly on the video feed. This application combines the power of computer vision and deep learning to deliver a seamless and interactive Sudoku-solving experience. + +## ๐Ÿงฎ **What I had done!** +1. Video Capture and Preprocessing: Implemented video capture to continuously get frames from the webcam. Applied image preprocessing techniques to enhance the quality of the captured frames for better digit recognition. +2. Digit Recognition: Trained a Convolutional Neural Network (CNN) model to recognize digits from the preprocessed images. Integrated the trained model into the application to identify digits on the Sudoku board. +3. Sudoku Solver: Developed an algorithm to solve the Sudoku puzzle using the recognized digits. Ensured the solution is accurate and efficient. +4. Overlay Solution: Implemented a method to overlay the solved Sudoku puzzle back onto the original video feed, displaying the solution in real time. + + The step by step images are provided in the Images Folder. + +## ๐Ÿ“š **Libraries Needed** +1. keras==2.3.1 +2. numpy==1.22.0 +3. opencv-python==4.2.0.34 +4. scipy==1.4.1 +5. import-ipynb==0.1.3 + +## **How To Start?** +`sudoku testing.ipynb` is the entry of the application. Just run the file and show the sudoku image as soon as the window opens. Make sure to have all the requirements. + +## ๐Ÿ“ข **Conclusion** +The real-time Sudoku solver successfully combines image processing and machine learning techniques to deliver an interactive and efficient solution for solving Sudoku puzzles. The application demonstrates high accuracy in digit recognition and puzzle-solving, thanks to the trained CNN model. +By leveraging CNN for digit recognition and combining it with efficient puzzle-solving algorithms, the project provides a seamless user experience in solving Sudoku puzzles directly from a live video feed. The accuracy and efficiency of the solution make it a valuable tool for Sudoku enthusiasts and showcase the potential of integrating image processing and machine learning for real-time applications. + +## Video Demonstration +![Real Time Sudoku Solver](https://github.com/abckhush/DL-Simplified/assets/127378920/6485b261-b624-46e5-8947-61c13390fe11) + +## โœ’๏ธ **Your Signature** +Khushi Kalra
+Github
+LinkedIn diff --git a/Real Time Sudoku Solver/requirements.txt b/Real Time Sudoku Solver/requirements.txt new file mode 100644 index 000000000..5eb9a406d --- /dev/null +++ b/Real Time Sudoku Solver/requirements.txt @@ -0,0 +1,5 @@ +keras==2.3.1 +numpy==1.22.0 +opencv-python==4.2.0.34 +scipy==1.4.1 +import-ipynb==0.1.3