Add chessboard detection for calibration

cmd/elephanttalk/main.go

@@ -10,15 +10,18 @@ import (
 var testpage string
 
 func main() {
+	// talk.PrintCalibrationPage()
+	// talk.PrintPageFromShorthand("ygybr", "brgry", "gbgyg", "bgryy", `(claim this 'outlined 'blue)`)
+
 	// instead of using all coloured dots to identify pages, only use the corner dots
-	talk.UseSimplifiedIDs()
+	// talk.UseSimplifiedIDs()
 
 	//page1
 	//talk.AddPageFromShorthand("ygybr", "brgry", "gbgyg", "bgryy", `(claim this 'outlined 'blue)`)
 	talk.AddPageFromShorthand("ygybr", "brgry", "gbgyg", "bgryy", `(claim this 'pointing 30)`)
 
 	//page2
-	talk.AddPageFromShorthand("yggyg", "rgyrb", "bybbg", "brgrg", `(claim this 'highlighted 'red)`)
+	// talk.AddPageFromShorthand("yggyg", "rgyrb", "bybbg", "brgrg", `(claim this 'highlighted 'red)`)
 
 	//page that always counts as recognised but doesnt have to be present physically
 	talk.AddBackgroundPage(testpage)

go.mod

@@ -1,8 +1,10 @@
 module github.com/deosjr/elephanttalk
 
-go 1.20
+go 1.21
+
+toolchain go1.22.2
 
 require (
 	github.com/deosjr/whistle v0.0.0-20230606141022-90a4546b49c5
-	gocv.io/x/gocv v0.27.0
+	gocv.io/x/gocv v0.36.1
 )

go.sum

@@ -4,6 +4,23 @@ github.com/deosjr/whistle v0.0.0-20230604162944-34851f1db745 h1:TLicDWljBcTIBOVR
 github.com/deosjr/whistle v0.0.0-20230604162944-34851f1db745/go.mod h1:AqJExKTUYXaa+5DDBu0J+d4ajZbabQEiafNepi/Jwvk=
 github.com/deosjr/whistle v0.0.0-20230606141022-90a4546b49c5 h1:aSuo5bPaU/iE4sp08cWaua0BLHEPK26yIvHWFeQhaGs=
 github.com/deosjr/whistle v0.0.0-20230606141022-90a4546b49c5/go.mod h1:AqJExKTUYXaa+5DDBu0J+d4ajZbabQEiafNepi/Jwvk=
+github.com/hybridgroup/mjpeg v0.0.0-20140228234708-4680f319790e/go.mod h1:eagM805MRKrioHYuU7iKLUyFPVKqVV6um5DAvCkUtXs=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 gocv.io/x/gocv v0.27.0 h1:3X8I74ULsWHd4m7DQRv2Nqx5VkKscfUFnKgLNodiboI=
 gocv.io/x/gocv v0.27.0/go.mod h1:n4LnYjykU6y9gn48yZf4eLCdtuSb77XxSkW6g0wGf/A=
+gocv.io/x/gocv v0.28.0 h1:hweRS9Js60YEZPZzjhU5I+0E2ngazquLlO78zwnrFvY=
+gocv.io/x/gocv v0.28.0/go.mod h1:oc6FvfYqfBp99p+yOEzs9tbYF9gOrAQSeL/dyIPefJU=
+gocv.io/x/gocv v0.32.0 h1:KPj/piKjLJQ0oKc0y0GFwTrnNX8zaK++118ereaYQkU=
+gocv.io/x/gocv v0.32.0/go.mod h1:oc6FvfYqfBp99p+yOEzs9tbYF9gOrAQSeL/dyIPefJU=
+gocv.io/x/gocv v0.32.1 h1:BC9hHs5+47nVgySUFVKntc6RsF3SULFzqk6OV9xz+C0=
+gocv.io/x/gocv v0.32.1/go.mod h1:oc6FvfYqfBp99p+yOEzs9tbYF9gOrAQSeL/dyIPefJU=
+gocv.io/x/gocv v0.33.0 h1:WDtaBrq92AKrhepYzEktydDzNSm3t5k7ciawZK4rns8=
+gocv.io/x/gocv v0.33.0/go.mod h1:oc6FvfYqfBp99p+yOEzs9tbYF9gOrAQSeL/dyIPefJU=
+gocv.io/x/gocv v0.34.0 h1:lx180sKUAMzox3+gH65wLu2mZSJk9iy8BVXI4kBoymM=
+gocv.io/x/gocv v0.34.0/go.mod h1:oc6FvfYqfBp99p+yOEzs9tbYF9gOrAQSeL/dyIPefJU=
+gocv.io/x/gocv v0.35.0 h1:Qaxb5KdVyy8Spl4S4K0SMZ6CVmKtbfoSGQAxRD3FZlw=
+gocv.io/x/gocv v0.35.0/go.mod h1:oc6FvfYqfBp99p+yOEzs9tbYF9gOrAQSeL/dyIPefJU=
+gocv.io/x/gocv v0.36.0 h1:PMAm97jT5Czh954xu8VE0wX/zNb7zo3RqrmLvawHM4I=
+gocv.io/x/gocv v0.36.0/go.mod h1:lmS802zoQmnNvXETpmGriBqWrENPei2GxYx5KUxJsMA=
+gocv.io/x/gocv v0.36.1 h1:6XkEaPOk7h/umjy+MXgSEtSeCIgcPJhccUjrJFhjdTY=
+gocv.io/x/gocv v0.36.1/go.mod h1:lmS802zoQmnNvXETpmGriBqWrENPei2GxYx5KUxJsMA=

talk/calibration.go

@@ -5,6 +5,7 @@ import (
 	"image"
 	"image/color"
 	"math"
+	"time"
 
 	"gocv.io/x/gocv"
 )
@@ -16,6 +17,139 @@ type calibrationResults struct {
 	referenceColors []color.RGBA
 }
 
+func chessBoardCalibration(webcam *gocv.VideoCapture, debugwindow, projection *gocv.Window) calibrationResults {
+	w := 13
+	h := 6
+	// prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
+	objectPoints := gocv.NewPoints3fVector()
+	defer objectPoints.Close()
+
+	gocv.NewPoint3fVectorFromPoints([]gocv.Point3f{})
+
+	objp := make([][]float32, w*h)
+	for i := range objp {
+		objp[i] = make([]float32, 3)
+	}
+
+	for i := 0; i < h; i++ {
+		p3fv := gocv.NewPoint3fVector()
+		defer p3fv.Close()
+
+		for j := 0; j < w; j++ {
+			objp[i*w+j][0] = float32(j)
+			objp[i*w+j][1] = float32(i)
+			objp[i*w+j][2] = float32(0)
+
+			point := gocv.NewPoint3f(objp[i*w+j][0], objp[i*w+j][1], objp[i*w+j][2])
+			p3fv.Append(point)
+		}
+
+		objectPoints.Append(p3fv)
+	}
+
+	img := gocv.NewMat()
+	defer img.Close()
+
+	cimg := gocv.NewMatWithSize(beamerHeight, beamerWidth, gocv.MatTypeCV8UC3)
+
+	fi := frameInput{
+		webcam:      webcam,
+		debugWindow: debugwindow,
+		projection:  projection,
+		img:         img,
+		cimg:        cimg,
+	}
+
+	termCriteria := gocv.NewTermCriteria(gocv.Count+gocv.EPS, 30, 0.001)
+	waitMillis := 100
+
+	for {
+		start := time.Now()
+		if ok := fi.webcam.Read(&fi.img); !ok {
+			break
+		}
+		if fi.img.Empty() {
+			continue
+		}
+
+		gray_img := gocv.NewMat()
+		defer gray_img.Close()
+
+		// convert the rgb frame into gray
+		gocv.CvtColor(fi.img, &gray_img, gocv.ColorBGRToGray)
+
+		// Find the chess board corners
+		corners := gocv.NewMat()
+		defer corners.Close()
+		found := gocv.FindChessboardCorners(gray_img, image.Pt(w, h), &corners, gocv.CalibCBAdaptiveThresh+gocv.CalibCBFastCheck)
+		fnd_str := ""
+		if found {
+			fnd_str = "FOUND"
+		} else {
+			fnd_str = "NOT FOUND"
+		}
+		gocv.PutText(&fi.img, fnd_str, image.Pt(0, 40), 0, .5, color.RGBA{255, 0, 0, 0}, 2)
+
+		if found {
+			corners2 := gocv.NewMat()
+			defer corners2.Close()
+			corners.CopyTo(&corners2)
+			gocv.CornerSubPix(gray_img, &corners2, image.Pt(11, 11), image.Pt(-1, -1), termCriteria)
+
+			imgPoints := gocv.NewPoints2fVector()
+			defer imgPoints.Close()
+
+			for i := 0; i < h; i++ {
+				points := []gocv.Point2f{}
+
+				for j := 0; j < w; j++ {
+					p := corners2.GetVecfAt(i, j)
+					points = append(points, gocv.Point2f{X: p[0], Y: p[1]})
+				}
+
+				pv := gocv.NewPoint2fVectorFromPoints(points)
+				defer pv.Close()
+				imgPoints.Append(pv)
+
+				// // log the corners to the terminal
+				// fmt.Println(imgpoints.ToPoints())
+			}
+
+			mtx := gocv.NewMat()
+			defer mtx.Close()
+			dist := gocv.NewMat()
+			defer dist.Close()
+			rvecs := gocv.NewMat()
+			defer rvecs.Close()
+			tvecs := gocv.NewMat()
+			defer tvecs.Close()
+			result := gocv.CalibrateCamera(objectPoints, imgPoints, image.Pt(w, h), &mtx, &dist, &rvecs, &tvecs, 0)
+			fmt.Println(result)
+
+			break
+		}
+
+		// Draw and display the corners
+		gocv.DrawChessboardCorners(&fi.img, image.Pt(w, h), corners, found)
+
+		fps := time.Second / time.Since(start)
+		gocv.PutText(&fi.img, fmt.Sprintf("FPS: %d", fps), image.Pt(0, 20), 0, .5, color.RGBA{}, 2)
+
+		fi.debugWindow.IMShow(fi.img)
+		fi.projection.IMShow(fi.cimg)
+		key := fi.debugWindow.WaitKey(waitMillis)
+		if key >= 0 {
+			break
+		}
+	}
+
+	pixPerCM := 0 / 1.0
+	displacement := point{0, 0}
+	displayRatio := 1.0
+	colorSamples := []color.RGBA{}
+	return calibrationResults{pixPerCM, displacement, displayRatio, colorSamples}
+}
+
 func calibration(webcam *gocv.VideoCapture, debugwindow, projection *gocv.Window) calibrationResults {
 	//calibrationPage()
 

talk/vision.go

@@ -14,7 +14,8 @@ import (
 var (
 	// detected from webcam output instead!
 	//webcamWidth, webcamHeight = 1280, 720
-	beamerWidth, beamerHeight = 1280, 720
+	// beamerWidth, beamerHeight = 1280, 720
+	beamerWidth, beamerHeight = 1920, 1080
 )
 
 func Run() {
@@ -29,7 +30,8 @@ func Run() {
 	projection := gocv.NewWindow("projector")
 	defer projection.Close()
 
-	cResults := calibration(webcam, debugwindow, projection)
+	cResults := chessBoardCalibration(webcam, debugwindow, projection)
+	// cResults := calibration(webcam, debugwindow, projection)
 	fmt.Println(cResults)
 	/*
 		cResults := calibrationResults{
@@ -39,7 +41,7 @@ func Run() {
 			referenceColors: []color.RGBA{{201, 66, 67, 0}, {88, 101, 65, 0}, {74, 57, 88, 0}, {217, 109, 72, 0}},
 		}
 	*/
-	vision(webcam, debugwindow, projection, cResults)
+	// vision(webcam, debugwindow, projection, cResults)
 }
 
 type frameInput struct {
@@ -70,7 +72,7 @@ func frameloop(fi frameInput, f func(image.Image, map[image.Rectangle][]circle),
 		gocv.PutText(&fi.img, fmt.Sprintf("FPS: %d", fps), image.Pt(0, 20), 0, .5, color.RGBA{}, 2)
 
 		fi.debugWindow.IMShow(fi.img)
-		fi.projection.IMShow(fi.cimg)
+		// fi.projection.IMShow(fi.cimg)
 		key := fi.debugWindow.WaitKey(waitMillis)
 		if key >= 0 {
 			return nil
@@ -162,6 +164,7 @@ func vision(webcam *gocv.VideoCapture, debugwindow, projection *gocv.Window, cRe
 			cornersByTop[corner.m.p] = corner
 			corners = append(corners, corner)
 		}
+		// fmt.Print("corners: ", corners)
 
 		// attempt to update corners if their colors dont match corner that was really close to it previous frame
 		// persisted corners are guaranteed to have matched an existing page