From b417ce0584ed4b0c76612b1a0b3b205a17f416da Mon Sep 17 00:00:00 2001 From: Nikita Kiselov Date: Wed, 10 Mar 2021 17:30:44 +0200 Subject: [PATCH 01/14] Added new gestures and detection parameters tuning --- config.txt | 3 ++- gestures/tello_gesture_controller.py | 27 +++++++++++++------ main.py | 7 +++-- .../keypoint_classifier_label.csv | 2 +- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/config.txt b/config.txt index cc6f06f..267a3f1 100644 --- a/config.txt +++ b/config.txt @@ -1,6 +1,7 @@ device = 0 width = 960 height = 540 -min_detection_confidence = 0.5 +min_detection_confidence = 0.7 min_tracking_confidence = 0.5 +buffer_len = 5 is_keyboard = True \ No newline at end of file diff --git a/gestures/tello_gesture_controller.py b/gestures/tello_gesture_controller.py index c31cabb..da3824e 100644 --- a/gestures/tello_gesture_controller.py +++ b/gestures/tello_gesture_controller.py @@ -1,5 +1,6 @@ from djitellopy import Tello + class TelloGestureController: def __init__(self, tello: Tello): self.tello = tello @@ -16,23 +17,33 @@ def gesture_control(self, gesture_buffer): print("GESTURE", gesture_id) if not self._is_landing: - if gesture_id == 0: + if gesture_id == 0: # Forward self.forw_back_velocity = 30 - elif gesture_id == 1: + elif gesture_id == 1: # STOP + self.forw_back_velocity = self.up_down_velocity = \ + self.left_right_velocity = self.yaw_velocity = 0 + if gesture_id == 5: # Back self.forw_back_velocity = -30 - elif gesture_id == 2: - self.forw_back_velocity = 0 - elif gesture_id == 3: + + elif gesture_id == 2: # UP + self.up_down_velocity = 25 + elif gesture_id == 4: # DOWN + self.up_down_velocity = -25 + + elif gesture_id == 3: # LAND self._is_landing = True self.forw_back_velocity = self.up_down_velocity = \ self.left_right_velocity = self.yaw_velocity = 0 self.tello.land() + + elif gesture_id == 6: # LEFT + self.left_right_velocity = 20 + elif gesture_id == 7: # RIGHT + self.left_right_velocity = -20 + elif gesture_id == -1: self.forw_back_velocity = self.up_down_velocity = \ self.left_right_velocity = self.yaw_velocity = 0 self.tello.send_rc_control(self.left_right_velocity, self.forw_back_velocity, self.up_down_velocity, self.yaw_velocity) - - - diff --git a/main.py b/main.py index efe6024..88ef4db 100644 --- a/main.py +++ b/main.py @@ -29,6 +29,9 @@ def get_args(): parser.add("--min_tracking_confidence", help='min_tracking_confidence', type=float) + parser.add("--buffer_len", + help='Length of gesture buffer', + type=int) args = parser.parse_args() @@ -64,7 +67,7 @@ def main(): # Take-off drone - # tello.takeoff() + tello.takeoff() cap = tello.get_frame_read() @@ -74,7 +77,7 @@ def main(): gesture_detector = GestureRecognition(args.use_static_image_mode, args.min_detection_confidence, args.min_tracking_confidence) - gesture_buffer = GestureBuffer(buffer_len=5) + gesture_buffer = GestureBuffer(buffer_len=args.buffer_len) def tello_control(key, keyboard_controller, gesture_controller): global gesture_buffer diff --git a/model/keypoint_classifier/keypoint_classifier_label.csv b/model/keypoint_classifier/keypoint_classifier_label.csv index 3198d4b..0969a59 100644 --- a/model/keypoint_classifier/keypoint_classifier_label.csv +++ b/model/keypoint_classifier/keypoint_classifier_label.csv @@ -1,7 +1,7 @@ Forward Stop Up -OK +Land Down Back Left From ed0ac5bfd5500b2900bdb6740be755a99bc5c3f5 Mon Sep 17 00:00:00 2001 From: Nikita Kiselov Date: Thu, 11 Mar 2021 16:31:47 +0200 Subject: [PATCH 02/14] Update README.md --- README.md | 149 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 148 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index aa36d3e..8e0d986 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,151 @@ -# tello-gesture-control +# DJI Tello Visual Gesture control + +## Contains of the repo + +## Index +1. [Introduction](#Introduction) +2. [Setup](#Setup) + 1. [Install pip packages](#1.-Installing-pip-packages) + 2. [Connect and test Tello](#2.-Connect-Tello) +3. [Usage](#Usage) + 1. [Keyboard control](#Keyboard control) + 2. [Gesture control](#Gesture control) +4. [Adding new gestures](#Adding new gestures) + 1. [Technical description](#Technical details of gesture detector) + 2. [Creating dataset](#Creating dataset with new geastures) + 3. [Retrain model](#Notebook for retraining model) +5. [Repository structure](#Repository structure) + +## Introduction +... +## Setup +### 1. Installing pip packages +First, we need to install python dependencies. Make sure you that you are using `python3.7` + +List of packages +```sh +ConfigArgParse == 1.2.3 +djitellopy == 1.5 +numpy == 1.19.3 +opencv_python == 4.5.1.48 +tensorflow == 2.4.1 +mediapipe == 0.8.2 +``` + +Install +```sh +# requirements.txt +pip3 install -r requirements.txt +``` +### 2. Connect Tello +Turn on drone and connect computer to its WiFi + +Next, run the following code to verify connectivity + +wifi.png + +```sh +python3 tests/test_connection.py +``` + +On successful connection + +```json +Send command: command +Response: b'ok' +``` + +If you get such output, you may need to check your connection with the drone + +```json +Send command: command +Timeout exceed on command command +Command command was unsuccessful. Message: False +``` + +## Usage +The most interesting part is demo. There are 2 types of control: keyboard and gesture. You can change between control types during the flight. Below is a complete description of both types. + +Run the following command to start the tello control : + +```sh +python3 main.py +``` + +This script will start the python window with visualization like this: + +WINDOW.img + +### Keyboard control +(To control the drone with your keyboard, first press the `Left Shift` key.) + +The following is a list of keys and action description - + +* `Left Shift` -> Toggle Keyboard controls +* `Right Shft` -> Take off drone +* `Space` -> Land drone +* `Up arrow` -> Increase Altitude +* `Down arrow` -> Decrease Altitude +* `Left arrow` -> Pan left +* `Right arrow` -> Pan right +* `w` -> Move forward +* `a` -> Move left +* `s` -> Move down +* `d` -> Move right + +### Gesture control + +By pressing `g` you activate gesture control mode. Here is a full list of gestures that are available now. + +GESTURES_IMAGE.img + +## Adding new gestures +Hand recognition detector can add and change training data to retrain the model on the own gestures. But before this, +there are technical details of the detector to understand how it works and how it can be improved +### Technical details of gesture detector +Mediapipe Hand keypoints recognition is returning 3D coordinated of 20 hand landmarks. For our +model we will use only 2D coordinates. + +landmarks.png + +Than, this points are preprocessed for training the model in the following way. + +preprocessing.png + +After that, we can use data to train our model. Keypoint calssifier is a simple Neural network with such +structure + +neural_network_structure.png + +*you can check how the structure was formed +### Creating dataset with new gestures +Press "k" to enter the mode to save key points（displayed as 「MODE:Logging Key Point」） + +mode.img + +If you press "0" to "9", the key points will be added to "model/keypoint_classifier/keypoint.csv" as shown below. +1st column: Pressed number (used as class ID), 2nd and subsequent columns: Key point coordinates + +table.img + +In the initial state, 7 types of learning data are included as was shown [here](#Gesture control). If necessary, add 3 or later, or delete the existing data of csv to prepare the training data. +### Notebook for retraining model +Open "[Keypoint_model_training.ipynb](Keypoint_model_training.ipynb)" in Jupyter Notebook or Google Colab. +Change the number of training data classes,the value of "NUM_CLASSES = 3", and path to teh dataset. Then, execute all cells +and download `.tflite` model + +showgif.gif + +Do not forget to modify or add labels in `"model/keypoint_classifier/keypoint_classifier_label.csv"` + +Bonus + +The last part of the notebook is a grid search for model using TensorBoard. Run the GridSearch part of the notebook to +get test result with different parameters + +grid_search.img + +## Repository structure # TODO - [ ] Motion gesture support (LSTM) From ccfd535ad9c669cd426ed6f682598907f8e60711 Mon Sep 17 00:00:00 2001 From: Nikita Kiselov Date: Fri, 12 Mar 2021 12:52:06 +0200 Subject: [PATCH 03/14] Added connection test and updated README --- README.md | 113 +++++++++++++++++++++++++++++++-------- tests/connection_test.py | 14 +++++ 2 files changed, 105 insertions(+), 22 deletions(-) create mode 100644 tests/connection_test.py diff --git a/README.md b/README.md index 8e0d986..2866ed4 100644 --- a/README.md +++ b/README.md @@ -12,12 +12,17 @@ 2. [Gesture control](#Gesture control) 4. [Adding new gestures](#Adding new gestures) 1. [Technical description](#Technical details of gesture detector) - 2. [Creating dataset](#Creating dataset with new geastures) + 2. [Creating dataset](#Creating dataset with new gestures) 3. [Retrain model](#Notebook for retraining model) 5. [Repository structure](#Repository structure) ## Introduction ... +The main goal of this project is to control drone using hand gestures without any gloves or additional equipment. +Just camera on the drone or your smartphone(soon), laptop and human hand. + +**laptop+drone+hand.img** + ## Setup ### 1. Installing pip packages First, we need to install python dependencies. Make sure you that you are using `python3.7` @@ -34,7 +39,6 @@ mediapipe == 0.8.2 Install ```sh -# requirements.txt pip3 install -r requirements.txt ``` ### 2. Connect Tello @@ -42,7 +46,7 @@ Turn on drone and connect computer to its WiFi Next, run the following code to verify connectivity -wifi.png +**wifi.png** ```sh python3 tests/test_connection.py @@ -51,16 +55,29 @@ python3 tests/test_connection.py On successful connection ```json +1. Connection test: Send command: command Response: b'ok' + + +2. Video stream test: +Send command: streamon +Response: b'ok' ``` If you get such output, you may need to check your connection with the drone ```json +1. Connection test: Send command: command Timeout exceed on command command Command command was unsuccessful. Message: False + + +2. Video stream test: +Send command: streamon +Timeout exceed on command streamon +Command streamon was unsuccessful. Message: False ``` ## Usage @@ -74,30 +91,33 @@ python3 main.py This script will start the python window with visualization like this: -WINDOW.img +**WINDOW.img** ### Keyboard control (To control the drone with your keyboard, first press the `Left Shift` key.) The following is a list of keys and action description - -* `Left Shift` -> Toggle Keyboard controls -* `Right Shft` -> Take off drone +* `k` -> Toggle Keyboard controls +* `g` -> Toggle Gesture controls +* `Left Shift` -> Take off drone #TODO * `Space` -> Land drone -* `Up arrow` -> Increase Altitude -* `Down arrow` -> Decrease Altitude -* `Left arrow` -> Pan left -* `Right arrow` -> Pan right * `w` -> Move forward +* `s` -> Move back * `a` -> Move left -* `s` -> Move down * `d` -> Move right +* `e` -> Rotate clockwise +* `q` -> Rotate counter-clockwise +* `r` -> Move up +* `f` -> Move down +* `Esc` -> End program and land the drone + ### Gesture control By pressing `g` you activate gesture control mode. Here is a full list of gestures that are available now. -GESTURES_IMAGE.img +**GESTURES_IMAGE.img** ## Adding new gestures Hand recognition detector can add and change training data to retrain the model on the own gestures. But before this, @@ -106,27 +126,27 @@ there are technical details of the detector to understand how it works and how i Mediapipe Hand keypoints recognition is returning 3D coordinated of 20 hand landmarks. For our model we will use only 2D coordinates. -landmarks.png +**landmarks.png** Than, this points are preprocessed for training the model in the following way. -preprocessing.png +**preprocessing.png** -After that, we can use data to train our model. Keypoint calssifier is a simple Neural network with such +After that, we can use data to train our model. Keypoint classifier is a simple Neural network with such structure -neural_network_structure.png +**neural_network_structure.png** -*you can check how the structure was formed +_check [here](#Grid-Search) to understand how the architecture was selected_ ### Creating dataset with new gestures Press "k" to enter the mode to save key points（displayed as 「MODE:Logging Key Point」） -mode.img +**mode.img** If you press "0" to "9", the key points will be added to "model/keypoint_classifier/keypoint.csv" as shown below. 1st column: Pressed number (used as class ID), 2nd and subsequent columns: Key point coordinates -table.img +**table.img** In the initial state, 7 types of learning data are included as was shown [here](#Gesture control). If necessary, add 3 or later, or delete the existing data of csv to prepare the training data. ### Notebook for retraining model @@ -134,18 +154,67 @@ Open "[Keypoint_model_training.ipynb](Keypoint_model_training.ipynb)" in Jupyter Change the number of training data classes,the value of "NUM_CLASSES = 3", and path to teh dataset. Then, execute all cells and download `.tflite` model -showgif.gif +**showgif.gif** Do not forget to modify or add labels in `"model/keypoint_classifier/keypoint_classifier_label.csv"` -Bonus +#### Grid Search The last part of the notebook is a grid search for model using TensorBoard. Run the GridSearch part of the notebook to get test result with different parameters -grid_search.img +**grid_search.img** ## Repository structure +
+│  main.py
+│  Keypoint_model_training.ipynb
+│  config.txt
+│  requirements.txt
+│  
+├─model
+│  └─keypoint_classifier
+│      │  keypoint.csv
+│      │  keypoint_classifier.hdf5
+│      │  keypoint_classifier.py
+│      │  keypoint_classifier.tflite
+│      └─ keypoint_classifier_label.csv
+│ 
+├─gestures
+│   │  gesture_recognition.py
+│   │  tello_gesture_controller.py
+│   └─ tello_keyboard_controller.py
+│          
+├─tests
+│   └─connection_test.py
+│ 
+└─utils
+    └─cvfpscalc.py
+
+### app.py +Main app which controls functionality of drone control and gesture recognition
+App also includes mode to collect training data for adding new gestures.
+ +### keypoint_classification.ipynb +This is a model training script for hand sign recognition. + +### model/keypoint_classifier +This directory stores files related to gesture recognition.
+ +* Training data(keypoint.csv) +* Trained model(keypoint_classifier.tflite) +* Label data(keypoint_classifier_label.csv) +* Inference module(keypoint_classifier.py) + +### gestures/ +This directory stores files related to drone controllers and gesture modules.
+ +* Keyboard controller (tello_keyboard_controller.py) +* Gesture controller(tello_keyboard_controller.py) +* Gesture recognition module(keypoint_classifier_label.csv) + +### utils/cvfpscalc.py +Module for FPS measurement. # TODO - [ ] Motion gesture support (LSTM) diff --git a/tests/connection_test.py b/tests/connection_test.py new file mode 100644 index 0000000..d9be227 --- /dev/null +++ b/tests/connection_test.py @@ -0,0 +1,14 @@ +from djitellopy import Tello + +if __name__ == '__main__': + + print('1. Connection test:') + tello = Tello() + tello.connect() + print('\n') + + print('2. Video stream test:') + tello.streamon() + print('\n') + + tello.end() From ce9654e9312695eb7ad0851a35356cff92768f03 Mon Sep 17 00:00:00 2001 From: Nikita Kiselov Date: Fri, 12 Mar 2021 12:53:51 +0200 Subject: [PATCH 04/14] Index fix --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2866ed4..749ff27 100644 --- a/README.md +++ b/README.md @@ -8,12 +8,12 @@ 1. [Install pip packages](#1.-Installing-pip-packages) 2. [Connect and test Tello](#2.-Connect-Tello) 3. [Usage](#Usage) - 1. [Keyboard control](#Keyboard control) - 2. [Gesture control](#Gesture control) + * [Keyboard control](#Keyboard control) + * [Gesture control](#Gesture control) 4. [Adding new gestures](#Adding new gestures) - 1. [Technical description](#Technical details of gesture detector) - 2. [Creating dataset](#Creating dataset with new gestures) - 3. [Retrain model](#Notebook for retraining model) + * [Technical description](#Technical details of gesture detector) + * [Creating dataset](#Creating dataset with new gestures) + * [Retrain model](#Notebook for retraining model) 5. [Repository structure](#Repository structure) ## Introduction From 90d375d016862a79df43b205ecfa058ce6c14de2 Mon Sep 17 00:00:00 2001 From: Nikita Kiselov Date: Fri, 12 Mar 2021 12:56:22 +0200 Subject: [PATCH 05/14] Fix index v2 --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 749ff27..e39c4b2 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,13 @@ 1. [Install pip packages](#1.-Installing-pip-packages) 2. [Connect and test Tello](#2.-Connect-Tello) 3. [Usage](#Usage) - * [Keyboard control](#Keyboard control) - * [Gesture control](#Gesture control) -4. [Adding new gestures](#Adding new gestures) - * [Technical description](#Technical details of gesture detector) - * [Creating dataset](#Creating dataset with new gestures) - * [Retrain model](#Notebook for retraining model) -5. [Repository structure](#Repository structure) + * [Keyboard control](##Keyboard-control) + * [Gesture control](#Gesture-control) +4. [Adding new gestures](#Adding-new-gestures) + * [Technical description](#Technical-details-of-gesture-detector) + * [Creating dataset](#Creating-dataset-with-new-gestures) + * [Retrain model](#Notebook-for-retraining-model) +5. [Repository structure](#Repository-structure) ## Introduction ... From bf3eefcd3d2a8de5b4c1d0e2311d93b73311f1f7 Mon Sep 17 00:00:00 2001 From: Nikita Kiselov Date: Fri, 12 Mar 2021 13:11:25 +0200 Subject: [PATCH 06/14] added git lfs installation to the README --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e39c4b2..5adf190 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,14 @@ structure _check [here](#Grid-Search) to understand how the architecture was selected_ ### Creating dataset with new gestures -Press "k" to enter the mode to save key points（displayed as 「MODE:Logging Key Point」） +First, pull datasets from Git LFS. [Here](https://github.com/git-lfs/git-lfs/wiki/Installation) is the instruction how +to install LFS. Then, run the command to pull default csv files +```sh +git lfs install +git lfs pull +``` + +After that, run `main.py` and press "k" to enter the mode to save key points（displayed as 「MODE:Logging Key Point」） **mode.img** From 58cc5140b9d9680fc8961fcd4d83b89c4735caca Mon Sep 17 00:00:00 2001 From: Nikita Kiselov Date: Fri, 12 Mar 2021 13:18:32 +0200 Subject: [PATCH 07/14] added images --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5adf190..1b11598 100644 --- a/README.md +++ b/README.md @@ -44,9 +44,9 @@ pip3 install -r requirements.txt ### 2. Connect Tello Turn on drone and connect computer to its WiFi -Next, run the following code to verify connectivity +wifi_connection -**wifi.png** +Next, run the following code to verify connectivity ```sh python3 tests/test_connection.py @@ -117,7 +117,8 @@ The following is a list of keys and action description - By pressing `g` you activate gesture control mode. Here is a full list of gestures that are available now. -**GESTURES_IMAGE.img** +![gestures](https://user-images.githubusercontent.com/13486777/110933057-f1035e80-8334-11eb-8458-988af973804e.JPG) + ## Adding new gestures Hand recognition detector can add and change training data to retrain the model on the own gestures. But before this, @@ -126,7 +127,8 @@ there are technical details of the detector to understand how it works and how i Mediapipe Hand keypoints recognition is returning 3D coordinated of 20 hand landmarks. For our model we will use only 2D coordinates. -**landmarks.png** +![landmarks_list](https://user-images.githubusercontent.com/13486777/110933339-49d2f700-8335-11eb-9588-5f68a2677ff0.png) + Than, this points are preprocessed for training the model in the following way. From 984ec28b391bbea7e34008d90861637648e4b8e8 Mon Sep 17 00:00:00 2001 From: Nikita Kiselov Date: Mon, 15 Mar 2021 17:08:24 +0200 Subject: [PATCH 08/14] Added intro video --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b11598..25cc9b6 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,9 @@ 5. [Repository structure](#Repository-structure) ## Introduction -... + +![ezgif com-optimize](https://user-images.githubusercontent.com/13486777/111168690-fb2e9280-85aa-11eb-894f-fe70633072fd.gif) + The main goal of this project is to control drone using hand gestures without any gloves or additional equipment. Just camera on the drone or your smartphone(soon), laptop and human hand. @@ -46,6 +48,7 @@ Turn on drone and connect computer to its WiFi wifi_connection + Next, run the following code to verify connectivity ```sh From a8212a8d58c49daff79f9a47ba12e7e38969ea17 Mon Sep 17 00:00:00 2001 From: Nikita Kiselov Date: Tue, 16 Mar 2021 12:38:04 +0200 Subject: [PATCH 09/14] Added all required media --- README.md | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 25cc9b6..ed45b5b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ # DJI Tello Visual Gesture control -## Contains of the repo +![ezgif com-optimize](https://user-images.githubusercontent.com/13486777/111168690-fb2e9280-85aa-11eb-894f-fe70633072fd.gif) + +The main goal of this project is to control drone using hand gestures without any gloves or additional equipment. +Just camera on the drone or your smartphone(soon), laptop and human hand. ## Index 1. [Introduction](#Introduction) @@ -15,15 +18,14 @@ * [Creating dataset](#Creating-dataset-with-new-gestures) * [Retrain model](#Notebook-for-retraining-model) 5. [Repository structure](#Repository-structure) - -## Introduction - -![ezgif com-optimize](https://user-images.githubusercontent.com/13486777/111168690-fb2e9280-85aa-11eb-894f-fe70633072fd.gif) -The main goal of this project is to control drone using hand gestures without any gloves or additional equipment. -Just camera on the drone or your smartphone(soon), laptop and human hand. +## Introduction +This project lies on two main parts - DJI Tello drone and Mediapipe fast hand keypoints recognition.
+DJI Tello is a perfect drone for any kind of programming experiments. It has rich Python API (also Swift is available) which helps to almost fully control drone and utilise its camera for Computer vision.
+On the other hand - Mediapipe. It is an amazing ML platform with many robust solutions like Face mesh, Hand Keypoints detection and Objectron. Moreover, their model can be used on mobile platform with on-device acceleration. -**laptop+drone+hand.img** +Going back to this project, here is a setup that you need: +![starter_pack_signed](https://user-images.githubusercontent.com/13486777/111294166-b65e3680-8652-11eb-8225-c1fb1e5b867d.JPG) ## Setup ### 1. Installing pip packages @@ -94,7 +96,8 @@ python3 main.py This script will start the python window with visualization like this: -**WINDOW.img** +window + ### Keyboard control (To control the drone with your keyboard, first press the `Left Shift` key.) @@ -135,12 +138,14 @@ model we will use only 2D coordinates. Than, this points are preprocessed for training the model in the following way. -**preprocessing.png** +![preprocessing](https://user-images.githubusercontent.com/13486777/111294503-11902900-8653-11eb-9856-a50fe96e750e.png) + After that, we can use data to train our model. Keypoint classifier is a simple Neural network with such structure -**neural_network_structure.png** +![model](https://user-images.githubusercontent.com/13486777/111294522-16ed7380-8653-11eb-9fed-e472c8a9a039.png) + _check [here](#Grid-Search) to understand how the architecture was selected_ ### Creating dataset with new gestures @@ -156,9 +161,9 @@ After that, run `main.py` and press "k" to enter the mode to save key points（d **mode.img** If you press "0" to "9", the key points will be added to "model/keypoint_classifier/keypoint.csv" as shown below. -1st column: Pressed number (used as class ID), 2nd and subsequent columns: Key point coordinates +1st column: Pressed number (class ID), 2nd and subsequent columns: Keypoint coordinates -**table.img** +Screenshot 2021-03-16 at 12 34 06 In the initial state, 7 types of learning data are included as was shown [here](#Gesture control). If necessary, add 3 or later, or delete the existing data of csv to prepare the training data. ### Notebook for retraining model @@ -166,7 +171,8 @@ Open "[Keypoint_model_training.ipynb](Keypoint_model_training.ipynb)" in Jupyter Change the number of training data classes,the value of "NUM_CLASSES = 3", and path to teh dataset. Then, execute all cells and download `.tflite` model -**showgif.gif** +![notebook](https://user-images.githubusercontent.com/13486777/111295516-1ef9e300-8654-11eb-9f59-6f7a85b99076.gif) + Do not forget to modify or add labels in `"model/keypoint_classifier/keypoint_classifier_label.csv"` @@ -175,7 +181,8 @@ Do not forget to modify or add labels in `"model/keypoint_classifier/keypoint_cl The last part of the notebook is a grid search for model using TensorBoard. Run the GridSearch part of the notebook to get test result with different parameters -**grid_search.img** +grid_search + ## Repository structure 

From f8d8814e063e7809f406fa29222a95c005299dc6 Mon Sep 17 00:00:00 2001
From: Nikita Kiselov 
Date: Tue, 16 Mar 2021 12:53:37 +0200
Subject: [PATCH 10/14] Move media to  tag

---
 README.md | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index ed45b5b..e87bd9c 100644
--- a/README.md
+++ b/README.md
@@ -1,9 +1,10 @@
 # DJI Tello Visual Gesture control
 
-![ezgif com-optimize](https://user-images.githubusercontent.com/13486777/111168690-fb2e9280-85aa-11eb-894f-fe70633072fd.gif)
-
 The main goal of this project is to control drone using hand gestures without any gloves or additional equipment.
-Just camera on the drone or your smartphone(soon), laptop and human hand.
+Just camera on the drone or your smartphone(soon), laptop and human hand.

+
+demo_gif
+
 
 ## Index
 1. [Introduction](#Introduction)
@@ -25,7 +26,8 @@ DJI Tello is a perfect drone for any kind of programming experiments. It has ric
 On the other hand - Mediapipe. It is an amazing ML platform with many robust solutions like Face mesh, Hand Keypoints detection and Objectron. Moreover, their model can be used on mobile platform with on-device acceleration.
 
 Going back to this project, here is a setup that you need:
-![starter_pack_signed](https://user-images.githubusercontent.com/13486777/111294166-b65e3680-8652-11eb-8225-c1fb1e5b867d.JPG)
+
+starter_pack
 
 ## Setup
 ### 1. Installing pip packages
@@ -96,7 +98,7 @@ python3 main.py
 
 This script will start the python window with visualization like this:
 
-window
+window
 
 
 ### Keyboard control
@@ -123,8 +125,7 @@ The following is a list of keys and action description -
 
 By pressing `g` you activate gesture control mode. Here is a full list of gestures that are available now.
 
-![gestures](https://user-images.githubusercontent.com/13486777/110933057-f1035e80-8334-11eb-8458-988af973804e.JPG)
-
+gestures_list
 
 ## Adding new gestures
 Hand recognition detector can add and change training data to retrain the model on the own gestures. But before this,
@@ -133,18 +134,19 @@ there are technical details of the detector to understand how it works and how i
 Mediapipe Hand keypoints recognition is returning 3D coordinated of 20 hand landmarks. For our
 model we will use only 2D coordinates.
 
-![landmarks_list](https://user-images.githubusercontent.com/13486777/110933339-49d2f700-8335-11eb-9588-5f68a2677ff0.png)
+gestures_list
 
 
 Than, this points are preprocessed for training the model in the following way.
 
-![preprocessing](https://user-images.githubusercontent.com/13486777/111294503-11902900-8653-11eb-9856-a50fe96e750e.png)
+preprocessing
 
 
 After that, we can use data to train our model. Keypoint classifier is a simple Neural network with such 
 structure
 
-![model](https://user-images.githubusercontent.com/13486777/111294522-16ed7380-8653-11eb-9fed-e472c8a9a039.png)
+model_structure
+
 
 
 _check [here](#Grid-Search) to understand how the architecture was selected_
@@ -163,7 +165,7 @@ After that, run `main.py` and press "k" to enter the mode to save key points（d
 If you press "0" to "9", the key points will be added to "model/keypoint_classifier/keypoint.csv" as shown below.
 1st column: Pressed number (class ID), 2nd and subsequent columns: Keypoint coordinates
 
-Screenshot 2021-03-16 at 12 34 06
+keypoints_table
 
 In the initial state, 7 types of learning data are included as was shown [here](#Gesture control). If necessary, add 3 or later, or delete the existing data of csv to prepare the training data.
 ### Notebook for retraining model
@@ -171,7 +173,7 @@ Open "[Keypoint_model_training.ipynb](Keypoint_model_training.ipynb)" in Jupyter
 Change the number of training data classes,the value of "NUM_CLASSES = 3", and path to teh dataset. Then, execute all cells
 and download `.tflite` model
 
-![notebook](https://user-images.githubusercontent.com/13486777/111295516-1ef9e300-8654-11eb-9f59-6f7a85b99076.gif)
+notebook_gif
 
 
 Do not forget to modify or add labels in `"model/keypoint_classifier/keypoint_classifier_label.csv"`
@@ -181,7 +183,7 @@ Do not forget to modify or add labels in `"model/keypoint_classifier/keypoint_cl
 The last part of the notebook is a grid search for model using TensorBoard. Run the GridSearch part of the notebook to
 get test result with different parameters
 
-grid_search
+grid_search
 
 
 ## Repository structure

From 1371ce444c8247e0622f723439f1439bce511862 Mon Sep 17 00:00:00 2001
From: Nikita Kiselov 
Date: Tue, 16 Mar 2021 13:00:14 +0200
Subject: [PATCH 11/14] Added colab badge and visualization improvements

---
 README.md                       | 13 ++++++++-----
 gestures/gesture_recognition.py | 12 ++++++------
 main.py                         |  2 +-
 3 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index e87bd9c..2f02d82 100644
--- a/README.md
+++ b/README.md
@@ -158,7 +158,8 @@ git lfs install
 git lfs pull
 ```
 
-After that, run `main.py` and press "k" to enter the mode to save key points（displayed as 「MODE:Logging Key Point」）
+After that, run `main.py` and press "k" to enter the mode to save key points
+(displayed as 「MODE:Logging Key Point」）
 
 **mode.img**
 
@@ -169,8 +170,10 @@ If you press "0" to "9", the key points will be added to "model/keypoint_classif
 
 In the initial state, 7 types of learning data are included as was shown [here](#Gesture control). If necessary, add 3 or later, or delete the existing data of csv to prepare the training data.
 ### Notebook for retraining model
-Open "[Keypoint_model_training.ipynb](Keypoint_model_training.ipynb)" in Jupyter Notebook or Google Colab.
-Change the number of training data classes,the value of "NUM_CLASSES = 3", and path to teh dataset. Then, execute all cells
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/kinivi/tello-gesture-control/blob/main/Keypoint_model_training.ipynb)
+
+Open [Keypoint_model_training.ipynb](Keypoint_model_training.ipynb) in Jupyter Notebook or Google Colab.
+Change the number of training data classes,the value of **NUM_CLASSES = 3**, and path to the dataset. Then, execute all cells
 and download `.tflite` model
 
 notebook_gif
@@ -179,9 +182,9 @@ and download `.tflite` model
 Do not forget to modify or add labels in `"model/keypoint_classifier/keypoint_classifier_label.csv"`
 
 #### Grid Search
+❗️ Important ❗️ The last part of the notebook is an experimental part of the notebook which main functionality is to test hyperparameters 
+of the model structure. In a nutshell: grid search using TensorBoard visualization. Feel free to use it for your experiments.
 
-The last part of the notebook is a grid search for model using TensorBoard. Run the GridSearch part of the notebook to
-get test result with different parameters
 
 grid_search
 
diff --git a/gestures/gesture_recognition.py b/gestures/gesture_recognition.py
index 3660159..5e95abe 100644
--- a/gestures/gesture_recognition.py
+++ b/gestures/gesture_recognition.py
@@ -458,12 +458,12 @@ def _draw_info_text(self, image, brect, handedness, hand_sign_text,
         cv.putText(image, info_text, (brect[0] + 5, brect[1] - 4),
                    cv.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 1, cv.LINE_AA)
 
-        if finger_gesture_text != "":
-            cv.putText(image, "Finger Gesture:" + finger_gesture_text, (10, 60),
-                       cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 0), 4, cv.LINE_AA)
-            cv.putText(image, "Finger Gesture:" + finger_gesture_text, (10, 60),
-                       cv.FONT_HERSHEY_SIMPLEX, 1.0, (255, 255, 255), 2,
-                       cv.LINE_AA)
+        # if finger_gesture_text != "":
+        #     cv.putText(image, "Finger Gesture:" + finger_gesture_text, (10, 60),
+        #                cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 0), 4, cv.LINE_AA)
+        #     cv.putText(image, "Finger Gesture:" + finger_gesture_text, (10, 60),
+        #                cv.FONT_HERSHEY_SIMPLEX, 1.0, (255, 255, 255), 2,
+        #                cv.LINE_AA)
 
         return image
 
diff --git a/main.py b/main.py
index 88ef4db..f729a19 100644
--- a/main.py
+++ b/main.py
@@ -140,7 +140,7 @@ def tello_battery(tello):
         # Battery status and image rendering
         cv.putText(debug_image, "Battery: {}".format(battery_status), (5, 720 - 5),
                    cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
-        cv.imshow('Hand Gesture Recognition', debug_image)
+        cv.imshow('Tello Gesture Recognition', debug_image)
 
     tello.land()
     tello.end()

From 9d50cb48eaa6ab5d867ee631be007e8b2c7fa08e Mon Sep 17 00:00:00 2001
From: Nikita Kiselov 
Date: Tue, 16 Mar 2021 13:07:12 +0200
Subject: [PATCH 12/14] Fix local link

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 2f02d82..6a9796c 100644
--- a/README.md
+++ b/README.md
@@ -168,7 +168,7 @@ If you press "0" to "9", the key points will be added to "model/keypoint_classif
 
 keypoints_table
 
-In the initial state, 7 types of learning data are included as was shown [here](#Gesture control). If necessary, add 3 or later, or delete the existing data of csv to prepare the training data.
+In the initial state, 7 types of learning data are included as was shown [here](#Gesture-control). If necessary, add 3 or later, or delete the existing data of csv to prepare the training data.
 ### Notebook for retraining model
 [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/kinivi/tello-gesture-control/blob/main/Keypoint_model_training.ipynb)
 
@@ -186,7 +186,7 @@ Do not forget to modify or add labels in `"model/keypoint_classifier/keypoint_cl
 of the model structure. In a nutshell: grid search using TensorBoard visualization. Feel free to use it for your experiments.
 
 
-grid_search
+grid_search
 
 
 ## Repository structure

From 58ecd55f75f4fe853827794c9d1570fa1b236a11 Mon Sep 17 00:00:00 2001
From: Nikita Kiselov 
Date: Tue, 16 Mar 2021 13:25:12 +0200
Subject: [PATCH 13/14] Added writing mode image

---
 README.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 6a9796c..4c73691 100644
--- a/README.md
+++ b/README.md
@@ -159,11 +159,12 @@ git lfs pull
 ```
 
 After that, run `main.py` and press "k" to enter the mode to save key points
-(displayed as 「MODE:Logging Key Point」）
+(displayed as **MODE:Logging Key Point**）
 
-**mode.img**
+writing_mode
 
-If you press "0" to "9", the key points will be added to "model/keypoint_classifier/keypoint.csv" as shown below.
+
+If you press "0" to "9", the key points will be added to [model/keypoint_classifier/keypoint.csv](model/keypoint_classifier/keypoint.csv) as shown below.

 1st column: Pressed number (class ID), 2nd and subsequent columns: Keypoint coordinates
 
 keypoints_table

From afcd4876adf036dba36be7d7adc92ca76b88dac3 Mon Sep 17 00:00:00 2001
From: Nikita Kiselov 
Date: Tue, 16 Mar 2021 13:42:06 +0200
Subject: [PATCH 14/14] Edit grammar and Title

---
 README.md | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index 4c73691..22afef5 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
-# DJI Tello Visual Gesture control
+# DJI Tello Hand Gesture control
 
-The main goal of this project is to control drone using hand gestures without any gloves or additional equipment.
+The main goal of this project is to control the drone using hand gestures without any gloves or additional equipment.
 Just camera on the drone or your smartphone(soon), laptop and human hand.

 
 demo_gif
@@ -21,11 +21,13 @@ Just camera on the drone or your smartphone(soon), laptop and human hand.

 5. [Repository structure](#Repository-structure)
 
 ## Introduction
-This project lies on two main parts - DJI Tello drone and Mediapipe fast hand keypoints recognition.
 
-DJI Tello is a perfect drone for any kind of programming experiments. It has rich Python API (also Swift is available) which helps to almost fully control drone and utilise its camera for Computer vision.

-On the other hand - Mediapipe. It is an amazing ML platform with many robust solutions like Face mesh, Hand Keypoints detection and Objectron. Moreover, their model can be used on mobile platform with on-device acceleration.
+This project relies on two main parts - DJI Tello drone and Mediapipe fast hand keypoints recognition.
 
-Going back to this project, here is a setup that you need:
+DJI Tello is a perfect drone for any kind of programming experiments. It has a rich Python API (also Swift is available) which helps to almost fully control a drone, create drone swarms and utilise its camera for Computer vision.
+
+Mediapipe is an amazing ML platform with many robust solutions like Face mesh, Hand Keypoints detection and Objectron. Moreover, their model can be used on the mobile platforms with on-device acceleration.
+
+Here is a starter-pack that you need:
 
 starter_pack
 
@@ -137,7 +139,7 @@ model we will use only 2D coordinates.
 gestures_list
 
 
-Than, this points are preprocessed for training the model in the following way.
+Then, these points are preprocessed for training the model in the following way.
 
 preprocessing
 
@@ -151,14 +153,14 @@ structure
 
 _check [here](#Grid-Search) to understand how the architecture was selected_
 ### Creating dataset with new gestures
-First, pull datasets from Git LFS. [Here](https://github.com/git-lfs/git-lfs/wiki/Installation) is the instruction how 
+First, pull datasets from Git LFS. [Here](https://github.com/git-lfs/git-lfs/wiki/Installation) is the instruction of how 
 to install LFS. Then, run the command to pull default csv files
 ```sh
 git lfs install
 git lfs pull
 ```
 
-After that, run `main.py` and press "k" to enter the mode to save key points
+After that, run `main.py` and press "n" to enter the mode to save key points
 (displayed as **MODE:Logging Key Point**）
 
 writing_mode
@@ -183,8 +185,7 @@ and download `.tflite` model
 Do not forget to modify or add labels in `"model/keypoint_classifier/keypoint_classifier_label.csv"`
 
 #### Grid Search
-❗️ Important ❗️ The last part of the notebook is an experimental part of the notebook which main functionality is to test hyperparameters 
-of the model structure. In a nutshell: grid search using TensorBoard visualization. Feel free to use it for your experiments.
+❗️ Important ❗️ The last part of the notebook is an experimental part of the notebook which main functionality is to test hyperparameters of the model structure. In a nutshell: grid search using TensorBoard visualization. Feel free to use it for your experiments.
 
 
 grid_search
@@ -217,7 +218,7 @@ of the model structure. In a nutshell: grid search using TensorBoard visualizati
     └─cvfpscalc.py
### app.py -Main app which controls functionality of drone control and gesture recognition
+Main app which controls the functionality of drone control and gesture recognition
App also includes mode to collect training data for adding new gestures.
### keypoint_classification.ipynb