-
Notifications
You must be signed in to change notification settings - Fork 2
/
CS2_tiny.py
854 lines (473 loc) · 104 KB
/
CS2_tiny.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
#!/usr/bin/env python
# coding: utf-8
# # Most of the code is shamefully copied from [theAIGuysCode](https://github.com/theAIGuysCode/YOLOv4-Cloud-Tutorial)
# In[1]:
from google.colab import drive
drive.mount('/content/drive')
# In[2]:
get_ipython().system('ln -s /content/drive/MyDrive /mygdrive')
# In[3]:
get_ipython().system('ls /mygdrive')
# In[4]:
from cv2 import imread, imwrite, hconcat, vconcat, resize, INTER_AREA, FONT_HERSHEY_SIMPLEX, rectangle, putText, INTER_LINEAR, LINE_AA
import os
from numpy import round_, round
from numpy.random import randint
from pandas import DataFrame, concat, read_csv
from json import load
from math import ceil
from time import time, sleep
from google.colab.patches import cv2_imshow
import pandas as pd
# In[5]:
# clone darknet repo
get_ipython().system('git clone https://github.com/nagi1995/darknet')
# In[6]:
# change makefile to have GPU and OPENCV enabled
get_ipython().run_line_magic('cd', 'darknet')
get_ipython().system("sed -i 's/OPENCV=0/OPENCV=1/' Makefile")
#!sed -i 's/GPU=0/GPU=1/' Makefile
#!sed -i 's/CUDNN=0/CUDNN=1/' Makefile
#!sed -i 's/CUDNN_HALF=0/CUDNN_HALF=1/' Makefile
# In[7]:
# verify CUDA
get_ipython().system('/usr/local/cuda/bin/nvcc --version')
# In[8]:
# make darknet (builds darknet so that you can then use the darknet executable file to run or train object detectors)
get_ipython().system('make')
# In[9]:
os.getcwd()
# In[10]:
# this is where my datasets are stored within my Google Drive (I created a yolov4 folder to store all important files for custom training)
get_ipython().system('ls /mygdrive/CS2/vedai/')
# In[11]:
# copy over both datasets into the root directory of the Colab VM (comment out test.zip if you are not using a validation dataset)
get_ipython().system('cp /mygdrive/CS2/vedai/data/train_tiled.zip ../')
get_ipython().system('cp /mygdrive/CS2/vedai/data/cv_tiled.zip ../')
get_ipython().system('cp /mygdrive/CS2/vedai/data/test_tiled.zip ../')
# In[ ]:
# unzip the datasets and their contents so that they are now in /darknet/data/ folder
get_ipython().system('unzip ../cv_tiled.zip -d data/')
get_ipython().system('unzip ../train_tiled.zip -d data/')
get_ipython().system('unzip ../test_tiled.zip -d data/')
get_ipython().system('rm -rf ../test_tiled.zip')
get_ipython().system('rm -rf ../cv_tiled.zip')
get_ipython().system('rm -rf ../train_tiled.zip')
# In[ ]:
# download cfg to google drive and change its name
#!cp cfg/yolov4-tiny-custom.cfg /mygdrive/CS2//vedai/yolo/yolov4-tiny-custom.cfg
# In[ ]:
# to download to local machine (change its name to yolov4-obj.cfg once you download)
#download('cfg/yolov4-custom.cfg')
# Now you need to edit the .cfg to fit your needs based on your object detector. Open it up in a code or text editor to do so.
#
# If you downloaded cfg to google drive you can use the built in **Text Editor** by going to your google drive and double clicking on yolov4-obj.cfg and then clicking on the **Open with** drop down and selectin **Text Editor**.
#
# **(Image from previous tutorial so don't mind different file name)**
#
# 
#
# I recommend having **batch = 64** and **subdivisions = 16** for ultimate results. If you run into any issues then up subdivisions to 32.
#
# Make the rest of the changes to the cfg based on how many classes you are training your detector on.
#
# **Note:**
# I set my **max_batches = 6000**, **steps = 4800, 5400**, I changed the **classes = 1** in the three YOLO layers and **filters = 18** in the three convolutional layers before the YOLO layers.
#
# How to Configure Your Variables:
#
# width = 416
#
# height = 416
# **(these can be any multiple of 32, 416 is standard, you can sometimes improve results by making value larger like 608 but will slow down training)**
#
# max_batches = (# of classes) * 2000
# **(but no less than 6000 so if you are training for 1, 2, or 3 classes it will be 6000, however detector for 5 classes would have max_batches=10000)**
#
# steps = (80% of max_batches), (90% of max_batches)
# **(so if your max_batches = 10000, then steps = 8000, 9000)**
#
# filters = (# of classes + 5) * 3
# **(so if you are training for one class then your filters = 18, but if you are training for 4 classes then your filters = 27)**
#
#
# **Optional:** If you run into memory issues or find the training taking a super long time. In each of the three yolo layers in the cfg, change one line from random = 1 to **random = 0** to speed up training but slightly reduce accuracy of model. Will also help save memory if you run into any memory issues.
#
# In[13]:
# upload the custom .cfg back to cloud VM from Google Drive
get_ipython().system('cp /mygdrive/CS2//vedai/yolo/yolov4-tiny-custom.cfg ./cfg')
# In[14]:
# upload the obj.names and obj.data files to cloud VM from Google Drive
get_ipython().system('cp /mygdrive/CS2/vedai/obj.names ./data')
get_ipython().system('cp /mygdrive/CS2/vedai/obj.data ./data')
# In[15]:
get_ipython().system('cp /mygdrive/CS2/vedai/train_tiled.txt ./data')
get_ipython().system('cp /mygdrive/CS2/vedai/cv_tiled.txt ./data')
get_ipython().system('cp /mygdrive/CS2/vedai/test_tiled.txt ./data')
# In[16]:
# verify that the newly generated train.txt and test.txt can be seen in our darknet/data folder
get_ipython().system('ls data/')
# If you are uncertain whether generating the files worked and want to double check that everything went as planned, double click on **train.txt** on the left side File Explorer and it should look like this.
#
# 
#
# It will contain one line for each training image path.
# In[ ]:
get_ipython().system('wget https://github.com/AlexeyAB/darknet/releases/download/darknet_yolo_v4_pre/yolov4-tiny.conv.29')
# In[ ]:
# train your custom detector! (uncomment %%capture below if you run into memory issues or your Colab is crashing)
# %%capture
get_ipython().system('./darknet detector train data/obj.data cfg/yolov4-tiny-custom.cfg yolov4-tiny.conv.29 -dont_show -map')
# **TRICK**: If for some reason you get an error or your Colab goes idle during training, you have not lost your partially trained model and weights! Every 100 iterations a weights file called **yolov4-obj_last.weights** is saved to **mydrive/yolov4/backup/** folder (wherever your backup folder is). This is why we created this folder in our Google drive and not on the cloud VM. If your runtime crashes and your backup folder was in your cloud VM you would lose your weights and your training progress.
#
# We can kick off training from our last saved weights file so that we don't have to restart! WOOHOO! Just run the following command but with your backup location.
# ```
# !./darknet detector train data/obj.data cfg/yolov4-obj.cfg /mydrive/yolov4/backup/yolov4-obj_last.weights -dont_show
# ```
# In[ ]:
# kick off training from where it last saved
get_ipython().system('./darknet detector train data/obj.data cfg/yolov4-tiny-custom.cfg /mygdrive/CS2/vedai/yolo/backup/yolov4-tiny-custom_last.weights -dont_show -map')
# # Checking the Mean Average Precision (mAP) of Your Model
# If you didn't run the training with the '-map- flag added then you can still find out the mAP of your model after training. Run the following command on any of the saved weights from the training to see the mAP value for that specific weight's file. I would suggest to run it on multiple of the saved weights to compare and find the weights with the highest mAP as that is the most accurate one!
#
# **NOTE:** If you think your final weights file has overfitted then it is important to run these mAP commands to see if one of the previously saved weights is a more accurate model for your classes.
# In[ ]:
get_ipython().system('./darknet detector map data/obj.data cfg/yolov4-tiny-custom.cfg /mygdrive/CS2/vedai/yolo/backup/yolov4-tiny-custom_best.weights -thresh 0.9')
# # Run Your Custom Object Detector!!!
# You have done it! You now have a custom object detector to make your very own detections. Time to test it out and have some fun!
# In[17]:
# need to set our custom cfg to test mode
get_ipython().system("sed -i 's/batch=64/batch=1/' cfg/yolov4-tiny-custom.cfg")
get_ipython().system("sed -i 's/subdivisions=16/subdivisions=1/' cfg/yolov4-tiny-custom.cfg")
# # Detection on vedai [dataset](https://downloads.greyc.fr/vedai/) (dim: 1024 x 1024)
# In[ ]:
get_ipython().system("sed -i 's/width=416/width=1024/' cfg/yolov4-tiny-custom.cfg")
get_ipython().system("sed -i 's/height=416/height=1024/' cfg/yolov4-tiny-custom.cfg")
# In[ ]:
get_ipython().run_cell_magic('time', '', '\n!./darknet detector test data/obj.data cfg/yolov4-tiny-custom.cfg /mygdrive/CS2/vedai/yolo/backup/yolov4-tiny-custom_best.weights /mygdrive/test_image/00000041.png')
# In[ ]:
cv2_imshow(imread("/content/darknet/predictions.jpg"))
# # OS + darknet build occupies 0.92GB RAM
# # RAM usage (1.57 GB)
#
# <img src='https://imgur.com/wzkpTOz.png'>
# # Detections on VisDrone2019 [dataset](https://github.com/VisDrone/VisDrone-Dataset) (dim: 1360 x 725)
# In[ ]:
get_ipython().system("sed -i 's/width=1024/width=1376/' cfg/yolov4-tiny-custom.cfg")
get_ipython().system("sed -i 's/height=1024/height=768/' cfg/yolov4-tiny-custom.cfg")
# In[ ]:
get_ipython().run_cell_magic('time', '', '!./darknet detector test data/obj.data cfg/yolov4-tiny-custom.cfg /mygdrive/CS2/vedai/yolo/backup/yolov4-tiny-custom_best.weights /mygdrive/test_image/0000006_00611_d_0000002.jpg')
# In[ ]:
cv2_imshow(imread("/content/darknet/predictions.jpg"))
# # RAM usage (1.58 GB)
# <img src='https://imgur.com/xnuIkad.png'>
# # Detections on Munich [dataset](https://pbafreesoftware.eoc.dlr.de/MunichDatasetVehicleDetection-2015-old.zip) (dim: 5616 x 3744)
# In[ ]:
get_ipython().system("sed -i 's/width=1376/width=5632/' cfg/yolov4-tiny-custom.cfg")
get_ipython().system("sed -i 's/height=768/height=3744/' cfg/yolov4-tiny-custom.cfg")
# In[ ]:
get_ipython().run_cell_magic('time', '', '!./darknet detector test data/obj.data cfg/yolov4-tiny-custom.cfg /mygdrive/CS2/vedai/yolo/backup/yolov4-tiny-custom_best.weights /mygdrive/test_image/2012-04-26-Muenchen-Tunnel_4K0G0110.JPG')
# # detections
#
# <img src='https://imgur.com/vsqN1U1.png'>
#
# # RAM usage (11.67 GB)
#
# <img src='https://imgur.com/XZErrsw.png'>
# In[ ]:
os.getcwd()
# In[ ]:
get_ipython().system("sed -i 's/width=5632/width=416/' cfg/yolov4-tiny-custom.cfg")
get_ipython().system("sed -i 's/height=3744/height=416/' cfg/yolov4-tiny-custom.cfg")
# # Detection on vedai [dataset](https://downloads.greyc.fr/vedai/)
# 1. without tiling image (compressing image to 416 x 416)
# In[ ]:
get_ipython().run_cell_magic('time', '', '!./darknet detector test data/obj.data cfg/yolov4-tiny-custom.cfg /mygdrive/CS2/vedai/yolo/backup/yolov4-tiny-custom_best.weights /mygdrive/test_image/00000041.png')
# In[ ]:
cv2_imshow(imread("predictions.jpg"))
# 2. with tiling images (dividing images to 416 x 416 sub-images)
# In[ ]:
get_ipython().system('rm -rf "/content/tiled_images/"')
get_ipython().system('mkdir "/content/tiled_images/"')
# In[ ]:
get_ipython().run_cell_magic('time', '', '\n# read image and make tiles each of dimensions size x size\nim_name = "00000041.png"\next = im_name.split(".")[-1]\nim = imread("/mygdrive/test_image/" + im_name)\nsize = 416\nh, w, _ = im.shape\nh_new = ceil(h/size) * size\nw_new = ceil(w/size) * size\nscale_h = h_new/h\nscale_w = w_new/w\nresized_im = resize(im, (w_new, h_new), INTER_LINEAR)\n\n\n\ntiled_images_path = "/content/tiled_images/"\n\ntiled_ims_list = []\n\nfor i in range(h_new//size):\n for j in range(w_new//size):\n tiled = resized_im[i*size : (i+1)*size, j*size : (j+1)*size, :]\n tiled_im_name = tiled_images_path + im_name.split(".")[0] + "_" + str(i) + "_" + str(j) + "." + ext\n tiled_ims_list.append(tiled_im_name)\n df = DataFrame(tiled_ims_list)\n # saving the path of tiled images so as to feed it to yolo model\n df.to_csv("/content/tiled_images.txt", index = False, header = False)\n # saving tiled image\n imwrite(tiled_im_name, tiled)\n\n# detect objects on individual tiles and store detected objects in json format\nstart = time()\n!./darknet detector test data/obj.data cfg/yolov4-tiny-custom.cfg /mygdrive/CS2/vedai/yolo/backup/yolov4-tiny-custom_best.weights -ext_output -dont_show -out /content/result.json < /content/tiled_images.txt \nend = time()\n\n# read objects location from json and store in .txt files\nf = open("/content/result.json", )\nresult = load(f)\nf.close()\nfor res in result:\n if len(res["objects"]) > 0:\n annot_path = res["filename"].replace("." + ext, ".txt")\n lines = []\n for obj in res["objects"]:\n coords = obj["relative_coordinates"]\n lines.append([obj["class_id"], coords["center_x"], coords["center_y"], coords["width"], coords["height"], obj["name"], obj["confidence"]])\n \n df = DataFrame(lines)\n # saving the location of detected objects\n df.to_csv(annot_path, sep = " ", index = False, header = False)\n\n\n\n\n\n# combine tiled images\nannotations_list = []\ncol_list = []\nfor i in range(ceil(h/size)):\n row_list = []\n for j in range(ceil(w/size)):\n tiled_image_name = tiled_images_path + im_name.split(".")[0] + "_" + str(i) + "_" + str(j) + "." + ext\n row_list.append(imread(tiled_image_name))\n annot_path = tiled_image_name.replace("." + ext, ".txt")\n \n if os.path.exists(annot_path): \n annotations = read_csv(annot_path, sep = " ", names = ["class", "xc", "yc", "yolo_w", "yolo_h", "obj_name", "confidence"])\n \n # converting locations from yolo to pascal VOC format\n annotations["w"] = annotations["yolo_w"]*size\n annotations["h"] = annotations["yolo_h"]*size\n annotations["x"] = annotations["xc"]*size - annotations["w"]/2 + j*size\n annotations["y"] = annotations["yc"]*size - annotations["h"]/2 + i*size\n \n # scaling locations to original image\n annotations["w"] = round_(annotations["w"]/scale_w)\n annotations["h"] = round_(annotations["h"]/scale_h)\n annotations["x"] = round_(annotations["x"]/scale_w)\n annotations["y"] = round_(annotations["y"]/scale_h)\n \n annotations = annotations[["class", "x", "y", "w", "h", "obj_name", "confidence"]]\n annotations_list.append(annotations)\n \n # Reference: https://www.geeksforgeeks.org/concatenate-images-using-opencv-in-python/\n col_list.append(hconcat(row_list))\n\nscaled_image = vconcat(col_list)\nreconstructed_image = resize(scaled_image, (w, h), interpolation = INTER_AREA)\n\n#imwrite("reconstructed_" + im_name + "." + ext, reconstructed_image)\ndf = concat(annotations_list, axis = 0)\n#df.to_csv("reconstructed_" + im_name + ".txt", sep = " ", index = False, header = False) \n\n\nfor index, row in df.iterrows():\n rectangle(reconstructed_image, (int(row["x"]), int(row["y"])), (int(row["x"]+row["w"]) , int(row["y"]+row["h"])), (255, 0, 0), 2)\n putText(reconstructed_image, row["obj_name"] + " " + str(round(row["confidence"], 2)), (int(row["x"])-4, int(row["y"])-4), FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 1, LINE_AA)\ncv2_imshow(reconstructed_image)')
# In[ ]:
print(end-start) # 9 sub-images
# # Detections on VisDrone2019 [dataset](https://github.com/VisDrone/VisDrone-Dataset)
# 1. without tiling image (compressing image to 416 x 416)
# In[ ]:
get_ipython().run_cell_magic('time', '', '!./darknet detector test data/obj.data cfg/yolov4-tiny-custom.cfg /mygdrive/CS2/vedai/yolo/backup/yolov4-tiny-custom_best.weights /mygdrive/test_image/0000006_00611_d_0000002.jpg')
# In[ ]:
cv2_imshow(imread("predictions.jpg"))
# 2. with tiling images (dividing images to 416 x 416 sub-images)
# In[ ]:
get_ipython().system('rm -rf "/content/tiled_images/"')
get_ipython().system('mkdir "/content/tiled_images/"')
# In[ ]:
get_ipython().run_cell_magic('time', '', '\n# read image and make tiles each of dimensions size x size\nim_name = "0000006_00611_d_0000002.jpg"\next = im_name.split(".")[-1]\nim = imread("/mygdrive/test_image/" + im_name)\nsize = 416\nh, w, _ = im.shape\nh_new = ceil(h/size) * size\nw_new = ceil(w/size) * size\nscale_h = h_new/h\nscale_w = w_new/w\nresized_im = resize(im, (w_new, h_new), INTER_LINEAR)\n\n\n\ntiled_images_path = "/content/tiled_images/"\n\ntiled_ims_list = []\n\nfor i in range(h_new//size):\n for j in range(w_new//size):\n tiled = resized_im[i*size : (i+1)*size, j*size : (j+1)*size, :]\n tiled_im_name = tiled_images_path + im_name.split(".")[0] + "_" + str(i) + "_" + str(j) + "." + ext\n tiled_ims_list.append(tiled_im_name)\n df = DataFrame(tiled_ims_list)\n # saving the path of tiled images so as to feed it to yolo model\n df.to_csv("/content/tiled_images.txt", index = False, header = False)\n # saving tiled image\n imwrite(tiled_im_name, tiled)\n\n# detect objects on individual tiles and store detected objects in json format\nstart = time()\n!./darknet detector test data/obj.data cfg/yolov4-tiny-custom.cfg /mygdrive/CS2/vedai/yolo/backup/yolov4-tiny-custom_best.weights -ext_output -dont_show -out /content/result.json < /content/tiled_images.txt \nend = time()\n\n# read objects location from json and store in .txt files\nf = open("/content/result.json", )\nresult = load(f)\nf.close()\nfor res in result:\n if len(res["objects"]) > 0:\n annot_path = res["filename"].replace("." + ext, ".txt")\n lines = []\n for obj in res["objects"]:\n coords = obj["relative_coordinates"]\n lines.append([obj["class_id"], coords["center_x"], coords["center_y"], coords["width"], coords["height"], obj["name"], obj["confidence"]])\n \n df = DataFrame(lines)\n # saving the location of detected objects\n df.to_csv(annot_path, sep = " ", index = False, header = False)\n\n\n\n\n\n# combine tiled images\nannotations_list = []\ncol_list = []\nfor i in range(ceil(h/size)):\n row_list = []\n for j in range(ceil(w/size)):\n tiled_image_name = tiled_images_path + im_name.split(".")[0] + "_" + str(i) + "_" + str(j) + "." + ext\n row_list.append(imread(tiled_image_name))\n annot_path = tiled_image_name.replace("." + ext, ".txt")\n \n if os.path.exists(annot_path): \n annotations = read_csv(annot_path, sep = " ", names = ["class", "xc", "yc", "yolo_w", "yolo_h", "obj_name", "confidence"])\n \n # converting locations from yolo to pascal VOC format\n annotations["w"] = annotations["yolo_w"]*size\n annotations["h"] = annotations["yolo_h"]*size\n annotations["x"] = annotations["xc"]*size - annotations["w"]/2 + j*size\n annotations["y"] = annotations["yc"]*size - annotations["h"]/2 + i*size\n \n # scaling locations to original image\n annotations["w"] = round_(annotations["w"]/scale_w)\n annotations["h"] = round_(annotations["h"]/scale_h)\n annotations["x"] = round_(annotations["x"]/scale_w)\n annotations["y"] = round_(annotations["y"]/scale_h)\n \n annotations = annotations[["class", "x", "y", "w", "h", "obj_name", "confidence"]]\n annotations_list.append(annotations)\n \n # Reference: https://www.geeksforgeeks.org/concatenate-images-using-opencv-in-python/\n col_list.append(hconcat(row_list))\n\nscaled_image = vconcat(col_list)\nreconstructed_image = resize(scaled_image, (w, h), interpolation = INTER_AREA)\n\n#imwrite("reconstructed_" + im_name + "." + ext, reconstructed_image)\ndf = concat(annotations_list, axis = 0)\n#df.to_csv("reconstructed_" + im_name + ".txt", sep = " ", index = False, header = False) \n\n\nfor index, row in df.iterrows():\n rectangle(reconstructed_image, (int(row["x"]), int(row["y"])), (int(row["x"]+row["w"]) , int(row["y"]+row["h"])), (255, 0, 0), 2)\n putText(reconstructed_image, row["obj_name"] + " " + str(round(row["confidence"], 2)), (int(row["x"])-4, int(row["y"])-4), FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 1, LINE_AA)\ncv2_imshow(reconstructed_image)\n')
# In[ ]:
print(end-start) # 8 sub-images
# # Detections on Munich [dataset](https://pbafreesoftware.eoc.dlr.de/MunichDatasetVehicleDetection-2015-old.zip)
# 1. without tiling image (compressing image to 416 x 416)
# In[ ]:
get_ipython().run_cell_magic('time', '', '!./darknet detector test data/obj.data cfg/yolov4-tiny-custom.cfg /mygdrive/CS2/vedai/yolo/backup/yolov4-tiny-custom_best.weights /mygdrive/test_image/2012-04-26-Muenchen-Tunnel_4K0G0110.JPG')
# In[ ]:
cv2_imshow(imread("predictions.jpg"))
# <img src='https://imgur.com/FLpTXjY.png'>
# 2. with tiling images (dividing images to 416 x 416 sub-images)
# In[ ]:
get_ipython().system('rm -rf "/content/tiled_images/"')
get_ipython().system('mkdir "/content/tiled_images/"')
# In[ ]:
get_ipython().run_cell_magic('time', '', '\n# read image and make tiles each of dimensions size x size\nim_name = "2012-04-26-Muenchen-Tunnel_4K0G0110.JPG"\next = im_name.split(".")[-1]\nim = imread("/mygdrive/test_image/" + im_name)\nsize = 416\nh, w, _ = im.shape\nh_new = ceil(h/size) * size\nw_new = ceil(w/size) * size\nscale_h = h_new/h\nscale_w = w_new/w\nresized_im = resize(im, (w_new, h_new), INTER_LINEAR)\n\n\n\ntiled_images_path = "/content/tiled_images/"\n\ntiled_ims_list = []\n\nfor i in range(h_new//size):\n for j in range(w_new//size):\n tiled = resized_im[i*size : (i+1)*size, j*size : (j+1)*size, :]\n tiled_im_name = tiled_images_path + im_name.split(".")[0] + "_" + str(i) + "_" + str(j) + "." + ext\n tiled_ims_list.append(tiled_im_name)\n df = DataFrame(tiled_ims_list)\n # saving the path of tiled images so as to feed it to yolo model\n df.to_csv("/content/tiled_images.txt", index = False, header = False)\n # saving tiled image\n imwrite(tiled_im_name, tiled)\n\n\n# detect objects on individual tiles and store detected objects in json format\nstart = time()\n!./darknet detector test data/obj.data cfg/yolov4-tiny-custom.cfg /mygdrive/CS2/vedai/yolo/backup/yolov4-tiny-custom_best.weights -ext_output -dont_show -out /content/result.json < /content/tiled_images.txt \nend = time()\n\n# read objects location from json and store in .txt files\nf = open("/content/result.json", )\nresult = load(f)\nf.close()\nfor res in result:\n if len(res["objects"]) > 0:\n annot_path = res["filename"].replace("." + ext, ".txt")\n lines = []\n for obj in res["objects"]:\n coords = obj["relative_coordinates"]\n lines.append([obj["class_id"], coords["center_x"], coords["center_y"], coords["width"], coords["height"], obj["name"], obj["confidence"]])\n \n df = DataFrame(lines)\n # saving the location of detected objects\n df.to_csv(annot_path, sep = " ", index = False, header = False)\n\n\n\n\n\n# combine tiled images\nannotations_list = []\ncol_list = []\nfor i in range(ceil(h/size)):\n row_list = []\n for j in range(ceil(w/size)):\n tiled_image_name = tiled_images_path + im_name.split(".")[0] + "_" + str(i) + "_" + str(j) + "." + ext\n row_list.append(imread(tiled_image_name))\n annot_path = tiled_image_name.replace("." + ext, ".txt")\n \n if os.path.exists(annot_path): \n annotations = read_csv(annot_path, sep = " ", names = ["class", "xc", "yc", "yolo_w", "yolo_h", "obj_name", "confidence"])\n \n # converting locations from yolo to pascal VOC format\n annotations["w"] = annotations["yolo_w"]*size\n annotations["h"] = annotations["yolo_h"]*size\n annotations["x"] = annotations["xc"]*size - annotations["w"]/2 + j*size\n annotations["y"] = annotations["yc"]*size - annotations["h"]/2 + i*size\n \n # scaling locations to original image\n annotations["w"] = round_(annotations["w"]/scale_w)\n annotations["h"] = round_(annotations["h"]/scale_h)\n annotations["x"] = round_(annotations["x"]/scale_w)\n annotations["y"] = round_(annotations["y"]/scale_h)\n \n annotations = annotations[["class", "x", "y", "w", "h", "obj_name", "confidence"]]\n annotations_list.append(annotations)\n \n # Reference: https://www.geeksforgeeks.org/concatenate-images-using-opencv-in-python/\n col_list.append(hconcat(row_list))\n\nscaled_image = vconcat(col_list)\nreconstructed_image = resize(scaled_image, (w, h), interpolation = INTER_AREA)\n\n#imwrite("reconstructed_" + im_name + "." + ext, reconstructed_image)\ndf = concat(annotations_list, axis = 0)\n#df.to_csv("reconstructed_" + im_name + ".txt", sep = " ", index = False, header = False) \n\n\nfor index, row in df.iterrows():\n rectangle(reconstructed_image, (int(row["x"]), int(row["y"])), (int(row["x"]+row["w"]) , int(row["y"]+row["h"])), (255, 0, 0), 2)\n putText(reconstructed_image, row["obj_name"] + " " + str(round(row["confidence"], 2)), (int(row["x"])-4, int(row["y"])-4), FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 1, LINE_AA)\n#cv2_imshow(reconstructed_image)\nimwrite("/content/reconstructed_" + im_name + "." + ext, reconstructed_image)')
# # detections
#
# <img src='https://imgur.com/6Jqn0nq.png'>
#
# # RAM usage (1.23 GB)
# <img src='https://imgur.com/lmHQwhA.png'>
#
# In[ ]:
print(end-start) # 126 sub-images
# # Error Analysis
# In[ ]:
get_ipython().system('cp /mygdrive/CS2/vedai/data/test.zip ../')
get_ipython().system('unzip ../test.zip -d ../')
get_ipython().system('rm -rf ../test.zip')
# In[100]:
get_ipython().system('rm -rf ../sample_test_data')
get_ipython().system('mkdir ../sample_test_data')
# In[175]:
def datapipeline(path, im_name):
# read image and make tiles each of dimensions size x size
colors = [
(190, 200, 68),
(125, 102, 60),
(54, 186, 32),
(246, 119, 66),
(61, 196, 32),
(139, 126, 1),
(14, 40, 254),
(55, 113, 168),
(144, 45, 240),
(127, 32, 61)
]
obj_names = [
"car",
"truck",
"plane",
"tractor",
"camping_car",
"boat",
"pickup",
"other",
"van"
]
ext = im_name.split(".")[-1]
im = imread(path + im_name)
size = 416
h, w, _ = im.shape
h_new = ceil(h/size) * size
w_new = ceil(w/size) * size
scale_h = h_new/h
scale_w = w_new/w
resized_im = resize(im, (w_new, h_new), INTER_LINEAR)
get_ipython().system('rm -rf "/content/tiled_images/"')
get_ipython().system('mkdir "/content/tiled_images/"')
tiled_images_path = "/content/tiled_images/"
tiled_ims_list = []
for i in range(h_new//size):
for j in range(w_new//size):
tiled = resized_im[i*size : (i+1)*size, j*size : (j+1)*size, :]
tiled_im_name = tiled_images_path + im_name.split(".")[0] + "_" + str(i) + "_" + str(j) + "." + ext
tiled_ims_list.append(tiled_im_name)
df = DataFrame(tiled_ims_list)
# saving the path of tiled images so as to feed it to yolo model
df.to_csv("/content/tiled_images.txt", index = False, header = False)
# saving tiled image
imwrite(tiled_im_name, tiled)
# detect objects on individual tiles and store detected objects in json format
start = time()
get_ipython().system('./darknet detector test data/obj.data cfg/yolov4-tiny-custom.cfg /mygdrive/CS2/vedai/yolo/backup/yolov4-tiny-custom_best.weights -ext_output -dont_show -out /content/result.json < /content/tiled_images.txt -thresh 0.1')
end = time()
# read objects location from json and store in .txt files
f = open("/content/result.json", )
result = load(f)
f.close()
for res in result:
if len(res["objects"]) > 0:
annot_path = res["filename"].replace("." + ext, ".txt")
lines = []
for obj in res["objects"]:
coords = obj["relative_coordinates"]
lines.append([obj["class_id"], coords["center_x"], coords["center_y"], coords["width"], coords["height"], obj["name"], obj["confidence"]])
df = DataFrame(lines)
# saving the location of detected objects
df.to_csv(annot_path, sep = " ", index = False, header = False)
# combine tiled images
annotations_list = []
col_list = []
for i in range(ceil(h/size)):
row_list = []
for j in range(ceil(w/size)):
tiled_image_name = tiled_images_path + im_name.split(".")[0] + "_" + str(i) + "_" + str(j) + "." + ext
row_list.append(imread(tiled_image_name))
annot_path = tiled_image_name.replace("." + ext, ".txt")
if os.path.exists(annot_path):
annotations = read_csv(annot_path, sep = " ", names = ["class", "xc", "yc", "yolo_w", "yolo_h", "obj_name", "confidence"])
# converting locations from yolo to pascal VOC format
annotations["w"] = annotations["yolo_w"]*size
annotations["h"] = annotations["yolo_h"]*size
annotations["x"] = annotations["xc"]*size - annotations["w"]/2 + j*size
annotations["y"] = annotations["yc"]*size - annotations["h"]/2 + i*size
# scaling locations to original image
annotations["w"] = round_(annotations["w"]/scale_w)
annotations["h"] = round_(annotations["h"]/scale_h)
annotations["x"] = round_(annotations["x"]/scale_w)
annotations["y"] = round_(annotations["y"]/scale_h)
annotations = annotations[["class", "x", "y", "w", "h", "obj_name", "confidence"]]
annotations_list.append(annotations)
# Reference: https://www.geeksforgeeks.org/concatenate-images-using-opencv-in-python/
col_list.append(hconcat(row_list))
scaled_image = vconcat(col_list)
reconstructed_image = resize(scaled_image, (w, h), interpolation = INTER_AREA)
sleep(5)
if len(annotations_list) > 0:
df = concat(annotations_list, axis = 0)
for index, row in df.iterrows():
print(row["obj_name"])
rectangle(reconstructed_image, (int(row["x"]), int(row["y"])), (int(row["x"]+row["w"]) , int(row["y"]+row["h"])), colors[obj_names.index(row["obj_name"])], 2)
putText(reconstructed_image, row["obj_name"] + " " + str(round(row["confidence"], 2)), (int(row["x"])-4, int(row["y"])-4), FONT_HERSHEY_SIMPLEX, 1, colors[obj_names.index(row["obj_name"])], 1, LINE_AA)
else:
print("No object detected")
cv2_imshow(reconstructed_image)
# In[126]:
def display_images_and_ground_truth(img_path):
annot_path = img_path.replace(".png", ".txt")
df = pd.read_csv(annot_path, sep = " ", names = ["class", "xc", "yc", "w", "h"])
im = imread(img_path)
h, w, _ = im.shape
df["w"] = round_(df["w"]*w)
df["h"] = round_(df["h"]*h)
df["x"] = round_(df["xc"]*w - df["w"]/2)
df["y"] = round_(df["yc"]*h - df["h"]/2)
for index, row in df.iterrows():
print(obj_names[int(row["class"])])
rectangle(im, (int(row["x"]), int(row["y"])), (int(row["x"]) + int(row["w"]), int(row["y"]) + int(row["h"])), colors[int(row["class"])], 2)
putText(im, obj_names[int(row["class"])], (int(row["x"])-4, int(row["y"])-4), FONT_HERSHEY_SIMPLEX, 1, colors[int(row["class"])], 1, LINE_AA)
cv2_imshow(im)
# In[106]:
import random
import shutil
count = 0
while True:
if count >= 10:
break
f = random.choice(os.listdir("/content/test"))
if f.endswith(".png"):
print(f)
count += 1
# ### detection-1
# In[127]:
display_images_and_ground_truth("/content/test/00000896.png")
# In[150]:
datapipeline("/content/test/", "00000896.png")
# * Observation: There are only two objects in the image and both of them are correctly detected with a very good machine confidence
# ### detection-2
# In[135]:
display_images_and_ground_truth("/content/test/00000220.png")
# In[151]:
datapipeline("/content/test/", "00000220.png")
# * Only one object is present in the image, but our model found two objects. The **car** class detected has machine confidence of 0.25 and **pickup** has very good machine confidence.
# ### detection-3
# In[138]:
display_images_and_ground_truth("/content/test/00001221.png")
# In[168]:
datapipeline("/content/test/", "00001221.png")
# * No object is detected even if the machine confidence is made to 0.01.
# ### detection-4
# In[141]:
display_images_and_ground_truth("/content/test/00000221.png")
# In[153]:
datapipeline("/content/test/", "00000221.png")
# * Two objects are detected with a good machine confidence.
# ### detection-5
# In[143]:
display_images_and_ground_truth("/content/test/00001135.png")
# In[158]:
# machine confidence threshold = 0.25
datapipeline("/content/test/", "00001135.png")
# In[171]:
# machine confidence threshold = 0.15
datapipeline("/content/test/", "00001135.png")
# * One important thing to note is when images are tiled, a single object may atmost be divided into 4 pieces and each piece will be present in one sub-image tile.
# * We can see that when confidence threshold is 0.25, the back portion of **truck** is not detected but when we reduce confidence to 0.15, backportion of **truck** is also detected.
# ### detection-6
# In[145]:
display_images_and_ground_truth("/content/test/00000607.png")
# In[159]:
datapipeline("/content/test/", "00000607.png")
# * **pickup** is detected correctly but **van** is detected as **car** with a fair amount of confidence
# ### detection-7
# In[147]:
display_images_and_ground_truth("/content/test/00000252.png")
# In[160]:
# confidence = 0.25
datapipeline("/content/test/", "00000252.png")
# * From above, we can see that **car** object is detected correctly with very good confidence.
# ### detection-8
# In[161]:
display_images_and_ground_truth("/content/test/00001115.png")
# In[162]:
# confidence = 0.25
datapipeline("/content/test/", "00001115.png")
# * Overlapping objects are detected but one of the detected overlapping object (**car**) has 0.3 confidence and other (**pickup**) has 0.87 confidence.
# * **tractor** and **truck** object is not detected.
# * One of the **truck** object is detected as **pickup** object.
# * There is no ground truth label, but an **pickup** object is detected with confidence = 0.41
# In[177]:
# confidence = 0.1
datapipeline("/content/test/", "00001115.png")
# * When confidence threshold is decreased to 0.1, **tractor** object is detected and there are other overlapping objects detected.
# ### detection-9
# In[163]:
display_images_and_ground_truth("/content/test/00000206.png")
# In[164]:
datapipeline("/content/test/", "00000206.png")
# * There is a single **truck** object and that is detected with good confidence.
# ### detection-10
# In[165]:
display_images_and_ground_truth("/content/test/00000357.png")
# In[166]:
datapipeline("/content/test/", "00000357.png")
# * Both the objects are detected with good confidence.
# # modified notation in ([source](https://towardsdatascience.com/evaluating-performance-of-an-object-detection-model-137a349c517b))
#
# TP: if IoU ≥0.5, classify the object detection as True Positive(TP)
#
# FP: if Iou <0.5, then it is a wrong detection and classify it as False Positive(FP), and when wrong object is detected.
#
# FN: When a ground truth is present in the image and model failed to detect the object, classify it as False Negative(FN).
#
# For above calculation of TP, FP, FN, we will consider machine confidence threshold of 0.25
#
# TP = 18
#
# FP = 4
#
# FN = 4
#
# * For a sample of 10 images, we can see that, most of the objects are getting detected by our model and there are some False Positives and Negatives.
#
#
#
# # mAP of test data
#
# **Note:** We should modify the **valid** path in **obj.names** to **data/test_tiled.txt** for mAP calculation of test data
# In[17]:
get_ipython().system('./darknet detector map data/obj.data cfg/yolov4-tiny-custom.cfg /mygdrive/CS2/vedai/yolo/backup/yolov4-tiny-custom_best.weights')
# # Overall Observations
#
# * When images are tiled, a single object may atmost be divided into 4 pieces and each piece will be present in one sub-image tile.
# * In EDA, it was observed that, **truck**, **pickup** and **van** look similar.
# * **other** class contains two different objects namely **JCB** and **proclainer**