diff --git a/gluoncv/data/pascal_voc/detection.py b/gluoncv/data/pascal_voc/detection.py
index 28b73472c7..e1e378f3bf 100644
--- a/gluoncv/data/pascal_voc/detection.py
+++ b/gluoncv/data/pascal_voc/detection.py
@@ -130,6 +130,9 @@ def _load_label(self, idx):
                 label.append([xmin, ymin, xmax, ymax, cls_id, difficult])
             except AssertionError as e:
                 logging.warning("Invalid label at %s, %s", anno_path, e)
+            if not label:
+                # dummy invalid labels if no valid objects are found
+                label.append([-1, -1, -1, -1, -1, -1])
         return np.array(label)
 
     def _validate_label(self, xmin, ymin, xmax, ymax, width, height):
diff --git a/gluoncv/model_zoo/yolo/yolo_target.py b/gluoncv/model_zoo/yolo/yolo_target.py
index 14def2be6b..eee27d20d0 100644
--- a/gluoncv/model_zoo/yolo/yolo_target.py
+++ b/gluoncv/model_zoo/yolo/yolo_target.py
@@ -103,6 +103,8 @@ def forward(self, img, xs, anchors, offsets, gt_boxes, gt_ids, gt_mixratio=None)
                 for m in range(matches.shape[1]):
                     if valid_gts[b, m] < 1:
                         break
+                    if np_gt_ids[b, m, 0] < 0:  # ignore fake bbox
+                        break
                     match = int(matches[b, m])
                     nlayer = np.nonzero(num_anchors > match)[0][0]
                     height = xs[nlayer].shape[2]