Skip to content
This repository has been archived by the owner on Nov 20, 2023. It is now read-only.

Replaced fixed calibration matrices with the correct ones #76

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Replaced fixed calibration matrices with the correct ones #76

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions data/crop.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,9 @@ def align_img_and_pc(img_dir, pc_dir, calib_dir):
calib_dir = CALIB_ROOT + '%06d.txt' % frame

points = align_img_and_pc(img_dir, pc_dir, calib_dir)

output_name = PC_ROOT + frame + '.bin'
l = str(frame)
l = "0"*(6-len(l)) +l
output_name = PC_ROOT + l + '.bin'
points[:,:4].astype('float32').tofile(output_name)


Expand Down
22 changes: 18 additions & 4 deletions model/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,8 +178,20 @@ def train_step(self, session, data, train=False, summary=False):
vox_number = data[3]
vox_coordinate = data[4]
print('train', tag)
# Load the matrices of the current data samples:
P,Tr,R = [], [], []
for ta in tag:
if "aug" in ta:
tt = ta[4:10]
else:
tt = ta
Pi, Tri, Ri = load_calib( os.path.join( cfg.CALIB_DIR, tt + '.txt' ) )
P.append(Pi)
Tr.append(Tri)
R.append(Ri)

pos_equal_one, neg_equal_one, targets = cal_rpn_target(
label, self.rpn_output_shape, self.anchors, cls=cfg.DETECT_OBJ, coordinate='lidar')
label, self.rpn_output_shape, self.anchors, cls=cfg.DETECT_OBJ, coordinate='lidar',T_VELO_2_CAM=Tr, R_RECT_0=R)
pos_equal_one_for_reg = np.concatenate(
[np.tile(pos_equal_one[..., [0]], 7), np.tile(pos_equal_one[..., [1]], 7)], axis=-1)
pos_equal_one_sum = np.clip(np.sum(pos_equal_one, axis=(
Expand Down Expand Up @@ -269,10 +281,12 @@ def predict_step(self, session, data, summary=False, vis=False):
vox_coordinate = data[4]
img = data[5]
lidar = data[6]

# Currently the prediction does not support batches
P, Tr, R = load_calib( os.path.join( cfg.CALIB_DIR.replace("training","validation"), tag[0] + '.txt' ) )

if summary or vis:
batch_gt_boxes3d = label_to_gt_box3d(
label, cls=self.cls, coordinate='lidar')
label, cls=self.cls, coordinate='lidar', T_VELO_2_CAM=[Tr], R_RECT_0=[R])
print('predict', tag)
input_feed = {}
input_feed[self.is_train] = False
Expand Down Expand Up @@ -320,7 +334,7 @@ def predict_step(self, session, data, summary=False, vis=False):
# only summry 1 in a batch
cur_tag = tag[0]
P, Tr, R = load_calib( os.path.join( cfg.CALIB_DIR, cur_tag + '.txt' ) )

front_image = draw_lidar_box3d_on_image(img[0], ret_box3d[0], ret_score[0],
batch_gt_boxes3d[0], P2=P, T_VELO_2_CAM=Tr, R_RECT_0=R)

Expand Down
3 changes: 2 additions & 1 deletion test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,8 @@
for tag, result in zip(tags, results):
of_path = os.path.join(args.output_path, 'data', tag + '.txt')
with open(of_path, 'w+') as f:
labels = box3d_to_label([result[:, 1:8]], [result[:, 0]], [result[:, -1]], coordinate='lidar')[0]
P, Tr, R = load_calib(os.path.join( cfg.CALIB_DIR.replace("training","validation"), tag + '.txt') )
labels = box3d_to_label([result[:, 1:8]], [result[:, 0]], [result[:, -1]], coordinate='lidar',P2 =P, T_VELO_2_CAM=Tr, R_RECT_0=R)[0]
for line in labels:
f.write(line)
print('write out {} objects to {}'.format(len(labels), tag))
Expand Down
5 changes: 2 additions & 3 deletions train.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ def main(_):
global save_model_dir
start_epoch = 0
global_counter = 0

gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=cfg.GPU_MEMORY_FRACTION,
visible_device_list=cfg.GPU_AVAILABLE,
allow_growth=True)
Expand Down Expand Up @@ -119,12 +118,12 @@ def main(_):
f.write( 'train: {} @ epoch:{}/{} loss: {:.4f} reg_loss: {:.4f} cls_loss: {:.4f} cls_pos_loss: {:.4f} cls_neg_loss: {:.4f} forward time: {:.4f} batch time: {:.4f} \n'.format(counter, epoch, args.max_epoch, ret[0], ret[1], ret[2], ret[3], ret[4], forward_time, batch_time) )

#print(counter, summary_interval, counter % summary_interval)
if counter % summary_interval == 0:
if counter % summary_interval == 0 and False: # Momentaneously this is skipped
print("summary_interval now")
summary_writer.add_summary(ret[-1], global_counter)

#print(counter, summary_val_interval, counter % summary_val_interval)
if counter % summary_val_interval == 0:
if counter % summary_val_interval == 0 and False: # Momentaneously this is skipped
print("summary_val_interval now")
batch = sample_test_data(val_dir, args.single_batch_size * cfg.GPU_USE_COUNT, multi_gpu_sum=cfg.GPU_USE_COUNT)

Expand Down
27 changes: 14 additions & 13 deletions utils/data_aug.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,16 @@ def aug_data(tag, object_dir):
label = np.array([line for line in open(os.path.join(
object_dir, 'label_2', tag + '.txt'), 'r').readlines()]) # (N')
cls = np.array([line.split()[0] for line in label]) # (N')
gt_box3d = label_to_gt_box3d(np.array(label)[np.newaxis, :], cls='', coordinate='camera')[
P, Tr, R = load_calib( os.path.join( cfg.CALIB_DIR, tag + '.txt' )
gt_box3d = label_to_gt_box3d(np.array(label)[np.newaxis, :], cls='', coordinate='camera', T_VELO_2_CAM=Tr, R_RECT_0=R)[
0] # (N', 7) x, y, z, h, w, l, r

choice = np.random.randint(0, 10)
if choice >= 7:
# disable this augmention here. current implementation will decrease the performances
lidar_center_gt_box3d = camera_to_lidar_box(gt_box3d)
lidar_center_gt_box3d = camera_to_lidar_box(gt_box3d,T_VELO_2_CAM=Tr, R_RECT_0=R)
lidar_corner_gt_box3d = center_to_corner_box3d(
lidar_center_gt_box3d, coordinate='lidar')
lidar_center_gt_box3d, coordinate='lidar',T_VELO_2_CAM=Tr, R_RECT_0=R)
for idx in range(len(lidar_corner_gt_box3d)):
# TODO: precisely gather the point
is_collision = True
Expand All @@ -50,14 +51,14 @@ def aug_data(tag, object_dir):
t_z = np.random.normal()
# check collision
tmp = box_transform(
lidar_center_gt_box3d[[idx]], t_x, t_y, t_z, t_rz, 'lidar')
lidar_center_gt_box3d[[idx]], t_x, t_y, t_z, t_rz, 'lidar',T_VELO_2_CAM=Tr, R_RECT_0=R)
is_collision = False
for idy in range(idx):
x1, y1, w1, l1, r1 = tmp[0][[0, 1, 4, 5, 6]]
x2, y2, w2, l2, r2 = lidar_center_gt_box3d[idy][[
0, 1, 4, 5, 6]]
iou = cal_iou2d(np.array([x1, y1, w1, l1, r1], dtype=np.float32),
np.array([x2, y2, w2, l2, r2], dtype=np.float32))
np.array([x2, y2, w2, l2, r2], dtype=np.float32),T_VELO_2_CAM=Tr, R_RECT_0=R)
if iou > 0:
is_collision = True
_count += 1
Expand All @@ -81,29 +82,29 @@ def aug_data(tag, object_dir):
lidar[bound_box, 0:3] = point_transform(
lidar[bound_box, 0:3], t_x, t_y, t_z, rz=t_rz)
lidar_center_gt_box3d[idx] = box_transform(
lidar_center_gt_box3d[[idx]], t_x, t_y, t_z, t_rz, 'lidar')
lidar_center_gt_box3d[[idx]], t_x, t_y, t_z, t_rz, 'lidar',T_VELO_2_CAM=Tr, R_RECT_0=R)

gt_box3d = lidar_to_camera_box(lidar_center_gt_box3d)
gt_box3d = lidar_to_camera_box(lidar_center_gt_box3d,T_VELO_2_CAM=Tr, R_RECT_0=R)
newtag = 'aug_{}_1_{}'.format(
tag, np.random.randint(1, 1024))
elif choice < 7 and choice >= 4:
# global rotation
angle = np.random.uniform(-np.pi / 4, np.pi / 4)
lidar[:, 0:3] = point_transform(lidar[:, 0:3], 0, 0, 0, rz=angle)
lidar_center_gt_box3d = camera_to_lidar_box(gt_box3d)
lidar_center_gt_box3d = box_transform(lidar_center_gt_box3d, 0, 0, 0, r=angle, coordinate='lidar')
gt_box3d = lidar_to_camera_box(lidar_center_gt_box3d)
lidar_center_gt_box3d = camera_to_lidar_box(gt_box3d,T_VELO_2_CAM=Tr, R_RECT_0=R)
lidar_center_gt_box3d = box_transform(lidar_center_gt_box3d, 0, 0, 0, r=angle, coordinate='lidar',T_VELO_2_CAM=Tr, R_RECT_0=R)
gt_box3d = lidar_to_camera_box(lidar_center_gt_box3d,T_VELO_2_CAM=Tr, R_RECT_0=R)
newtag = 'aug_{}_2_{:.4f}'.format(tag, angle).replace('.', '_')
else:
# global scaling
factor = np.random.uniform(0.95, 1.05)
lidar[:, 0:3] = lidar[:, 0:3] * factor
lidar_center_gt_box3d = camera_to_lidar_box(gt_box3d)
lidar_center_gt_box3d = camera_to_lidar_box(gt_box3d,T_VELO_2_CAM=Tr, R_RECT_0=R)
lidar_center_gt_box3d[:, 0:6] = lidar_center_gt_box3d[:, 0:6] * factor
gt_box3d = lidar_to_camera_box(lidar_center_gt_box3d)
gt_box3d = lidar_to_camera_box(lidar_center_gt_box3d,T_VELO_2_CAM=Tr, R_RECT_0=R)
newtag = 'aug_{}_3_{:.4f}'.format(tag, factor).replace('.', '_')

label = box3d_to_label(gt_box3d[np.newaxis, ...], cls[np.newaxis, ...], coordinate='camera')[0] # (N')
label = box3d_to_label(gt_box3d[np.newaxis, ...], cls[np.newaxis, ...], coordinate='camera',P2=P,T_VELO_2_CAM=Tr, R_RECT_0=R)[0] # (N')
voxel_dict = process_pointcloud(lidar)
return newtag, rgb, lidar, voxel_dict, label

Expand Down
30 changes: 23 additions & 7 deletions utils/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
from config import cfg
from utils.box_overlaps import *

# Global variable: Allow the use of fixed P Tr matrices
ALLOW_FIXED_MATRICES = False

#-- util function to load calib matrices
CAM = 2
Expand Down Expand Up @@ -69,9 +71,13 @@ def angle_in_limit(angle):

def camera_to_lidar(x, y, z, T_VELO_2_CAM=None, R_RECT_0=None):
if type(T_VELO_2_CAM) == type(None):
if ALLOW_FIXED_MATRICES == False:
raise ValueError('Not allowed to use fixed matrix')
T_VELO_2_CAM = np.array(cfg.MATRIX_T_VELO_2_CAM)

if type(R_RECT_0) == type(None):
if ALLOW_FIXED_MATRICES == False:
raise ValueError('Not allowed to use fixed matrix')
R_RECT_0 = np.array(cfg.MATRIX_R_RECT_0)

p = np.array([x, y, z, 1])
Expand Down Expand Up @@ -101,9 +107,13 @@ def camera_to_lidar_point(points, T_VELO_2_CAM=None, R_RECT_0=None):
points = np.hstack([points, np.ones((N, 1))]).T # (N,4) -> (4,N)

if type(T_VELO_2_CAM) == type(None):
if ALLOW_FIXED_MATRICES == False:
raise ValueError('Not allowed to use fixed matrix')
T_VELO_2_CAM = np.array(cfg.MATRIX_T_VELO_2_CAM)

if type(R_RECT_0) == type(None):
if ALLOW_FIXED_MATRICES == False:
raise ValueError('Not allowed to use fixed matrix')
R_RECT_0 = np.array(cfg.MATRIX_R_RECT_0)

points = np.matmul(np.linalg.inv(R_RECT_0), points)
Expand All @@ -119,9 +129,13 @@ def lidar_to_camera_point(points, T_VELO_2_CAM=None, R_RECT_0=None):


if type(T_VELO_2_CAM) == type(None):
if ALLOW_FIXED_MATRICES == False:
raise ValueError('Not allowed to use fixed matrix')
T_VELO_2_CAM = np.array(cfg.MATRIX_T_VELO_2_CAM)

if type(R_RECT_0) == type(None):
if ALLOW_FIXED_MATRICES == False:
raise ValueError('Not allowed to use fixed matrix')
R_RECT_0 = np.array(cfg.MATRIX_R_RECT_0)

points = np.matmul(T_VELO_2_CAM, points)
Expand Down Expand Up @@ -164,7 +178,6 @@ def center_to_corner_box2d(boxes_center, coordinate='lidar', T_VELO_2_CAM=None,

return boxes3d_corner[:, 0:4, 0:2]


def center_to_corner_box3d(boxes_center, coordinate='lidar', T_VELO_2_CAM=None, R_RECT_0=None):
# (N, 7) -> (N, 8, 3)
N = boxes_center.shape[0]
Expand Down Expand Up @@ -329,6 +342,9 @@ def lidar_box3d_to_camera_box(boxes3d, cal_projection=False, P2 = None, T_VELO_2

lidar_boxes3d_corner = center_to_corner_box3d(boxes3d, coordinate='lidar', T_VELO_2_CAM=T_VELO_2_CAM, R_RECT_0=R_RECT_0)
if type(P2) == type(None):
if ALLOW_FIXED_MATRICES == False:
raise ValueError('Not allowed to use fixed matrix')

P2 = np.array(cfg.MATRIX_P2)

for n in range(num):
Expand Down Expand Up @@ -479,7 +495,7 @@ def label_to_gt_box3d(labels, cls='Car', coordinate='camera', T_VELO_2_CAM=None,
else: # all
acc_cls = []

for label in labels:
for counter,label in enumerate(labels):
boxes3d_a_label = []
for line in label:
ret = line.split()
Expand All @@ -488,7 +504,7 @@ def label_to_gt_box3d(labels, cls='Car', coordinate='camera', T_VELO_2_CAM=None,
box3d = np.array([x, y, z, h, w, l, r])
boxes3d_a_label.append(box3d)
if coordinate == 'lidar':
boxes3d_a_label = camera_to_lidar_box(np.array(boxes3d_a_label), T_VELO_2_CAM, R_RECT_0)
boxes3d_a_label = camera_to_lidar_box(np.array(boxes3d_a_label), T_VELO_2_CAM[counter], R_RECT_0[counter])

boxes3d.append(np.array(boxes3d_a_label).reshape(-1, 7))
return boxes3d
Expand Down Expand Up @@ -567,7 +583,7 @@ def cal_anchors():
return anchors


def cal_rpn_target(labels, feature_map_shape, anchors, cls='Car', coordinate='lidar'):
def cal_rpn_target(labels, feature_map_shape, anchors, cls='Car', coordinate='lidar', T_VELO_2_CAM=None, R_RECT_0=None):
# Input:
# labels: (N, N')
# feature_map_shape: (w, l)
Expand All @@ -578,7 +594,7 @@ def cal_rpn_target(labels, feature_map_shape, anchors, cls='Car', coordinate='li
# targets (N, w, l, 14)
# attention: cal IoU on birdview
batch_size = labels.shape[0]
batch_gt_boxes3d = label_to_gt_box3d(labels, cls=cls, coordinate=coordinate)
batch_gt_boxes3d = label_to_gt_box3d(labels, cls=cls, coordinate=coordinate,T_VELO_2_CAM=T_VELO_2_CAM, R_RECT_0=R_RECT_0)
# defined in eq(1) in 2.2
anchors_reshaped = anchors.reshape(-1, 7)
anchors_d = np.sqrt(anchors_reshaped[:, 4]**2 + anchors_reshaped[:, 5]**2)
Expand Down Expand Up @@ -726,7 +742,7 @@ def point_transform(points, tx, ty, tz, rx=0, ry=0, rz=0):
return points[:, 0:3]


def box_transform(boxes, tx, ty, tz, r=0, coordinate='lidar'):
def box_transform(boxes, tx, ty, tz, r=0, coordinate='lidar', T_VELO_2_CAM=None, R_RECT_0=None):
# Input:
# boxes: (N, 7) x y z h w l rz/y
# Output:
Expand All @@ -741,7 +757,7 @@ def box_transform(boxes, tx, ty, tz, r=0, coordinate='lidar'):
boxes_corner[idx] = point_transform(
boxes_corner[idx], tx, ty, tz, ry=r)

return corner_to_center_box3d(boxes_corner, coordinate=coordinate)
return corner_to_center_box3d(boxes_corner, coordinate=coordinate, T_VELO_2_CAM=T_VELO_2_CAM, R_RECT_0=R_RECT_0)


def cal_iou2d(box1, box2, T_VELO_2_CAM=None, R_RECT_0=None):
Expand Down