process_pascal.py

import argparse
import os
import copy

import numpy as np
import json
import torch
from PIL import Image, ImageDraw, ImageFont

# Grounding DINO
import GroundingDINO.groundingdino.datasets.transforms as T
from GroundingDINO.groundingdino.models import build_model
from GroundingDINO.groundingdino.util import box_ops
from GroundingDINO.groundingdino.util.slconfig import SLConfig
from GroundingDINO.groundingdino.util.utils import clean_state_dict, get_phrases_from_posmap, get_phrases_from_posmap_2

# segment anything
from segment_anything import build_sam, SamPredictor 
import cv2
import numpy as np
import matplotlib.pyplot as plt
import PIL.Image as Image


palette = [0, 0, 0, 128, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 128, 128, 0, 128, 0, 128, 128, 128, 128, 128,
           64, 0, 0, 192, 0, 0, 64, 128, 0, 192, 128, 0, 64, 0, 128, 192, 0, 128, 64, 128, 128, 192, 128, 128,
           0, 64, 0, 128, 64, 0, 0, 192, 0, 128, 192, 0, 0, 64, 128, 128, 64, 128, 0, 192, 128, 128, 192, 128,
           64, 64, 0, 192, 64, 0, 64, 192, 0, 192, 192, 0]

classes = ['aeroplane','bicycle','bird', 'boat','bottle','bus','car','cat','chair',
           'cow','diningtable','dog','horse','motorbike','person','pottedplant','sheep',
           'sofa','train','tvmonitor']

class_dict = dict(zip(classes, list(range(1, 21))))

# print(class_dict)

class_tree = {}
# add some propmt here:
# class_tree['person'] = ['']
class_tree['horse'] = ['halter', 'saddle']
class_tree['diningtable'] = ['bowl','plate','food','fruit','glass', 'dishes']
class_tree['tvmonitor'] = ['tv', 'monitor']
# class_tree['sofa'] = ['couch']
# class_tree['bottle'] = ['wine bottle', 'water bottle', 'canister']
# class_tree['pottedplant'] = ['couch']


for class_name, class_idx in list(class_dict.items()):
    if class_name in class_tree:
        sub_class_list = class_tree[class_name]
        for sub_class in sub_class_list:
            class_dict[sub_class] = class_idx

print(class_dict)
print(class_tree)



def load_image(image_path):
    # load image
    image_pil = Image.open(image_path).convert("RGB")  # load image

    transform = T.Compose(
        [
            T.RandomResize([800], max_size=1333),
            T.ToTensor(),
            T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
        ]
    )
    image, _ = transform(image_pil, None)  # 3, h, w
    return image_pil, image

def load_model(model_config_path, model_checkpoint_path, device):
    args = SLConfig.fromfile(model_config_path)
    args.device = device
    model = build_model(args)
    checkpoint = torch.load(model_checkpoint_path, map_location="cpu")
    load_res = model.load_state_dict(clean_state_dict(checkpoint["model"]), strict=False)
    # print(load_res)
    _ = model.eval()
    return model

def get_grounding_output(model, image, caption, box_threshold, text_threshold, with_logits=True, device="cpu"):
    caption = caption.lower()
    caption = caption.strip()
    if not caption.endswith("."):
        caption = caption + "."
    model = model.to(device)
    image = image.to(device)

    with torch.no_grad():
        outputs = model(image[None], captions=[caption])
    logits = outputs["pred_logits"].cpu().sigmoid()[0]  # (nq, 256)
    boxes = outputs["pred_boxes"].cpu()[0]  # (nq, 4)
    logits.shape[0]

    # filter output
    logits_filt = logits.clone()
    boxes_filt = boxes.clone()
    filt_mask = logits_filt.max(dim=1)[0] > box_threshold
    logits_filt = logits_filt[filt_mask]  # num_filt, 256
    boxes_filt = boxes_filt[filt_mask]  # num_filt, 4
    logits_filt.shape[0]

    # get phrase
    tokenlizer = model.tokenizer
    tokenized = tokenlizer(caption)
    
    # breakpoint()
    # build pred
    pred_phrases = []
    for logit, box in zip(logits_filt, boxes_filt):

        pred_phrase = get_phrases_from_posmap_2(logit > text_threshold, logit, tokenized, tokenlizer)

        if with_logits:
            pred_phrases.append(pred_phrase + f"({str(logit.max().item())[:4]})")
        else:
            pred_phrases.append(pred_phrase)

    return boxes_filt, pred_phrases

def show_mask(mask, ax, random_color=False):
    if random_color:
        color = np.concatenate([np.random.random(3), np.array([0.6])], axis=0)
    else:
        color = np.array([30/255, 144/255, 255/255, 0.6])
    h, w = mask.shape[-2:]
    mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)
    ax.imshow(mask_image)


def show_box(box, ax, label):
    x0, y0 = box[0], box[1]
    w, h = box[2] - box[0], box[3] - box[1]
    ax.add_patch(plt.Rectangle((x0, y0), w, h, edgecolor='green', facecolor=(0,0,0,0), lw=2)) 
    ax.text(x0, y0, label)


def save_mask_data(output_dir, mask_list, box_list, label_list, name, H, W):
    value = 0  # 0 for background

    mask_img = torch.zeros(H, W)
    for idx, mask in enumerate(mask_list):
        class_name, logit = label_list[idx].split('(')
        if class_name in class_dict:
            class_idx = class_dict[class_name]
            mask_img[mask.cpu().numpy()[0] == True] = class_idx 
    

    out = mask_img.numpy().astype(np.uint8)
    out = Image.fromarray(out, mode='P')
    out.putpalette(palette)
    out_name = os.path.join(output_dir, '{}.png'.format(name))
    out.save(out_name)


    json_data = [{
        'value': value,
        'label': 'background'
    }]

    for label, box in zip(label_list, box_list):
        value += 1
        name, logit = label.split('(')
        logit = logit[:-1] # the last is ')'
        json_data.append({
            'value': value,
            'label': name,
            'logit': float(logit),
            'box': box.numpy().tolist(),
        })
    with open(os.path.join(output_dir, 'mask.json'), 'w') as f:
        json.dump(json_data, f)
    

if __name__ == "__main__":

    parser = argparse.ArgumentParser("Grounded-Segment-Anything Demo", add_help=True)
    parser.add_argument("--config", type=str, required=True, help="path to config file")
    parser.add_argument(
        "--grounded_checkpoint", type=str, required=True, help="path to checkpoint file"
    )
    parser.add_argument(
        "--sam_checkpoint", type=str, required=True, help="path to checkpoint file"
    )
    parser.add_argument("--input_image", type=str, required=True, help="path to image file")
    parser.add_argument("--img_list", type=str, required=True, default='metadata/pascal/train_aug(id).txt')
    parser.add_argument("--im_path", default="/home/notebook/data/personal/S9050086/VOCdevkit/VOC2012/JPEGImages", type=str)


    parser.add_argument(
        "--output_dir", "-o", type=str, default="outputs", required=True, help="output directory"
    )   

    parser.add_argument("--box_threshold", type=float, default=0.3, help="box threshold")
    parser.add_argument("--text_threshold", type=float, default=0.25, help="text threshold")

    parser.add_argument("--device", type=str, default="cpu", help="running on cpu only!, default=False")
    args = parser.parse_args()


    with open(args.img_list) as f:
        img_list = []
        for line in f:
            img_list.append(line[:-1])
    
    cls_labels_dict = np.load('metadata/cls_labels.npy',allow_pickle=True).item()
    
    # cfg
    config_file = args.config  # change the path of the model config file
    grounded_checkpoint = args.grounded_checkpoint  # change the path of the model
    sam_checkpoint = args.sam_checkpoint
    output_dir = args.output_dir
    box_threshold = args.box_threshold
    text_threshold = args.text_threshold
    device = args.device

    # load model
    model = load_model(config_file, grounded_checkpoint, device=device)

    # make dir
    os.makedirs(output_dir, exist_ok=True)

    # initialize SAM
    predictor = SamPredictor(build_sam(checkpoint=sam_checkpoint).to(device))

    for index, name in enumerate(img_list):
        # load image
        image_path = os.path.join(args.im_path, '/{}.jpg'.format(name))
        image_pil, image = load_image(image_path)

        # get text prompt
        class_label = cls_labels_dict[name]
        text_prompt_list = []
        for i in range(20):
            if class_label[i] > 1e-5:
                text_prompt_list.append(classes[i])
                if classes[i] in list(class_tree.keys()):
                    sub_class_list = class_tree[classes[i]]
                    for sub_class in sub_class_list: 
                        text_prompt_list.append(sub_class)
        
        # breakpoint()

        if 'pottedplant' not in text_prompt_list:
            continue

        if len(text_prompt_list) == 1:
            text_prompt = text_prompt_list[0]
        else:
            text_prompt = '.'.join(text_prompt_list)
        
        print(index, name,  '--', text_prompt)

        # visualize raw image
        # image_pil.save(os.path.join(output_dir, "{}.jpg".format(name)))
    
        # run grounding dino model
        boxes_filt, pred_phrases = get_grounding_output(
            model, image, text_prompt, box_threshold, text_threshold, with_logits=True, device=device
        )

        image = cv2.imread(image_path)
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        predictor.set_image(image)

        size = image_pil.size
        H, W = size[1], size[0]
        for i in range(boxes_filt.size(0)):
            boxes_filt[i] = boxes_filt[i] * torch.Tensor([W, H, W, H])
            boxes_filt[i][:2] -= boxes_filt[i][2:] / 2
            boxes_filt[i][2:] += boxes_filt[i][:2]

        boxes_filt = boxes_filt.cpu()
        transformed_boxes = predictor.transform.apply_boxes_torch(boxes_filt, image.shape[:2]).to(device)

        try:
            masks, _, _ = predictor.predict_torch(
                point_coords = None,
                point_labels = None,
                boxes = transformed_boxes.to(device),
                multimask_output = False,
            )
        
        except: # in case nothing detected
            masks = []
        
        # print(pred_phrases)

        # draw output image
        plt.figure(figsize=(10, 10))
        plt.imshow(image)

        for mask in masks:
            show_mask(mask.cpu().numpy(), plt.gca(), random_color=True)
        for box, label in zip(boxes_filt, pred_phrases):
            show_box(box.numpy(), plt.gca(), label)

        plt.axis('off')
        plt.savefig(
            os.path.join(output_dir, "{}_sam.jpg".format(name)), 
            bbox_inches="tight", dpi=300, pad_inches=0.0
        )

        save_mask_data(output_dir, masks, boxes_filt, pred_phrases, name, H, W)
        plt.close()