From 5ac17bda838547b9167a75ba1eb1b2e98b680cab Mon Sep 17 00:00:00 2001 From: Joshua Lochner Date: Mon, 19 Feb 2024 14:57:06 +0200 Subject: [PATCH] Add support for OWLv2 models (#579) --- README.md | 1 + docs/snippets/6_supported-models.snippet | 1 + scripts/convert.py | 8 +++++++- scripts/supported_models.py | 13 +++++++++++++ src/models.js | 8 ++++++++ src/processors.js | 3 +++ 6 files changed, 33 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c026b23bd..db1f6f561 100644 --- a/README.md +++ b/README.md @@ -324,6 +324,7 @@ You can refine your search by selecting the task you're interested in (e.g., [te 1. **[Nougat](https://huggingface.co/docs/transformers/model_doc/nougat)** (from Meta AI) released with the paper [Nougat: Neural Optical Understanding for Academic Documents](https://arxiv.org/abs/2308.13418) by Lukas Blecher, Guillem Cucurull, Thomas Scialom, Robert Stojnic. 1. **[OPT](https://huggingface.co/docs/transformers/master/model_doc/opt)** (from Meta AI) released with the paper [OPT: Open Pre-trained Transformer Language Models](https://arxiv.org/abs/2205.01068) by Susan Zhang, Stephen Roller, Naman Goyal, Mikel Artetxe, Moya Chen, Shuohui Chen et al. 1. **[OWL-ViT](https://huggingface.co/docs/transformers/model_doc/owlvit)** (from Google AI) released with the paper [Simple Open-Vocabulary Object Detection with Vision Transformers](https://arxiv.org/abs/2205.06230) by Matthias Minderer, Alexey Gritsenko, Austin Stone, Maxim Neumann, Dirk Weissenborn, Alexey Dosovitskiy, Aravindh Mahendran, Anurag Arnab, Mostafa Dehghani, Zhuoran Shen, Xiao Wang, Xiaohua Zhai, Thomas Kipf, and Neil Houlsby. +1. **[OWLv2](https://huggingface.co/docs/transformers/model_doc/owlv2)** (from Google AI) released with the paper [Scaling Open-Vocabulary Object Detection](https://arxiv.org/abs/2306.09683) by Matthias Minderer, Alexey Gritsenko, Neil Houlsby. 1. **[Phi](https://huggingface.co/docs/transformers/main/model_doc/phi)** (from Microsoft) released with the papers - [Textbooks Are All You Need](https://arxiv.org/abs/2306.11644) by Suriya Gunasekar, Yi Zhang, Jyoti Aneja, Caio César Teodoro Mendes, Allie Del Giorno, Sivakanth Gopi, Mojan Javaheripi, Piero Kauffmann, Gustavo de Rosa, Olli Saarikivi, Adil Salim, Shital Shah, Harkirat Singh Behl, Xin Wang, Sébastien Bubeck, Ronen Eldan, Adam Tauman Kalai, Yin Tat Lee and Yuanzhi Li, [Textbooks Are All You Need II: phi-1.5 technical report](https://arxiv.org/abs/2309.05463) by Yuanzhi Li, Sébastien Bubeck, Ronen Eldan, Allie Del Giorno, Suriya Gunasekar and Yin Tat Lee. 1. **[Qwen2](https://huggingface.co/docs/transformers/model_doc/qwen2)** (from the Qwen team, Alibaba Group) released with the paper [Qwen Technical Report](https://arxiv.org/abs/2309.16609) by Jinze Bai, Shuai Bai, Yunfei Chu, Zeyu Cui, Kai Dang, Xiaodong Deng, Yang Fan, Wenbin Ge, Yu Han, Fei Huang, Binyuan Hui, Luo Ji, Mei Li, Junyang Lin, Runji Lin, Dayiheng Liu, Gao Liu, Chengqiang Lu, Keming Lu, Jianxin Ma, Rui Men, Xingzhang Ren, Xuancheng Ren, Chuanqi Tan, Sinan Tan, Jianhong Tu, Peng Wang, Shijie Wang, Wei Wang, Shengguang Wu, Benfeng Xu, Jin Xu, An Yang, Hao Yang, Jian Yang, Shusheng Yang, Yang Yao, Bowen Yu, Hongyi Yuan, Zheng Yuan, Jianwei Zhang, Xingxuan Zhang, Yichang Zhang, Zhenru Zhang, Chang Zhou, Jingren Zhou, Xiaohuan Zhou and Tianhang Zhu. 1. **[ResNet](https://huggingface.co/docs/transformers/model_doc/resnet)** (from Microsoft Research) released with the paper [Deep Residual Learning for Image Recognition](https://arxiv.org/abs/1512.03385) by Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. diff --git a/docs/snippets/6_supported-models.snippet b/docs/snippets/6_supported-models.snippet index 4eb762aa1..ed2c1bdff 100644 --- a/docs/snippets/6_supported-models.snippet +++ b/docs/snippets/6_supported-models.snippet @@ -59,6 +59,7 @@ 1. **[Nougat](https://huggingface.co/docs/transformers/model_doc/nougat)** (from Meta AI) released with the paper [Nougat: Neural Optical Understanding for Academic Documents](https://arxiv.org/abs/2308.13418) by Lukas Blecher, Guillem Cucurull, Thomas Scialom, Robert Stojnic. 1. **[OPT](https://huggingface.co/docs/transformers/master/model_doc/opt)** (from Meta AI) released with the paper [OPT: Open Pre-trained Transformer Language Models](https://arxiv.org/abs/2205.01068) by Susan Zhang, Stephen Roller, Naman Goyal, Mikel Artetxe, Moya Chen, Shuohui Chen et al. 1. **[OWL-ViT](https://huggingface.co/docs/transformers/model_doc/owlvit)** (from Google AI) released with the paper [Simple Open-Vocabulary Object Detection with Vision Transformers](https://arxiv.org/abs/2205.06230) by Matthias Minderer, Alexey Gritsenko, Austin Stone, Maxim Neumann, Dirk Weissenborn, Alexey Dosovitskiy, Aravindh Mahendran, Anurag Arnab, Mostafa Dehghani, Zhuoran Shen, Xiao Wang, Xiaohua Zhai, Thomas Kipf, and Neil Houlsby. +1. **[OWLv2](https://huggingface.co/docs/transformers/model_doc/owlv2)** (from Google AI) released with the paper [Scaling Open-Vocabulary Object Detection](https://arxiv.org/abs/2306.09683) by Matthias Minderer, Alexey Gritsenko, Neil Houlsby. 1. **[Phi](https://huggingface.co/docs/transformers/main/model_doc/phi)** (from Microsoft) released with the papers - [Textbooks Are All You Need](https://arxiv.org/abs/2306.11644) by Suriya Gunasekar, Yi Zhang, Jyoti Aneja, Caio César Teodoro Mendes, Allie Del Giorno, Sivakanth Gopi, Mojan Javaheripi, Piero Kauffmann, Gustavo de Rosa, Olli Saarikivi, Adil Salim, Shital Shah, Harkirat Singh Behl, Xin Wang, Sébastien Bubeck, Ronen Eldan, Adam Tauman Kalai, Yin Tat Lee and Yuanzhi Li, [Textbooks Are All You Need II: phi-1.5 technical report](https://arxiv.org/abs/2309.05463) by Yuanzhi Li, Sébastien Bubeck, Ronen Eldan, Allie Del Giorno, Suriya Gunasekar and Yin Tat Lee. 1. **[Qwen2](https://huggingface.co/docs/transformers/model_doc/qwen2)** (from the Qwen team, Alibaba Group) released with the paper [Qwen Technical Report](https://arxiv.org/abs/2309.16609) by Jinze Bai, Shuai Bai, Yunfei Chu, Zeyu Cui, Kai Dang, Xiaodong Deng, Yang Fan, Wenbin Ge, Yu Han, Fei Huang, Binyuan Hui, Luo Ji, Mei Li, Junyang Lin, Runji Lin, Dayiheng Liu, Gao Liu, Chengqiang Lu, Keming Lu, Jianxin Ma, Rui Men, Xingzhang Ren, Xuancheng Ren, Chuanqi Tan, Sinan Tan, Jianhong Tu, Peng Wang, Shijie Wang, Wei Wang, Shengguang Wu, Benfeng Xu, Jin Xu, An Yang, Hao Yang, Jian Yang, Shusheng Yang, Yang Yao, Bowen Yu, Hongyi Yuan, Zheng Yuan, Jianwei Zhang, Xingxuan Zhang, Yichang Zhang, Zhenru Zhang, Chang Zhou, Jingren Zhou, Xiaohuan Zhou and Tianhang Zhu. 1. **[ResNet](https://huggingface.co/docs/transformers/model_doc/resnet)** (from Microsoft Research) released with the paper [Deep Residual Learning for Image Recognition](https://arxiv.org/abs/1512.03385) by Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. diff --git a/scripts/convert.py b/scripts/convert.py index aff30a8b0..87ac56f99 100644 --- a/scripts/convert.py +++ b/scripts/convert.py @@ -93,6 +93,12 @@ 'per_channel': False, 'reduce_range': False, }, + + # Encoder-only models + 'owlv2': { + 'per_channel': False, + 'reduce_range': False, + }, } MODELS_WITHOUT_TOKENIZERS = [ @@ -403,7 +409,7 @@ def main(): with open(os.path.join(output_model_folder, 'tokenizer.json'), 'w', encoding='utf-8') as fp: json.dump(tokenizer_json, fp, indent=4) - elif config.model_type == 'owlvit': + elif config.model_type in ('owlvit', 'owlv2'): # Override default batch size to 1, needed because non-maximum suppression is performed for exporting. # For more information, see https://github.com/huggingface/optimum/blob/e3b7efb1257c011db907ef40ab340e795cc5684c/optimum/exporters/onnx/model_configs.py#L1028-L1032 export_kwargs['batch_size'] = 1 diff --git a/scripts/supported_models.py b/scripts/supported_models.py index fd9ea99cb..99f3421ab 100644 --- a/scripts/supported_models.py +++ b/scripts/supported_models.py @@ -666,6 +666,19 @@ 'PygmalionAI/pygmalion-350m', ], }, + 'owlv2': { + # Object detection (Zero-shot object detection) + # NOTE: Exported with --batch_size 1 + 'zero-shot-object-detection': [ + 'google/owlv2-base-patch16', + 'google/owlv2-base-patch16-finetuned', + 'google/owlv2-base-patch16-ensemble', + # TODO: add + # 'google/owlv2-large-patch14', + # 'google/owlv2-large-patch14-finetuned', + # 'google/owlv2-large-patch14-ensemble', + ], + }, 'owlvit': { # Object detection (Zero-shot object detection) # NOTE: Exported with --batch_size 1 diff --git a/src/models.js b/src/models.js index c02f0041d..46c112a0b 100644 --- a/src/models.js +++ b/src/models.js @@ -3832,6 +3832,12 @@ export class OwlViTModel extends OwlViTPreTrainedModel { } export class OwlViTForObjectDetection extends OwlViTPreTrainedModel { } ////////////////////////////////////////////////// +////////////////////////////////////////////////// +export class Owlv2PreTrainedModel extends PreTrainedModel { } +export class Owlv2Model extends Owlv2PreTrainedModel { } +export class Owlv2ForObjectDetection extends Owlv2PreTrainedModel { } +////////////////////////////////////////////////// + ////////////////////////////////////////////////// // Beit Models export class BeitPreTrainedModel extends PreTrainedModel { } @@ -5250,6 +5256,7 @@ const MODEL_MAPPING_NAMES_ENCODER_ONLY = new Map([ ['vit', ['ViTModel', ViTModel]], ['mobilevit', ['MobileViTModel', MobileViTModel]], ['owlvit', ['OwlViTModel', OwlViTModel]], + ['owlv2', ['Owlv2Model', Owlv2Model]], ['beit', ['BeitModel', BeitModel]], ['deit', ['DeiTModel', DeiTModel]], ['convnext', ['ConvNextModel', ConvNextModel]], @@ -5445,6 +5452,7 @@ const MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES = new Map([ const MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES = new Map([ ['owlvit', ['OwlViTForObjectDetection', OwlViTForObjectDetection]], + ['owlv2', ['Owlv2ForObjectDetection', Owlv2ForObjectDetection]], ]); const MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES = new Map([ diff --git a/src/processors.js b/src/processors.js index 281fa09cb..463f82551 100644 --- a/src/processors.js +++ b/src/processors.js @@ -818,6 +818,8 @@ export class OwlViTFeatureExtractor extends ImageFeatureExtractor { return post_process_object_detection(...args); } } +export class Owlv2ImageProcessor extends OwlViTFeatureExtractor { } // NOTE extends OwlViTFeatureExtractor + export class DeiTFeatureExtractor extends ImageFeatureExtractor { } export class BeitFeatureExtractor extends ImageFeatureExtractor { } export class DonutFeatureExtractor extends ImageFeatureExtractor { @@ -2113,6 +2115,7 @@ export class AutoProcessor { ViTFeatureExtractor, MobileViTFeatureExtractor, OwlViTFeatureExtractor, + Owlv2ImageProcessor, CLIPFeatureExtractor, ChineseCLIPFeatureExtractor, SiglipImageProcessor,