feat: adding mplugdocowl

docs/source/en/_toctree.yml

@@ -778,6 +778,8 @@
         title: MatCha
       - local: model_doc/mgp-str
         title: MGP-STR
+      - local: model_doc/mplugdocowl
+        title: mPLUGDocOwl
       - local: model_doc/nougat
         title: Nougat
       - local: model_doc/oneformer

docs/source/en/model_doc/mplugdocowl.md

@@ -0,0 +1,47 @@
+<!--Copyright 2024 The HuggingFace Team. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+⚠️ Note that this file is in Markdown but contain specific syntax for our doc-builder (similar to MDX) that may not be
+rendered properly in your Markdown viewer.
+
+-->
+
+# mPLUGDocOwl
+
+## Overview
+
+The mPLUGDocOwl model was proposed in [<INSERT PAPER NAME HERE>](<INSERT PAPER LINK HERE>) by <INSERT AUTHORS HERE>.
+<INSERT SHORT SUMMARY HERE>
+
+The abstract from the paper is the following:
+
+*<INSERT PAPER ABSTRACT HERE>*
+
+Tips:
+
+<INSERT TIPS ABOUT MODEL HERE>
+
+This model was contributed by [INSERT YOUR HF USERNAME HERE](https://huggingface.co/<INSERT YOUR HF USERNAME HERE>).
+The original code can be found [here](<INSERT LINK TO GITHUB REPO HERE>).
+
+
+## MPLUGDocOwlConfig
+
+[[autodoc]] MPLUGDocOwlConfig
+
+## MPLUGDocOwlProcessor
+
+[[autodoc]] MPLUGDocOwlProcessor
+
+## MPLUGDocOwlForConditionalGeneration
+
+[[autodoc]] MPLUGDocOwlForConditionalGeneration
+    - forward

src/transformers/__init__.py

@@ -486,6 +486,10 @@
         "LlavaConfig",
         "LlavaProcessor",
     ],
+    "models.mplugdocowl": [
+        "MPLUGDocOwlConfig",
+        "MPLUGDocOwlProcessor",
+    ],
     "models.llava_next": [
         "LlavaNextConfig",
         "LlavaNextProcessor",
@@ -1144,6 +1148,7 @@
     _import_structure["models.mobilenet_v1"].extend(["MobileNetV1FeatureExtractor", "MobileNetV1ImageProcessor"])
     _import_structure["models.mobilenet_v2"].extend(["MobileNetV2FeatureExtractor", "MobileNetV2ImageProcessor"])
     _import_structure["models.mobilevit"].extend(["MobileViTFeatureExtractor", "MobileViTImageProcessor"])
+    _import_structure["models.mplugdocowl"].extend(["MPLUGDocOwlImageProcessor"])
     _import_structure["models.nougat"].append("NougatImageProcessor")
     _import_structure["models.oneformer"].extend(["OneFormerImageProcessor"])
     _import_structure["models.owlv2"].append("Owlv2ImageProcessor")
@@ -2297,6 +2302,12 @@
             "LlavaPreTrainedModel",
         ]
     )
+    _import_structure["models.mplugdocowl"].extend(
+        [
+            "MPLUGDocOwlForConditionalGeneration",
+            "MPLUGDocOwlPreTrainedModel",
+        ]
+    )
     _import_structure["models.llava_next"].extend(
         [
             "LlavaNextForConditionalGeneration",
@@ -5037,6 +5048,10 @@
         LlavaConfig,
         LlavaProcessor,
     )
+    from .models.mplugdocowl import (
+        MPLUGDocOwlConfig,
+        MPLUGDocOwlProcessor,
+    )
     from .models.llava_next import (
         LlavaNextConfig,
         LlavaNextProcessor,
@@ -6692,6 +6707,10 @@
             LlavaForConditionalGeneration,
             LlavaPreTrainedModel,
         )
+        from .models.mplugdocowl import (
+            MPLUGDocOwlForConditionalGeneration,
+            MPLUGDocOwlPreTrainedModel,
+        )
         from .models.llava_next import (
             LlavaNextForConditionalGeneration,
             LlavaNextPreTrainedModel,

src/transformers/models/__init__.py

@@ -129,6 +129,7 @@
     lilt,
     llama,
     llava,
+    mplugdocowl,
     llava_next,
     longformer,
     longt5,

src/transformers/models/auto/configuration_auto.py

@@ -139,6 +139,7 @@
         ("lilt", "LiltConfig"),
         ("llama", "LlamaConfig"),
         ("llava", "LlavaConfig"),
+        ("mplugdocowl", "MPLUGDocOwlConfig"),
         ("llava_next", "LlavaNextConfig"),
         ("longformer", "LongformerConfig"),
         ("longt5", "LongT5Config"),
@@ -416,6 +417,7 @@
         ("llama2", "Llama2"),
         ("llama3", "Llama3"),
         ("llava", "LLaVa"),
+        ("mplugdocowl", "mPLUGDocOwl"),
         ("llava_next", "LLaVA-NeXT"),
         ("longformer", "Longformer"),
         ("longt5", "LongT5"),

src/transformers/models/auto/image_processing_auto.py

@@ -79,6 +79,7 @@
         ("layoutlmv3", "LayoutLMv3ImageProcessor"),
         ("levit", "LevitImageProcessor"),
         ("llava", "CLIPImageProcessor"),
+        ("mplugdocowl", "MPLUGDocOwlImageProcessor"),
         ("llava_next", "LlavaNextImageProcessor"),
         ("mask2former", "Mask2FormerImageProcessor"),
         ("maskformer", "MaskFormerImageProcessor"),

src/transformers/models/auto/modeling_auto.py

@@ -298,6 +298,7 @@
         ("idefics2", "Idefics2ForConditionalGeneration"),
         ("layoutlm", "LayoutLMForMaskedLM"),
         ("llava", "LlavaForConditionalGeneration"),
+        ("mplugdocowl", "MPLUGDocOwlForConditionalGeneration"),
         ("llava_next", "LlavaNextForConditionalGeneration"),
         ("longformer", "LongformerForMaskedLM"),
         ("luke", "LukeForMaskedLM"),
@@ -698,6 +699,7 @@
         ("instructblip", "InstructBlipForConditionalGeneration"),
         ("kosmos-2", "Kosmos2ForConditionalGeneration"),
         ("llava", "LlavaForConditionalGeneration"),
+        ("mplugdocowl", "MPLUGDocOwlForConditionalGeneration"),
         ("llava_next", "LlavaNextForConditionalGeneration"),
         ("paligemma", "PaliGemmaForConditionalGeneration"),
         ("pix2struct", "Pix2StructForConditionalGeneration"),

src/transformers/models/auto/processing_auto.py

@@ -67,6 +67,7 @@
         ("layoutlmv2", "LayoutLMv2Processor"),
         ("layoutlmv3", "LayoutLMv3Processor"),
         ("llava", "LlavaProcessor"),
+        ("mplugdocowl", "MPLUGDocOwlProcessor"),
         ("llava_next", "LlavaNextProcessor"),
         ("markuplm", "MarkupLMProcessor"),
         ("mctct", "MCTCTProcessor"),

src/transformers/models/auto/tokenization_auto.py

@@ -241,6 +241,7 @@
                 ),
             ),
             ("llava", ("LlamaTokenizer", "LlamaTokenizerFast" if is_tokenizers_available() else None)),
+            ("mplugdocowl", ("MPLUGDocOwlTokenizer", "MPLUGDocOwlTokenizerFast" if is_tokenizers_available() else None)),
             ("llava_next", ("LlamaTokenizer", "LlamaTokenizerFast" if is_tokenizers_available() else None)),
             ("longformer", ("LongformerTokenizer", "LongformerTokenizerFast" if is_tokenizers_available() else None)),
             (

src/transformers/models/llama/modeling_llama.py

@@ -50,7 +50,6 @@
 )
 from .configuration_llama import LlamaConfig
 
-
 if is_flash_attn_2_available():
     from flash_attn import flash_attn_func, flash_attn_varlen_func
     from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input  # noqa
@@ -685,8 +684,7 @@ def forward(
         attn_output = self.o_proj(attn_output)
 
         return attn_output, None, past_key_value
-
-
+
 LLAMA_ATTENTION_CLASSES = {
     "eager": LlamaAttention,
     "flash_attention_2": LlamaFlashAttention2,

src/transformers/models/mplugdocowl/__init__.py

@@ -0,0 +1,70 @@
+# Copyright 2024 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import TYPE_CHECKING
+
+from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available
+
+
+_import_structure = {
+    "configuration_mplugdocowl": ["MPLUGDocOwlConfig"],
+    "processing_mplugdocowl": ["MPLUGDocOwlProcessor"],
+}
+
+try:
+    if not is_vision_available():
+        raise OptionalDependencyNotAvailable()
+except OptionalDependencyNotAvailable:
+    pass
+else:
+    _import_structure["image_processing_mplugdocowl"] = ["MPLUGDocOwlImageProcessor"]
+
+try:
+    if not is_torch_available():
+        raise OptionalDependencyNotAvailable()
+except OptionalDependencyNotAvailable:
+    pass
+else:
+    _import_structure["modeling_mplugdocowl"] = [
+        "MPLUGDocOwlForConditionalGeneration",
+        "MPLUGDocOwlPreTrainedModel",
+    ]
+
+
+if TYPE_CHECKING:
+    from .configuration_mplugdocowl import MPLUGDocOwlConfig
+    from .processing_mplugdocowl import MPLUGDocOwlProcessor
+    try:
+        if not is_vision_available():
+            raise OptionalDependencyNotAvailable()
+    except OptionalDependencyNotAvailable:
+        pass
+    else:
+        from .image_processing_mplugdocowl import MPLUGDocOwlImageProcessor
+
+    try:
+        if not is_torch_available():
+            raise OptionalDependencyNotAvailable()
+    except OptionalDependencyNotAvailable:
+        pass
+    else:
+        from .modeling_mplugdocowl import (
+            MPLUGDocOwlForConditionalGeneration,
+            MPLUGDocOwlPreTrainedModel,
+        )
+
+
+else:
+    import sys
+
+    sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)