BLIP: this is correct now (#35081)

this is correct now
huggingface · Dec 5, 2024 · e682c17 · e682c17
1 parent 50189e3
commit e682c17
Show file tree

Hide file tree

Showing 4 changed files with 4 additions and 4 deletions.
diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py
@@ -2311,7 +2311,7 @@ def generate(
         if input_ids is None:
             start_tokens = [self.config.text_config.bos_token_id]
             if getattr(self.config, "image_token_index", None) is not None:
-                start_tokens += [self.config.image_token_index] * self.config.num_query_tokens
+                start_tokens = [self.config.image_token_index] * self.config.num_query_tokens + start_tokens
             input_ids = torch.tensor([start_tokens], dtype=torch.long, device=image_embeds.device)
             input_ids = input_ids.repeat(batch_size, 1)
 

diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py
@@ -1593,7 +1593,7 @@ def generate(
         if input_ids is None:
             start_tokens = [self.config.text_config.bos_token_id]
             if getattr(self.config, "image_token_index", None) is not None:
-                start_tokens += [self.config.image_token_index] * self.config.num_query_tokens
+                start_tokens = [self.config.image_token_index] * self.config.num_query_tokens + start_tokens
             input_ids = torch.tensor([start_tokens], dtype=torch.long, device=image_embeds.device)
             input_ids = input_ids.repeat(batch_size, 1)
 

diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py
@@ -1628,7 +1628,7 @@ def generate(
         if input_ids is None:
             start_tokens = [self.config.text_config.bos_token_id]
             if getattr(self.config, "video_token_index", None) is not None:
-                start_tokens += [self.config.video_token_index] * self.config.num_query_tokens * 4
+                start_tokens = [self.config.video_token_index] * self.config.num_query_tokens * 4 + start_tokens
             input_ids = torch.tensor([start_tokens], dtype=torch.long, device=image_embeds.device)
             input_ids = input_ids.repeat(batch_size, 1)
 

diff --git a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py
@@ -441,7 +441,7 @@ def generate(
         if input_ids is None:
             start_tokens = [self.config.text_config.bos_token_id]
             if getattr(self.config, "video_token_index", None) is not None:
-                start_tokens += [self.config.video_token_index] * self.config.num_query_tokens * 4
+                start_tokens = [self.config.video_token_index] * self.config.num_query_tokens * 4 + start_tokens
             input_ids = torch.tensor([start_tokens], dtype=torch.long, device=image_embeds.device)
             input_ids = input_ids.repeat(batch_size, 1)