From 5a6c5725edefe72ecbd028b0a5ea57edae1e35e8 Mon Sep 17 00:00:00 2001 From: Victor SANH Date: Tue, 26 Sep 2023 14:22:23 +0200 Subject: [PATCH 1/3] Fix position ids logic instantiation of idefics vision part --- .../models/custom_modeling/idefics_vision.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/idefics_vision.py b/server/text_generation_server/models/custom_modeling/idefics_vision.py index d933d7c12c0..60e9bec9388 100644 --- a/server/text_generation_server/models/custom_modeling/idefics_vision.py +++ b/server/text_generation_server/models/custom_modeling/idefics_vision.py @@ -88,12 +88,10 @@ def __init__(self, prefix, config, weights): self.num_patches = (self.image_size // self.patch_size) ** 2 self.num_positions = self.num_patches + 1 - # self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim) self.position_embedding = TensorParallelEmbedding( prefix="model.vision_model.embeddings.position_embedding", weights=weights ) - # self.register_buffer("position_ids", torch.arange(self.num_positions).expand((1, -1)), persistent=False) - self.position_ids = weights.get_tensor(f"{prefix}.position_ids") + self.register_buffer("position_ids", torch.arange(self.num_positions).expand((1, -1)), persistent=False) def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: batch_size = pixel_values.shape[0] From 1053e5d09abfcb476f0491ff8a742a55b0f95063 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 26 Sep 2023 14:36:08 +0200 Subject: [PATCH 2/3] Apply suggestions from code review --- .../models/custom_modeling/idefics_vision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/models/custom_modeling/idefics_vision.py b/server/text_generation_server/models/custom_modeling/idefics_vision.py index 60e9bec9388..0418281231e 100644 --- a/server/text_generation_server/models/custom_modeling/idefics_vision.py +++ b/server/text_generation_server/models/custom_modeling/idefics_vision.py @@ -91,7 +91,7 @@ def __init__(self, prefix, config, weights): self.position_embedding = TensorParallelEmbedding( prefix="model.vision_model.embeddings.position_embedding", weights=weights ) - self.register_buffer("position_ids", torch.arange(self.num_positions).expand((1, -1)), persistent=False) + self.position_ids = torch.arange(self.num_positions).expand((1, -1)) def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: batch_size = pixel_values.shape[0] From bf2b92217f97efa6f1cade29b685e803221b3a67 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 26 Sep 2023 15:07:38 +0200 Subject: [PATCH 3/3] Apply suggestions from code review --- .../models/custom_modeling/idefics_vision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/models/custom_modeling/idefics_vision.py b/server/text_generation_server/models/custom_modeling/idefics_vision.py index 0418281231e..30f07095e71 100644 --- a/server/text_generation_server/models/custom_modeling/idefics_vision.py +++ b/server/text_generation_server/models/custom_modeling/idefics_vision.py @@ -91,7 +91,7 @@ def __init__(self, prefix, config, weights): self.position_embedding = TensorParallelEmbedding( prefix="model.vision_model.embeddings.position_embedding", weights=weights ) - self.position_ids = torch.arange(self.num_positions).expand((1, -1)) + self.position_ids = torch.arange(self.num_positions).expand((1, -1)).to(device=weights.device) def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: batch_size = pixel_values.shape[0]