Skip to content

Commit

Permalink
Idefics2: sync added image tokens with transformers
Browse files Browse the repository at this point in the history 
Before this change, the number of reserved image tokens was not the
same as the number of images. Fixes #2029.

While at it, also remove all the image token handling duplication
in `prepare_input`.
  • Loading branch information
@danieldk
danieldk committed Jun 20, 2024
1 parent cdbf802 commit 99be263
Show file tree
Hide file tree
Showing 10 changed files with 1,519 additions and 1,332 deletions.
2,578 changes: 1,369 additions & 1,209 deletions integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_load.json
View file

Large diffs are not rendered by default.

20 changes: 10 additions & 10 deletions integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_simple.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,61 +8,61 @@
"tokens": [
{
"id": 330,
"logprob": -0.13000488,
"logprob": -0.08660889,
"special": false,
"text": " A"
},
{
"id": 13088,
"logprob": -0.6713867,
"logprob": -0.7089844,
"special": false,
"text": " chicken"
},
{
"id": 349,
"logprob": -0.2980957,
"logprob": -0.32885742,
"special": false,
"text": " is"
},
{
"id": 6398,
"logprob": -0.060638428,
"logprob": -0.05126953,
"special": false,
"text": " sitting"
},
{
"id": 356,
"logprob": -0.27319336,
"logprob": -0.35229492,
"special": false,
"text": " on"
},
{
"id": 264,
"logprob": -0.140625,
"logprob": -0.12561035,
"special": false,
"text": " a"
},
{
"id": 17972,
"logprob": -0.040405273,
"logprob": -0.038085938,
"special": false,
"text": " pile"
},
{
"id": 302,
"logprob": -0.0002708435,
"logprob": -0.00018656254,
"special": false,
"text": " of"
},
{
"id": 2445,
"logprob": -0.095336914,
"logprob": -0.07293701,
"special": false,
"text": " money"
},
{
"id": 28723,
"logprob": -0.0068359375,
"logprob": -0.004852295,
"special": false,
"text": "."
}
Expand Down
38 changes: 19 additions & 19 deletions integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_two_images.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,115 +8,115 @@
"tokens": [
{
"id": 415,
"logprob": -0.04421997,
"logprob": -0.039886475,
"special": false,
"text": " The"
},
{
"id": 12072,
"logprob": -0.13500977,
"logprob": -0.1430664,
"special": false,
"text": " cow"
},
{
"id": 349,
"logprob": -0.06750488,
"logprob": -0.056488037,
"special": false,
"text": " is"
},
{
"id": 6328,
"logprob": -0.6352539,
"logprob": -0.6855469,
"special": false,
"text": " standing"
},
{
"id": 356,
"logprob": -0.16186523,
"logprob": -0.1685791,
"special": false,
"text": " on"
},
{
"id": 272,
"logprob": -0.5078125,
"logprob": -0.50097656,
"special": false,
"text": " the"
},
{
"id": 10305,
"logprob": -0.017913818,
"logprob": -0.017303467,
"special": false,
"text": " beach"
},
{
"id": 304,
"logprob": -1.5205078,
"logprob": -1.3564453,
"special": false,
"text": " and"
},
{
"id": 272,
"logprob": -0.029174805,
"logprob": -0.017868042,
"special": false,
"text": " the"
},
{
"id": 13088,
"logprob": -0.003479004,
"logprob": -0.0027103424,
"special": false,
"text": " chicken"
},
{
"id": 349,
"logprob": -0.0035095215,
"logprob": -0.003156662,
"special": false,
"text": " is"
},
{
"id": 6398,
"logprob": -0.3088379,
"logprob": -0.37304688,
"special": false,
"text": " sitting"
},
{
"id": 356,
"logprob": -0.027755737,
"logprob": -0.034576416,
"special": false,
"text": " on"
},
{
"id": 264,
"logprob": -0.31884766,
"logprob": -0.29418945,
"special": false,
"text": " a"
},
{
"id": 17972,
"logprob": -0.047943115,
"logprob": -0.042877197,
"special": false,
"text": " pile"
},
{
"id": 302,
"logprob": -0.0002925396,
"logprob": -0.00028443336,
"special": false,
"text": " of"
},
{
"id": 2445,
"logprob": -0.02935791,
"logprob": -0.023223877,
"special": false,
"text": " money"
},
{
"id": 28723,
"logprob": -0.031219482,
"logprob": -0.018157959,
"special": false,
"text": "."
},
{
"id": 32002,
"logprob": -0.00034475327,
"logprob": -0.00018393993,
"special": true,
"text": "<end_of_utterance>"
},
Expand Down
2 changes: 1 addition & 1 deletion router/src/config.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ pub struct Idefics2 {}

impl Idefics2 {
pub fn get_number_of_features(&self, _height: usize, _width: usize) -> usize {
320
64
}
}

Expand Down
19 changes: 19 additions & 0 deletions router/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,25 @@ impl HubTokenizerConfig {
}
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "processor_class")]
pub enum HubPreprocessorConfig {
Idefics2Processor(Idefics2Preprocessor),
}

impl HubPreprocessorConfig {
pub fn from_file<P: AsRef<std::path::Path>>(filename: P) -> Option<Self> {
let content = std::fs::read_to_string(filename).ok()?;
serde_json::from_str(&content).ok()
}
}

#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Idefics2Preprocessor {
#[serde(default)]
do_image_splitting: bool,
}

#[derive(Debug, Clone, Deserialize, Default)]
pub struct HubProcessorConfig {
pub chat_template: Option<ChatTemplateVersions>,
Expand Down
12 changes: 11 additions & 1 deletion router/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ use std::io::BufReader;
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::path::{Path, PathBuf};
use text_generation_router::config::Config;
use text_generation_router::{server, HubModelInfo, HubProcessorConfig, HubTokenizerConfig};
use text_generation_router::{
server, HubModelInfo, HubPreprocessorConfig, HubProcessorConfig, HubTokenizerConfig,
};
use thiserror::Error;
use tokenizers::Tokenizer;
use tower_http::cors::AllowOrigin;
Expand Down Expand Up @@ -209,13 +211,15 @@ async fn main() -> Result<(), RouterError> {
tokenizer_filename,
config_filename,
tokenizer_config_filename,
preprocessor_config_filename,
processor_config_filename,
model_info,
) = match api {
Type::None => (
Some(local_path.join("tokenizer.json")),
Some(local_path.join("config.json")),
Some(local_path.join("tokenizer_config.json")),
Some(local_path.join("preprocessor_config.json")),
Some(local_path.join("processor_config.json")),
None,
),
Expand All @@ -232,6 +236,7 @@ async fn main() -> Result<(), RouterError> {
};
let config_filename = api_repo.get("config.json").await.ok();
let tokenizer_config_filename = api_repo.get("tokenizer_config.json").await.ok();
let preprocessor_config_filename = api_repo.get("preprocessor_config.json").await.ok();
let processor_config_filename = api_repo.get("processor_config.json").await.ok();

let model_info = if let Some(model_info) = get_model_info(&api_repo).await {
Expand All @@ -244,6 +249,7 @@ async fn main() -> Result<(), RouterError> {
tokenizer_filename,
config_filename,
tokenizer_config_filename,
preprocessor_config_filename,
processor_config_filename,
model_info,
)
Expand All @@ -258,6 +264,7 @@ async fn main() -> Result<(), RouterError> {
repo.get("tokenizer.json"),
repo.get("config.json"),
repo.get("tokenizer_config.json"),
repo.get("preprocessor_config.json"),
repo.get("processor_config.json"),
None,
)
Expand Down Expand Up @@ -295,6 +302,8 @@ async fn main() -> Result<(), RouterError> {
HubTokenizerConfig::default()
});

let preprocessor_config =
preprocessor_config_filename.and_then(HubPreprocessorConfig::from_file);
let processor_config = processor_config_filename
.and_then(HubProcessorConfig::from_file)
.unwrap_or_default();
Expand Down Expand Up @@ -356,6 +365,7 @@ async fn main() -> Result<(), RouterError> {
ngrok_authtoken,
ngrok_edge,
tokenizer_config,
preprocessor_config,
processor_config,
messages_api_enabled,
disable_grammar_support,
Expand Down
8 changes: 5 additions & 3 deletions router/src/server.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@ use crate::kserve::{
use crate::validation::ValidationError;
use crate::{
BestOfSequence, Details, ErrorResponse, FinishReason, GenerateParameters, GenerateRequest,
GenerateResponse, GrammarType, HubModelInfo, HubProcessorConfig, HubTokenizerConfig, Info,
Message, PrefillToken, SimpleToken, StreamDetails, StreamResponse, Token, TokenizeResponse,
Usage, Validation,
GenerateResponse, GrammarType, HubModelInfo, HubPreprocessorConfig, HubProcessorConfig,
HubTokenizerConfig, Info, Message, PrefillToken, SimpleToken, StreamDetails, StreamResponse,
Token, TokenizeResponse, Usage, Validation,
};
use crate::{
ChatCompletion, ChatCompletionChoice, ChatCompletionChunk, ChatCompletionComplete,
Expand Down Expand Up @@ -1421,6 +1421,7 @@ pub async fn run(
_ngrok_authtoken: Option<String>,
_ngrok_edge: Option<String>,
tokenizer_config: HubTokenizerConfig,
preprocessor_config: Option<HubPreprocessorConfig>,
processor_config: HubProcessorConfig,
messages_api_enabled: bool,
grammar_support: bool,
Expand Down Expand Up @@ -1634,6 +1635,7 @@ pub async fn run(
validation_workers,
tokenizer,
config,
preprocessor_config,
max_best_of,
max_stop_sequences,
max_top_n_tokens,
Expand Down
Loading

0 comments on commit 99be263

Please sign in to comment.