Skip to content

Commit

Permalink
Add CLIPTextModel and CLIPVisionModel (#829)
Browse files Browse the repository at this point in the history
* Add `CLIPTextModel` and `CLIPVisionModel`

* Fix jinja2 version for tests
  • Loading branch information
xenova authored Jul 1, 2024
1 parent 3072008 commit c89299f
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 1 deletion.
25 changes: 24 additions & 1 deletion src/models.js
Original file line number Diff line number Diff line change
Expand Up @@ -3099,6 +3099,18 @@ export class CLIPPreTrainedModel extends PreTrainedModel { }
*/
export class CLIPModel extends CLIPPreTrainedModel { }

/**
* The text model from CLIP without any head or projection on top.
*/
export class CLIPTextModel extends CLIPPreTrainedModel {
/** @type {PreTrainedModel.from_pretrained} */
static async from_pretrained(pretrained_model_name_or_path, options = {}) {
// Update default model file name if not provided
options.model_file_name ??= 'text_model';
return super.from_pretrained(pretrained_model_name_or_path, options);
}
}

/**
* CLIP Text Model with a projection layer on top (a linear layer on top of the pooled output)
*
Expand Down Expand Up @@ -3126,7 +3138,6 @@ export class CLIPModel extends CLIPPreTrainedModel { }
* ```
*/
export class CLIPTextModelWithProjection extends CLIPPreTrainedModel {

/** @type {PreTrainedModel.from_pretrained} */
static async from_pretrained(pretrained_model_name_or_path, options = {}) {
// Update default model file name if not provided
Expand All @@ -3135,6 +3146,18 @@ export class CLIPTextModelWithProjection extends CLIPPreTrainedModel {
}
}

/**
* The vision model from CLIP without any head or projection on top.
*/
export class CLIPVisionModel extends CLIPPreTrainedModel {
/** @type {PreTrainedModel.from_pretrained} */
static async from_pretrained(pretrained_model_name_or_path, options = {}) {
// Update default model file name if not provided
options.model_file_name ??= 'vision_model';
return super.from_pretrained(pretrained_model_name_or_path, options);
}
}

/**
* CLIP Vision Model with a projection layer on top (a linear layer on top of the pooled output)
*
Expand Down
1 change: 1 addition & 0 deletions tests/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@ sacremoses==0.0.53
sentencepiece==0.1.99
protobuf==4.24.3
rjieba==0.1.11
jinja2==3.1.0

0 comments on commit c89299f

Please sign in to comment.