Merge pull request #9 from marqo-ai/ali/cp-727-new-models

* update model registry with marqo 2.11.0 --------- Co-authored-by: Ali Khalilvandi <[email protected]>
marqo-ai · Aug 22, 2024 · 0689006 · 0689006
2 parents b1c7af5 + decf409
commit 0689006
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 2 deletions.
diff --git a/src/marqo_commons/model_registry/model_properties_data/hf_properties.py b/src/marqo_commons/model_registry/model_properties_data/hf_properties.py
@@ -175,8 +175,7 @@ def _get_hf_properties() -> Dict:
             dimensions=1024,
             tokens=512,
             notes="",
-            text_query_prefix="query: ",
-            text_chunk_prefix="passage: ",
+            text_query_prefix="Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:",
         ),
         "hf/GIST-large-Embedding-v0": HFModelProperties(
             name="avsolatorio/GIST-large-Embedding-v0",
@@ -187,41 +186,60 @@ def _get_hf_properties() -> Dict:
             name="BAAI/bge-small-en-v1.5",
             dimensions=384,
             tokens=512,
+            text_query_prefix="Represent this sentence for searching relevant passages: ",
         ),
         "hf/bge-base-en-v1.5": HFModelProperties(
             name="BAAI/bge-base-en-v1.5",
             dimensions=768,
             tokens=512,
+            text_query_prefix="Represent this sentence for searching relevant passages: ",
         ),
         "hf/bge-large-en-v1.5": HFModelProperties(
             name="BAAI/bge-large-en-v1.5",
             dimensions=1024,
             tokens=512,
+            text_query_prefix="Represent this sentence for searching relevant passages: ",
         ),
         "hf/bge-small-zh-v1.5": HFModelProperties(
             name="BAAI/bge-small-zh-v1.5",
             dimensions=512,
             tokens=512,
+            text_query_prefix="为这个句子生成表示以用于检索相关文章：",
         ),
         "hf/bge-base-zh-v1.5": HFModelProperties(
             name="BAAI/bge-base-zh-v1.5",
             dimensions=768,
             tokens=512,
+            text_query_prefix="为这个句子生成表示以用于检索相关文章：",
         ),
         "hf/bge-large-zh-v1.5": HFModelProperties(
             name="BAAI/bge-large-zh-v1.5",
             dimensions=1024,
             tokens=512,
+            text_query_prefix="为这个句子生成表示以用于检索相关文章：",
         ),
         "hf/snowflake-arctic-embed-l": HFModelProperties(
             name="Snowflake/snowflake-arctic-embed-l",
             dimensions=1024,
             tokens=512,
+            text_query_prefix="Represent this sentence for searching relevant passages: ",
         ),
         "hf/ember-v1": HFModelProperties(
             name="llmrails/ember-v1",
             dimensions=1024,
             tokens=512,
         ),
+        "hf/snowflake-arctic-embed-m-v1.5": HFModelProperties(
+            name="Snowflake/snowflake-arctic-embed-m-v1.5",
+            dimensions=768,
+            tokens=512,
+            text_query_prefix="Represent this sentence for searching relevant passages: ",
+        ),
+        "hf/snowflake-arctic-embed-m": HFModelProperties(
+            name="Snowflake/snowflake-arctic-embed-m",
+            dimensions=768,
+            tokens=512,
+            text_query_prefix="Represent this sentence for searching relevant passages: ",
+        ),
     }
     return HF_MODEL_PROPERTIES
diff --git a/src/marqo_commons/model_registry/model_properties_data/sbert_properties.py b/src/marqo_commons/model_registry/model_properties_data/sbert_properties.py
@@ -77,6 +77,11 @@ def _get_sbert_properties() -> Dict:
             dimensions=768,
             tokens=128,
         ),
+        "sentence-transformers/all-MiniLM-L12-v2": SbertProperties(
+            name="sentence-transformers/all-MiniLM-L12-v2",
+            dimensions=384,
+            tokens=256,
+        ),
     }
 
     return SBERT_MODEL_PROPERTIES