From 4431360d9625d5e86b6873e8e4809c44288a6d8d Mon Sep 17 00:00:00 2001 From: Arsen Ibragimov Date: Fri, 11 Oct 2024 21:56:53 +0200 Subject: [PATCH 1/2] add support for o1 models Added support for OpenAI o1-mini and o1-preview models by mapping them to the `o200k_base` encoding. This update ensures compatibility with the latest tiktoken changes as introduced in version 0.8.0 of the official library. Fixes compatibility issues with models not yet included in tiktoken-php. --- src/EncoderProvider.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/EncoderProvider.php b/src/EncoderProvider.php index 331dfdb..93776b5 100644 --- a/src/EncoderProvider.php +++ b/src/EncoderProvider.php @@ -47,6 +47,7 @@ final class EncoderProvider implements ResetInterface ], ]; private const MODEL_PREFIX_TO_ENCODING = [ + "o1-": "o200k_base", 'gpt-4o-' => 'o200k_base', 'gpt-4-' => 'cl100k_base', 'gpt-3.5-turbo-' => 'cl100k_base', From 06f762114995a00cdce12637caa2135bcacbc4ba Mon Sep 17 00:00:00 2001 From: Arsen Ibragimov Date: Sat, 12 Oct 2024 17:49:59 +0200 Subject: [PATCH 2/2] Fix syntax from JSON-style ("key": "value") to PHP array format ('key' => 'value'). Co-authored-by: Deni --- src/EncoderProvider.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/EncoderProvider.php b/src/EncoderProvider.php index 93776b5..6b29688 100644 --- a/src/EncoderProvider.php +++ b/src/EncoderProvider.php @@ -47,7 +47,7 @@ final class EncoderProvider implements ResetInterface ], ]; private const MODEL_PREFIX_TO_ENCODING = [ - "o1-": "o200k_base", + 'o1-' => 'o200k_base', 'gpt-4o-' => 'o200k_base', 'gpt-4-' => 'cl100k_base', 'gpt-3.5-turbo-' => 'cl100k_base',