Merge pull request #149 from dqbd/dqbd/gpt-5

Author: dqbd

.changeset/pink-apes-decide.md

@@ -0,0 +1,7 @@
+---
+"tiktoken": patch
+"js-tiktoken": patch
+"@dqbd/tiktoken": patch
+---
+
+Add gpt-5 models

.github/workflows/build_js.yml

@@ -1,6 +1,6 @@
 name: Build and Test JS/WASM
 
-on: [push, pull_request, workflow_dispatch]
+on: [pull_request, workflow_dispatch]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}

js/src/core.ts

@@ -318,7 +318,14 @@ export function getEncodingNameForModel(model: TiktokenModel) {
     case "gpt-4.1-nano":
     case "gpt-4.1-nano-2025-04-14":
     case "gpt-4.5-preview":
-    case "gpt-4.5-preview-2025-02-27": {
+    case "gpt-4.5-preview-2025-02-27":
+    case "gpt-5":
+    case "gpt-5-2025-08-07":
+    case "gpt-5-nano":
+    case "gpt-5-nano-2025-08-07":
+    case "gpt-5-mini":
+    case "gpt-5-mini-2025-08-07":
+    case "gpt-5-chat-latest": {
       return "o200k_base";
     }
     default:

tiktoken/model_to_encoding.json

@@ -97,5 +97,12 @@
     "gpt-4.1-nano": "o200k_base",
     "gpt-4.1-nano-2025-04-14": "o200k_base",
     "gpt-4.5-preview": "o200k_base",
-    "gpt-4.5-preview-2025-02-27": "o200k_base"
+    "gpt-4.5-preview-2025-02-27": "o200k_base",
+    "gpt-5": "o200k_base",
+    "gpt-5-2025-08-07": "o200k_base",
+    "gpt-5-nano": "o200k_base",
+    "gpt-5-nano-2025-08-07": "o200k_base",
+    "gpt-5-mini": "o200k_base",
+    "gpt-5-mini-2025-08-07": "o200k_base",
+    "gpt-5-chat-latest": "o200k_base"
 }

wasm/Cargo.toml

@@ -22,4 +22,10 @@ gloo-utils = { version = "0.1", features = ["serde"] }
 
 [features]
 default = ["inline"]
-inline = []
+inline = []
+
+[package.metadata.wasm-pack.profile.profiling]
+wasm-opt = ['-O', '']
+
+[package.metadata.wasm-pack.profile.release]
+wasm-opt = ['-O', '--enable-bulk-memory']

wasm/src/lib.rs

@@ -462,7 +462,14 @@ export type TiktokenModel =
     | "gpt-4.1-nano"
     | "gpt-4.1-nano-2025-04-14"
     | "gpt-4.5-preview"
-    | "gpt-4.5-preview-2025-02-27";
+    | "gpt-4.5-preview-2025-02-27"
+    | "gpt-5"
+    | "gpt-5-2025-08-07"
+    | "gpt-5-nano"
+    | "gpt-5-nano-2025-08-07"
+    | "gpt-5-mini"
+    | "gpt-5-mini-2025-08-07"
+    | "gpt-5-chat-latest";
 
 /**
  * @param {TiktokenModel} encoding
@@ -578,6 +585,13 @@ pub fn encoding_for_model(
         "gpt-4.1-nano-2025-04-14" => Ok("o200k_base"),
         "gpt-4.5-preview" => Ok("o200k_base"),
         "gpt-4.5-preview-2025-02-27" => Ok("o200k_base"),
+        "gpt-5" => Ok("o200k_base"),
+        "gpt-5-2025-08-07" => Ok("o200k_base"),
+        "gpt-5-nano" => Ok("o200k_base"),
+        "gpt-5-nano-2025-08-07" => Ok("o200k_base"),
+        "gpt-5-mini" => Ok("o200k_base"),
+        "gpt-5-mini-2025-08-07" => Ok("o200k_base"),
+        "gpt-5-chat-latest" => Ok("o200k_base"),
         model => Err(JsError::new(
             format!("Invalid model: {}", model.to_string()).as_str(),
         )),