feat: publish sanitized custom.97 source

tokenkit/README.md

@@ -0,0 +1,125 @@
+# TokenKit
+
+TokenKit 是从 [new-api](https://github.com/QuantumNous/new-api) 中拆分出的独立 Go 模块，用来在不调用上游 API 的情况下，计算或估算大模型请求的 token 数。
+
+它只做计数，不负责解析请求、下载图片或计费。调用方传入文本，或者图片的尺寸和 detail 设置，TokenKit 返回 token 数。可以脱离 new-api 主模块，嵌入其他 Go 网关、代理或客户端。
+
+## 能力
+
+- GPT 和 o 系列模型用 tiktoken 精确计数，GPT-4 和 GPT-3.5 走 `cl100k_base`，其余走 `o200k_base`
+- Claude、Gemini 和其他模型按字符类别加权估算，Claude Opus 4.7 前后的两套 tokenizer 分开处理
+- 按各家公开的尺寸规则计算图片 token，包括 OpenAI 的 detail 级别和 Gemini 3 的 media resolution
+- 不依赖 new-api 主模块、Gin、数据库或全局设置
+
+## 准确度
+
+文本权重是用真实 agent 会话校准的：从 Claude Code 和 Codex 的会话里取了 54 段，包括工具定义、工具调用和工具输出，和上游返回的 usage 逐条对比。
+
+| 模型 | 测试模型 | prompt 偏差 / 平均误差 | 输出文本 偏差 / 平均误差 |
+|---|---|---|---|
+| Claude（4.7 之前） | claude-haiku-4-5 | +0.5% / 3.3% | −0.6% / 3.8% |
+| Claude（4.7 及以后） | claude-sonnet-5 | +0.3% / 3.5% | −1.1% / 4.3% |
+| GPT（估算） | gpt-5.6-luna | +0.9% / 2.9% | −1.8% / 5.3% |
+| Gemini | gemini-3.7-flash | +0.7% / 4.2% | −1.3% / 4.7% |
+
+GPT 用 `Count` 时和 tiktoken 基本一致。有两处已知差异：
+- 只含缩进空格的空行会多算 1 个 token，来自 tiktoken-go v0.8.1 的分词正则，1.2 MB 的 agent 文本上一共多 3 个。
+- 超过 1024 字节、中间没有分隔的连续片段（例如没有标点的长段汉字、长串空格）会被切开再计数，误差不到 0.5%。不切的话，BPE 的耗时随片段长度平方增长：10 万个连续汉字要 20 秒左右，切开后约 0.1 秒。
+
+估算适合 agent 流量：代码、JSON、命令输出和中英文混排。几百 token 的短文本误差会大一些。
+
+图片规则和上游实测一致：OpenAI 的 patch 和 tile 计算、Claude 两个分辨率档位、Gemini 2.x 的切块。例外和注意事项见[图片](#图片)。
+
+## 安装
+
+TokenKit 要求 Go 1.26 或更高版本。
+
+```bash
+go get github.com/QuantumNous/new-api/tokenkit@latest
+```
+
+## 快速开始
+
+```go
+package main
+
+import (
+	"fmt"
+
+	"github.com/QuantumNous/new-api/tokenkit"
+)
+
+func main() {
+	prompt := "读取 main.go 并解释 init 函数"
+
+	// GPT 用 tiktoken 精确计数，其他模型估算
+	fmt.Println(tokenkit.Count("gpt-5.6-sol", prompt))
+	fmt.Println(tokenkit.Count("claude-opus-5-5", prompt))
+
+	// 只要估算，不跑 tokenizer，比如流式输出中途断开时补算 completion
+	fmt.Println(tokenkit.Estimate("gpt-5.6-sol", prompt))
+
+	// 一张 1920×1080 的截图
+	shot := tokenkit.Image{Width: 1920, Height: 1080}
+	fmt.Println(tokenkit.ImageTokens("claude-opus-5-5", shot))  // 2694
+	fmt.Println(tokenkit.ImageTokens("claude-haiku-4-5", shot)) // 1564
+	shot.Detail = "low"
+	fmt.Println(tokenkit.ImageTokens("gpt-5.6-sol", shot)) // 173
+}
+```
+
+## 模型分组
+
+`ModelFamily` 按模型名分组，不区分大小写，允许带厂商前缀和日期后缀，例如 `us.anthropic.claude-opus-4-7-v1:0`、`claude-sonnet-4-5-20250929`。
+
+| 分组 | 匹配 | 文本 |
+|---|---|---|
+| `FamilyOpenAI` | 名称含 `gpt-`、`o1`、`o3`、`o4`、`chatgpt` | `Count` 精确，`Estimate` 估算 |
+| `FamilyClaude` | Claude 4.7 之前，例如 Haiku 4.5、Sonnet 4.6 | 估算 |
+| `FamilyClaude47` | Claude 4.7 及以后，以及读不出版本号的名称，例如 Opus 4.7、Sonnet 5、Fable、Mythos | 估算 |
+| `FamilyGemini` | 名称含 `gemini` | 估算 |
+| `FamilyUnknown` | 其他模型 | 估算，权重未校准 |
+
+Claude 4.7 及以后的 tokenizer 对同样的文本多出约 30% 的 token，所以两组 Claude 用不同的权重。
+
+## 图片
+
+`ImageTokens(model, Image)` 返回一张图片占用的 prompt token。`Image.Width` 和 `Image.Height` 是像素尺寸，为 0 时按 1024×1024 计算。`ImageNeedsSize` 判断某个模型和 detail 设置下是否需要尺寸；返回 false 时可以不读取图片。
+
+| 模型 | 计算方式 |
+|---|---|
+| gpt-5.2 及以后、gpt-6 | 32px patch × 1.2，按模型和 detail 缩放；不传 detail 时 gpt-5.5 及以后按 `original` 计算 |
+| gpt-5、gpt-5.1、gpt-4o、gpt-4.1、o1、o3 | 512px tile，加 base tokens |
+| Claude 4.7 之前 | ⌈宽/28⌉ × ⌈高/28⌉，最长边 1568，最多 1568 个 patch，另加 4 |
+| Claude 4.7 及以后 | 同上，最长边 2576，最多 4784 个 patch，另加 3 |
+| Gemini 3 | 与尺寸无关，按 media resolution：280 / 560 / 1120 / 2240，默认 1120 |
+| Gemini 2.x | 两边都不超过 384 时 258；更大的图切成方块，每块 258，最多 12 块，另加 1 块缩略图 |
+| 其他模型 | 每张 520 |
+
+`Image.Detail` 填 OpenAI 的 detail（`low`、`high`、`auto`、`original`），或 Gemini 的 media resolution（`MEDIA_RESOLUTION_LOW` 等）。两类值只在对应的模型上生效，例如 Gemini 会忽略 `low`。
+
+注意事项：
+
+- 有两处按上游实际计费，而不是按官方文档。gpt-5.5 和 gpt-5.6 的 `high` 先缩到 2048×2048 以内，最短边再缩到 768，文档写的是 2500 patch 预算。gpt-6 的 `high` 先缩到 2048×2048 以内，再按 2500 patch 预算缩放，文档没有写 2048 这一步。
+- Gemini 3 用的是文档值。实测按图片宽高比低 2% 到 9%。
+- Gemini 2.x 的 4K 图会多算：3840×2160 实际 2322，这里给 3354。
+- 图片尺寸来自请求方，可能被伪造。超过 65535 像素的边会先缩小，单张最多按 30000 个 patch 计算，结果始终有上限。
+
+## 开发
+
+TokenKit 必须始终保持独立可构建，不依赖 new-api 主模块或 RelayKit。修改模块后，在 `tokenkit` 目录运行：
+
+```bash
+GOWORK=off go test ./...
+GOWORK=off go build ./...
+```
+
+`image_test.go` 里的期望值是上游实测值。改动图片规则时，用同样的方法重新测量：同一个请求带图和不带图各发一次，两次 input token 的差就是这张图的消耗。
+
+## 版本与兼容性
+
+TokenKit 当前使用 `v0.x` 版本，公开 API 可能在小版本中调整。估算权重和图片规则会随上游模型更新重新校准，同样的输入在不同版本之间可能得到不同的结果。
+
+## 许可证
+
+TokenKit 是 new-api 项目的一部分，遵循项目根目录中的 [GNU Affero General Public License v3.0](../LICENSE)。

tokenkit/go.mod

@@ -0,0 +1,15 @@
+module github.com/QuantumNous/new-api/tokenkit
+
+go 1.26
+
+require (
+	github.com/stretchr/testify v1.11.1
+	github.com/tiktoken-go/tokenizer v0.8.1
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dlclark/regexp2/v2 v2.5.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

tokenkit/go.sum

@@ -0,0 +1,14 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dlclark/regexp2/v2 v2.5.1 h1:E5Ug7Dh264W1ymdySmiHNcDG7fmsR307APCE5R07a20=
+github.com/dlclark/regexp2/v2 v2.5.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/tiktoken-go/tokenizer v0.8.1 h1:4obDoB6/dhdBt9xMweX4nww5cjdOq/nYF4ecwPq2+mg=
+github.com/tiktoken-go/tokenizer v0.8.1/go.mod h1:eLA0t6nGvn9mDc7gt90qt7pMat+gE9ViqwQ6l9B+tA4=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

tokenkit/image.go

@@ -0,0 +1,290 @@
+package tokenkit
+
+import (
+	"math"
+	"strings"
+)
+
+// Image describes one input image.
+type Image struct {
+	// Width and Height are the pixel dimensions. Zero or negative means
+	// unknown; where size matters, a 1024×1024 image is assumed.
+	Width, Height int
+	// Detail is the OpenAI detail level (low, high, auto, original) or the
+	// Gemini media resolution (MEDIA_RESOLUTION_LOW, _MEDIUM, _HIGH,
+	// _ULTRA_HIGH). Empty means the provider default. Each family ignores
+	// values meant for the other.
+	Detail string
+}
+
+const (
+	unknownImageSide = 1024
+	// Larger sides are scaled down first; OpenAI does the same for original
+	// detail and every other provider rejects such images.
+	maxImageSide = 65535
+	// OpenAI rejects images that need more patches, so no estimate exceeds it.
+	maxImagePatches = 30000
+	// Earlier flat estimate, kept for models whose sizing rules are unknown.
+	unknownFamilyImageTokens = 520
+)
+
+// ImageTokens returns the prompt tokens one image costs on model.
+func ImageTokens(model string, img Image) int {
+	w, h := img.Width, img.Height
+	if w <= 0 || h <= 0 {
+		w, h = unknownImageSide, unknownImageSide
+	}
+	if w > maxImageSide || h > maxImageSide {
+		w, h = fitWithin(w, h, maxImageSide)
+	}
+
+	switch ModelFamily(model) {
+	case FamilyOpenAI:
+		return openAIImageRuleFor(model, img.Detail).tokens(w, h)
+	case FamilyClaude:
+		// Standard tier; each image adds 4 tokens of framing (measured).
+		w, h = claudeResizedSize(w, h, 1568, 1568)
+		return ceilDiv(w, 28)*ceilDiv(h, 28) + 4
+	case FamilyClaude47:
+		// High-resolution tier; each image adds 3 tokens of framing (measured).
+		w, h = claudeResizedSize(w, h, 2576, 4784)
+		return ceilDiv(w, 28)*ceilDiv(h, 28) + 3
+	case FamilyGemini:
+		if geminiUsesMediaResolution(model) {
+			return geminiMediaResolutionTokens(img.Detail)
+		}
+		return geminiTileImageTokens(w, h)
+	default:
+		return unknownFamilyImageTokens
+	}
+}
+
+// ImageNeedsSize reports whether ImageTokens depends on the image dimensions
+// for model and detail. When it returns false, callers can skip reading the
+// image.
+func ImageNeedsSize(model, detail string) bool {
+	switch ModelFamily(model) {
+	case FamilyOpenAI:
+		rule := openAIImageRuleFor(model, detail)
+		return rule.tileTokens == 0 || !rule.lowDetail
+	case FamilyClaude, FamilyClaude47:
+		return true
+	case FamilyGemini:
+		return !geminiUsesMediaResolution(model)
+	default:
+		return false
+	}
+}
+
+// openAIImageRule is the sizing rule for one OpenAI model and detail level,
+// from https://developers.openai.com/api/docs/guides/images-vision and
+// measured upstream usage.
+type openAIImageRule struct {
+	// Tile-based models cost baseTokens plus tileTokens per 512px tile.
+	baseTokens, tileTokens int
+	lowDetail              bool
+
+	// Patch-based models cost 32px patches times multiplier. The multiplier
+	// is a float32 because upstream bills that way: 1225 patches × 1.2 bills
+	// 1471, not 1470.
+	multiplier float32
+	maxSide    int // fit within maxSide × maxSide
+	shortSide  int // scale the shortest side down to shortSide
+	budget     int // resizing patch budget
+}
+
+func openAIImageRuleFor(model, detail string) openAIImageRule {
+	model = strings.ToLower(model)
+	detail = strings.ToLower(detail)
+	switch detail {
+	case "low", "high", "original":
+	default:
+		detail = "auto"
+	}
+	major, minor, isGPT := modelVersion(model, "gpt-")
+
+	switch {
+	case isGPT && major >= 6:
+		// gpt-6-astra: auto means original. Upstream fits high within 2048
+		// before the 2,500-patch budget; the documentation omits the 2048 limit.
+		switch detail {
+		case "low":
+			return openAIImageRule{multiplier: 1.2, maxSide: 512}
+		case "high":
+			return openAIImageRule{multiplier: 1.2, maxSide: 2048, budget: 2500}
+		default:
+			return openAIImageRule{multiplier: 1.2}
+		}
+	case isGPT && major == 5 && minor >= 5:
+		// gpt-5.5 and gpt-5.6: auto means original. Upstream bills high as
+		// fit 2048 then shortest side 768, not the documented 2,500-patch budget.
+		switch detail {
+		case "low":
+			return openAIImageRule{multiplier: 1.2, maxSide: 512}
+		case "high":
+			return openAIImageRule{multiplier: 1.2, maxSide: 2048, shortSide: 768}
+		}
+		if minor == 5 {
+			return openAIImageRule{multiplier: 1.2, maxSide: 6000, budget: 10000}
+		}
+		return openAIImageRule{multiplier: 1.2}
+	case isGPT && major == 5 && minor == 4:
+		// gpt-5.4: auto means high.
+		switch detail {
+		case "low":
+			return openAIImageRule{multiplier: 1.2, maxSide: 2048, budget: 6144}
+		case "original":
+			return openAIImageRule{multiplier: 1.2, maxSide: 6000, budget: 10000}
+		default:
+			return openAIImageRule{multiplier: 1.2, maxSide: 2048, budget: 2500}
+		}
+	case isGPT && major == 5 && minor >= 2:
+		return openAIImageRule{multiplier: 1.2, maxSide: 2048, budget: 6144}
+	case isGPT && major == 5 && minor == 0 && strings.Contains(model, "nano"):
+		return openAIImageRule{multiplier: 1.5, budget: 1536}
+	case isGPT && major == 5 && minor == 0 && strings.Contains(model, "mini"):
+		return openAIImageRule{multiplier: 1.2, budget: 1536}
+	case isGPT && major == 5:
+		return openAIImageRule{baseTokens: 70, tileTokens: 140, lowDetail: detail == "low"}
+	case isGPT && major == 4 && minor == 1 && strings.Contains(model, "mini"):
+		return openAIImageRule{multiplier: 1.62, maxSide: 2048, budget: 6144}
+	case isGPT && major == 4 && minor == 1 && strings.Contains(model, "nano"):
+		return openAIImageRule{multiplier: 2.46, budget: 1536}
+	case strings.Contains(model, "4o-mini"):
+		return openAIImageRule{baseTokens: 2833, tileTokens: 5667, lowDetail: detail == "low"}
+	case strings.Contains(model, "o4-mini"):
+		return openAIImageRule{multiplier: 1.72, budget: 1536}
+	case !isGPT && (strings.Contains(model, "o1") || strings.Contains(model, "o3")):
+		return openAIImageRule{baseTokens: 75, tileTokens: 150, lowDetail: detail == "low"}
+	default:
+		// gpt-4o, gpt-4.1, gpt-4.5, and older vision models.
+		return openAIImageRule{baseTokens: 85, tileTokens: 170, lowDetail: detail == "low"}
+	}
+}
+
+func (r openAIImageRule) tokens(w, h int) int {
+	if r.tileTokens > 0 {
+		if r.lowDetail {
+			return r.baseTokens
+		}
+		w, h = fitWithin(w, h, 2048)
+		w, h = shrinkShortSide(w, h, 768)
+		return r.baseTokens + r.tileTokens*ceilDiv(w, 512)*ceilDiv(h, 512)
+	}
+
+	if r.maxSide > 0 {
+		w, h = fitWithin(w, h, r.maxSide)
+	}
+	patches := ceilDiv(w, 32) * ceilDiv(h, 32)
+	if short, long := min(w, h), max(w, h); r.shortSide > 0 && short > r.shortSide {
+		// Upstream keeps the long side unrounded here: 3000x2000 fits to
+		// 2048x1365, and 2048 x 768/1365 = 1152.3 needs 37 patch columns.
+		patches = ceilDiv(r.shortSide, 32) * ceilDiv(long*r.shortSide, short*32)
+	}
+	if r.budget > 0 && patches > r.budget {
+		shrink := math.Sqrt(float64(32*32*r.budget) / (float64(w) * float64(h)))
+		scaledW, scaledH := float64(w)*shrink/32, float64(h)*shrink/32
+		shrink *= min(math.Floor(scaledW)/scaledW, math.Floor(scaledH)/scaledH)
+		w, h = max(int(float64(w)*shrink), 1), max(int(float64(h)*shrink), 1)
+		patches = ceilDiv(w, 32) * ceilDiv(h, 32)
+	}
+	patches = min(patches, maxImagePatches)
+	return int(math.Ceil(float64(patches) * float64(r.multiplier)))
+}
+
+// claudeResizedSize is the size Claude scales an image to: the largest
+// aspect-preserving size whose padded sides fit maxEdge and whose 28px patches
+// fit maxTokens. Reference implementation:
+// https://platform.claude.com/docs/en/build-with-claude/vision-coordinates
+func claudeResizedSize(width, height, maxEdge, maxTokens int) (int, int) {
+	if claudeImageFits(width, height, maxEdge, maxTokens) {
+		return width, height
+	}
+	if height > width {
+		resizedH, resizedW := claudeResizedSize(height, width, maxEdge, maxTokens)
+		return resizedW, resizedH
+	}
+	aspectRatio := float64(width) / float64(height)
+	lo, hi := 1, width // lo always fits; hi never fits
+	for lo+1 < hi {
+		mid := (lo + hi) / 2
+		if claudeImageFits(mid, max(int(math.RoundToEven(float64(mid)/aspectRatio)), 1), maxEdge, maxTokens) {
+			lo = mid
+		} else {
+			hi = mid
+		}
+	}
+	return lo, max(int(math.RoundToEven(float64(lo)/aspectRatio)), 1)
+}
+
+func claudeImageFits(w, h, maxEdge, maxTokens int) bool {
+	return ceilDiv(w, 28)*28 <= maxEdge && ceilDiv(h, 28)*28 <= maxEdge && ceilDiv(w, 28)*ceilDiv(h, 28) <= maxTokens
+}
+
+// geminiUsesMediaResolution reports whether a lower-cased Gemini model bills
+// images by media resolution regardless of size (Gemini 3 and later, and names
+// without a version).
+func geminiUsesMediaResolution(model string) bool {
+	major, _, ok := modelVersion(strings.ToLower(model), "gemini-")
+	return !ok || major >= 3
+}
+
+// geminiMediaResolutionTokens returns the documented per-image cost for a
+// Gemini media resolution. Measured costs run 2-9% lower depending on aspect.
+func geminiMediaResolutionTokens(detail string) int {
+	level, ok := strings.CutPrefix(strings.ToLower(detail), "media_resolution_")
+	if !ok {
+		return 1120
+	}
+	switch level {
+	case "low":
+		return 280
+	case "medium":
+		return 560
+	case "ultra_high":
+		return 2240
+	default:
+		return 1120
+	}
+}
+
+// geminiTileImageTokens follows the Gemini 2.x rule measured on gemini-2.5-flash:
+// images up to 384px on both sides cost one 258-token tile; larger images are
+// cut into square tiles of a third less than the shortest side (256 to 768px),
+// at most 12, plus one thumbnail tile. 4K images bill below this cap.
+func geminiTileImageTokens(w, h int) int {
+	if w <= 384 && h <= 384 {
+		return 258
+	}
+	tile := min(max(min(w, h)*2/3, 256), 768)
+	tiles := min(ceilDiv(w, tile)*ceilDiv(h, tile), 12)
+	return (tiles + 1) * 258
+}
+
+// fitWithin scales an image down so neither side exceeds limit, rounding the
+// shorter side down. Smaller images are not enlarged.
+func fitWithin(w, h, limit int) (int, int) {
+	if w <= limit && h <= limit {
+		return w, h
+	}
+	if w >= h {
+		return limit, max(int(int64(h)*int64(limit)/int64(w)), 1)
+	}
+	return max(int(int64(w)*int64(limit)/int64(h)), 1), limit
+}
+
+// shrinkShortSide scales an image down so its shortest side is at most limit,
+// rounding the other side down.
+func shrinkShortSide(w, h, limit int) (int, int) {
+	if min(w, h) <= limit {
+		return w, h
+	}
+	if w <= h {
+		return limit, max(int(int64(h)*int64(limit)/int64(w)), 1)
+	}
+	return max(int(int64(w)*int64(limit)/int64(h)), 1), limit
+}
+
+func ceilDiv(a, b int) int {
+	return (a + b - 1) / b
+}

tokenkit/image_test.go

@@ -0,0 +1,124 @@
+package tokenkit
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// Unless marked otherwise, want is the per-image prompt-token cost measured
+// through upstream usage (request with the image minus the same request
+// without it) on 2026-10-07.
+func TestImageTokensMatchUpstreamUsage(t *testing.T) {
+	for _, tc := range []struct {
+		model  string
+		w, h   int
+		detail string
+		want   int
+	}{
+		// gpt-5.6: auto means original; high scales the shortest side to 768.
+		{"gpt-5.6-luna", 200, 200, "", 59},
+		{"gpt-5.6-luna", 1000, 1000, "", 1229},
+		{"gpt-5.6-luna", 1000, 1000, "auto", 1229},
+		{"gpt-5.6-luna", 1000, 1000, "original", 1229},
+		{"gpt-5.6-luna", 1000, 1000, "high", 692},
+		{"gpt-5.6-luna", 1000, 1000, "low", 308},
+		{"gpt-5.6-luna", 200, 200, "high", 59},
+		{"gpt-5.6-luna", 2048, 2048, "", 4916},
+		{"gpt-5.6-luna", 2048, 2048, "high", 692},
+		{"gpt-5.6-luna", 3840, 2160, "", 9793},
+		{"gpt-5.6-luna", 3840, 2160, "high", 1239},
+		{"gpt-5.6-luna", 3840, 2160, "low", 173},
+		{"gpt-5.6-luna", 600, 3000, "", 2144},
+		{"gpt-5.6-luna", 600, 3000, "high", 999},
+		{"gpt-5.6-luna", 600, 3000, "low", 77},
+		{"gpt-5.6-luna", 8000, 500, "", 4801},
+		{"gpt-5.6-luna", 8000, 500, "high", 308},
+		{"gpt-5.6-luna", 8000, 500, "low", 20},
+		{"gpt-5.6-luna", 3000, 2000, "high", 1066},
+		{"gpt-5.6-luna", 1170, 2532, "high", 1498},
+		{"gpt-5.6-luna", 1600, 1200, "high", 922},
+		// gpt-5.5 original is limited to 6000px and 10,000 patches.
+		{"gpt-5.5", 8000, 500, "", 2708},
+		{"gpt-5.5", 3840, 2160, "", 9793},
+		{"gpt-5.5", 2048, 2048, "high", 692},
+		// gpt-6-astra high fits within 2048, then applies a 2,500-patch budget.
+		{"gpt-6-astra", 1024, 1024, "high", 1229},
+		{"gpt-6-astra", 2048, 2048, "high", 3001},
+		{"gpt-6-astra", 3840, 2160, "high", 2765},
+		{"gpt-6-astra", 1170, 2532, "high", 2305},
+		{"gpt-6-astra", 3000, 2000, "high", 2881},
+		{"gpt-6-astra", 2048, 2048, "", 4916},
+		{"gpt-6-astra", 3840, 2160, "", 9793},
+		{"gpt-6-astra", 600, 3000, "", 2144},
+		{"gpt-6-astra", 3840, 2160, "low", 173},
+		// Tile-based models follow the documented formula.
+		{"gpt-4o", 1000, 1000, "", 85 + 170*4},
+		{"gpt-4o", 1000, 1000, "low", 85},
+		{"gpt-5.1", 200, 200, "high", 70 + 140},
+		// Claude: ceil(w/28) x ceil(h/28) after the tier resize, plus framing.
+		{"claude-haiku-4-5", 200, 200, "", 68},
+		{"claude-haiku-4-5", 1000, 1000, "", 1300},
+		{"claude-haiku-4-5", 1092, 1092, "", 1525},
+		{"claude-haiku-4-5", 1920, 1080, "", 1564},
+		{"claude-haiku-4-5", 1075, 1520, "", 1555},
+		{"claude-haiku-4-5", 2000, 1500, "", 1568},
+		{"claude-haiku-4-5", 3840, 2160, "", 1564},
+		{"claude-haiku-4-5", 4096, 512, "", 396},
+		{"claude-sonnet-5", 200, 200, "", 67},
+		{"claude-sonnet-5", 1000, 1000, "", 1299},
+		{"claude-sonnet-5", 1092, 1092, "", 1524},
+		{"claude-sonnet-5", 1920, 1080, "", 2694},
+		{"claude-sonnet-5", 1075, 1520, "", 2148},
+		{"claude-sonnet-5", 2000, 1500, "", 3891},
+		{"claude-sonnet-5", 3840, 2160, "", 4787},
+		{"claude-sonnet-5", 4096, 512, "", 1107},
+		// Gemini 2.x tiles.
+		{"gemini-2.5-flash", 384, 384, "", 258},
+		{"gemini-2.5-flash", 385, 385, "", 1290},
+		{"gemini-2.5-flash", 769, 769, "", 1290},
+		{"gemini-2.5-flash", 1536, 768, "", 1806},
+		{"gemini-2.5-flash", 1537, 768, "", 2322},
+		{"gemini-2.5-flash", 1920, 1080, "", 1806},
+		{"gemini-2.5-flash", 2000, 2000, "", 2580},
+		{"gemini-2.5-flash", 2304, 768, "", 2838},
+		{"gemini-2.5-flash", 3000, 1000, "", 2838},
+		{"gemini-2.5-flash", 2500, 2500, "", 3354},
+		{"gemini-2.5-flash", 4000, 3000, "", 3354},
+		// Gemini 3 bills by media resolution (documented values; measured 2-9% lower).
+		// An OpenAI detail level is not a Gemini media resolution.
+		{"gemini-3.7-flash", 1000, 1000, "", 1120},
+		{"gemini-3.7-flash", 1000, 1000, "MEDIA_RESOLUTION_LOW", 280},
+		{"gemini-3.7-flash", 1000, 1000, "media_resolution_medium", 560},
+		{"gemini-3.7-flash", 3840, 2160, "MEDIA_RESOLUTION_ULTRA_HIGH", 2240},
+		{"gemini-3.7-flash", 1000, 1000, "low", 1120},
+		// Unknown sizes assume 1024x1024.
+		{"gpt-5.6-sol", 0, 0, "", 1229},
+		{"claude-opus-4-7", 0, 0, "", 37*37 + 3},
+		{"gemini-2.5-flash", -1, 0, "", 1290},
+		{"deepseek-v4", 1000, 1000, "", 520},
+	} {
+		got := ImageTokens(tc.model, Image{Width: tc.w, Height: tc.h, Detail: tc.detail})
+		assert.Equal(t, tc.want, got, "%s %dx%d detail=%q", tc.model, tc.w, tc.h, tc.detail)
+	}
+}
+
+// Image dimensions come from client-supplied headers; the cost stays bounded.
+func TestImageTokensBoundForgedSizes(t *testing.T) {
+	const huge = 1 << 30
+	assert.Equal(t, 36001, ImageTokens("gpt-5.6-sol", Image{Width: huge, Height: huge}))
+	// The largest square within the visual-token budget: 39x39 and 69x69 patches.
+	assert.Equal(t, 39*39+4, ImageTokens("claude-sonnet-4-6", Image{Width: huge, Height: huge}))
+	assert.Equal(t, 69*69+3, ImageTokens("claude-opus-5-5", Image{Width: huge, Height: huge}))
+	assert.Equal(t, 13*258, ImageTokens("gemini-2.5-pro", Image{Width: huge, Height: huge}))
+}
+
+func TestImageNeedsSize(t *testing.T) {
+	assert.True(t, ImageNeedsSize("gpt-5.6-sol", "low"))
+	assert.True(t, ImageNeedsSize("gpt-4o", "high"))
+	assert.False(t, ImageNeedsSize("gpt-4o", "low"))
+	assert.True(t, ImageNeedsSize("claude-haiku-4-5", ""))
+	assert.True(t, ImageNeedsSize("gemini-2.5-flash", ""))
+	assert.False(t, ImageNeedsSize("gemini-3.7-flash", ""))
+	assert.False(t, ImageNeedsSize("deepseek-v4", ""))
+}

tokenkit/tokenizer.go

@@ -0,0 +1,102 @@
+package tokenkit
+
+import (
+	"strings"
+	"sync"
+	"unicode"
+	"unicode/utf8"
+
+	"github.com/tiktoken-go/tokenizer"
+	"github.com/tiktoken-go/tokenizer/codec"
+)
+
+var (
+	o200kBase  = sync.OnceValue(func() tokenizer.Codec { return codec.NewO200kBase() })
+	cl100kBase = sync.OnceValue(func() tokenizer.Codec { return codec.NewCl100kBase() })
+)
+
+// maxRunBytes bounds a run of letters, whitespace, or symbols passed to the
+// encoder in one piece. BPE cost grows with the square of a pre-tokenized
+// piece: 100,000 Chinese characters without punctuation take about 20 s in one
+// piece. Cutting longer runs keeps counting linear; ordinary text has no such
+// runs, and a cut changes the count only inside the run (under 0.5% for CJK).
+const maxRunBytes = 1024
+
+// Count returns the token count of text for model. GPT and o-series models
+// are counted with their tiktoken encoding; other models use Estimate.
+func Count(model, text string) int {
+	if text == "" {
+		return 0
+	}
+	model = strings.ToLower(model)
+	if ModelFamily(model) != FamilyOpenAI {
+		return Estimate(model, text)
+	}
+	encoder := o200kBase()
+	if usesCl100k(model) {
+		encoder = cl100kBase()
+	}
+
+	total, start := 0, 0
+	runClass, runStart := runNone, 0
+	for i, r := range text {
+		class := runClassOf(r)
+		if class != runClass {
+			runClass, runStart = class, i
+			continue
+		}
+		if class != runNumber && i-runStart >= maxRunBytes {
+			n, _ := encoder.Count(text[start:i])
+			total += n
+			start, runStart = i, i
+		}
+	}
+	n, _ := encoder.Count(text[start:])
+	return total + n
+}
+
+type runClass int
+
+const (
+	runNone runClass = iota
+	runLetter
+	runNumber
+	runSpace
+	runSymbol
+)
+
+// runClassOf groups runes the way tiktoken's pre-tokenizer joins them into
+// pieces. Digits are split into groups of three by the encoder itself.
+func runClassOf(r rune) runClass {
+	if r < utf8.RuneSelf {
+		switch {
+		case 'a' <= r && r <= 'z' || 'A' <= r && r <= 'Z':
+			return runLetter
+		case '0' <= r && r <= '9':
+			return runNumber
+		case r == ' ' || '\t' <= r && r <= '\r':
+			return runSpace
+		default:
+			return runSymbol
+		}
+	}
+	switch {
+	case unicode.IsLetter(r) || unicode.IsMark(r):
+		return runLetter
+	case unicode.IsNumber(r):
+		return runNumber
+	case unicode.IsSpace(r):
+		return runSpace
+	default:
+		return runSymbol
+	}
+}
+
+// usesCl100k reports whether an OpenAI model predates o200k_base: GPT-4 (but
+// not GPT-4o, GPT-4.1, or GPT-4.5) and GPT-3.5. Every later model, including
+// GPT-5, GPT-6, and the o-series, uses o200k_base.
+func usesCl100k(model string) bool {
+	model = strings.TrimPrefix(model, "ft:")
+	return model == "gpt-4" || strings.HasPrefix(model, "gpt-4-") ||
+		strings.HasPrefix(model, "gpt-3.5") || strings.HasPrefix(model, "gpt-35")
+}

tokenkit/tokenkit.go

@@ -0,0 +1,245 @@
+// Package tokenkit estimates LLM token counts for text and images without
+// calling an upstream API.
+//
+// Text is counted exactly with tiktoken for OpenAI GPT and o-series models and
+// estimated with per-tokenizer weights for Claude, Gemini, and other models.
+// Image costs follow each provider's published sizing rules. The weights were
+// calibrated against upstream usage on real agent traffic (Claude Code and
+// Codex sessions).
+package tokenkit
+
+import (
+	"math"
+	"strconv"
+	"strings"
+	"unicode"
+)
+
+// Family is a tokenizer family. Models in one family tokenize text alike.
+type Family string
+
+const (
+	FamilyOpenAI   Family = "openai"    // GPT and o-series models
+	FamilyClaude   Family = "claude"    // Claude models before Claude Opus 4.7
+	FamilyClaude47 Family = "claude4.7" // Claude Opus 4.7 and later, including Claude 5, Fable, and Mythos
+	FamilyGemini   Family = "gemini"    // Gemini models
+	FamilyUnknown  Family = "unknown"   // everything else
+)
+
+// ModelFamily classifies a model name, case-insensitively. Vendor prefixes and
+// date suffixes are allowed (for example us.anthropic.claude-opus-4-7-v1:0).
+func ModelFamily(model string) Family {
+	model = strings.ToLower(model)
+	switch {
+	case strings.Contains(model, "gemini"):
+		return FamilyGemini
+	case strings.Contains(model, "claude"):
+		if claudeUsesOpus47Tokenizer(model) {
+			return FamilyClaude47
+		}
+		return FamilyClaude
+	case isOpenAITextModel(model):
+		return FamilyOpenAI
+	default:
+		return FamilyUnknown
+	}
+}
+
+func isOpenAITextModel(model string) bool {
+	for _, marker := range []string{"gpt-", "o1", "o3", "o4", "chatgpt"} {
+		if strings.Contains(model, marker) {
+			return true
+		}
+	}
+	return false
+}
+
+// claudeUsesOpus47Tokenizer reports whether a lower-cased Claude model name uses
+// the tokenizer and high-resolution image tier introduced with Claude Opus 4.7.
+// Claude 4.7 and later use them; names without a version are treated as new.
+func claudeUsesOpus47Tokenizer(model string) bool {
+	major, minor, ok := modelVersion(model, "claude-")
+	if !ok {
+		return true
+	}
+	return major > 4 || (major == 4 && minor >= 7)
+}
+
+// modelVersion reads the first one- or two-digit version numbers after prefix,
+// skipping family names (claude-opus-4-7, claude-3-5-sonnet, gemini-3.7-flash).
+// A date such as 20250929 ends the version. ok is false when no version is found.
+func modelVersion(model, prefix string) (major, minor int, ok bool) {
+	_, name, found := strings.Cut(model, prefix)
+	if !found {
+		return 0, 0, false
+	}
+	version := make([]int, 0, 2)
+	for part := range strings.FieldsFuncSeq(name, func(r rune) bool { return r == '-' || r == '.' }) {
+		digits := len(part) - len(strings.TrimLeft(part, "0123456789"))
+		if digits == 0 || digits > 2 {
+			if len(version) > 0 {
+				break
+			}
+			continue
+		}
+		n, _ := strconv.Atoi(part[:digits])
+		version = append(version, n)
+		if len(version) == 2 || digits < len(part) {
+			break
+		}
+	}
+	switch len(version) {
+	case 0:
+		return 0, 0, false
+	case 1:
+		return version[0], 0, true
+	default:
+		return version[0], version[1], true
+	}
+}
+
+// weights are the estimated tokens per text feature.
+type weights struct {
+	Word       float64 // Latin word (each run of letters)
+	Number     float64 // each run of digits
+	CJK        float64 // each Chinese, Japanese, or Korean character
+	Symbol     float64 // each ordinary punctuation mark
+	MathSymbol float64 // each math symbol (∑, ∫, ∂, √, ...)
+	URLDelim   float64 // each URL delimiter (/ : ? & = ; # %)
+	AtSign     float64 // each @, which splits words
+	Emoji      float64 // each emoji
+	Newline    float64 // each newline or tab
+	Space      float64 // each other whitespace character
+}
+
+// Word, Number, CJK, Symbol, URLDelim, Newline, and Space are calibrated
+// against upstream counts on agent traffic. MathSymbol, AtSign, and Emoji make
+// up under 0.3% of that traffic and keep their earlier values. FamilyUnknown
+// keeps the earlier OpenAI weights; no other vendor was calibrated.
+var familyWeights = map[Family]weights{
+	FamilyGemini: {
+		Word: 0.89, Number: 2.42, CJK: 0.93, Symbol: 0.65, MathSymbol: 1.05, URLDelim: 1.3, AtSign: 2.5, Emoji: 1.08, Newline: 1.28, Space: 0,
+	},
+	FamilyClaude: {
+		Word: 1.24, Number: 1.53, CJK: 1.38, Symbol: 0.66, MathSymbol: 4.52, URLDelim: 0.74, AtSign: 2.82, Emoji: 2.6, Newline: 1.19, Space: 0.14,
+	},
+	FamilyClaude47: {
+		Word: 1.52, Number: 1.77, CJK: 1.4, Symbol: 0.89, MathSymbol: 4.52, URLDelim: 1.45, AtSign: 2.82, Emoji: 2.6, Newline: 1.38, Space: 0.21,
+	},
+	FamilyOpenAI: {
+		Word: 0.75, Number: 1.86, CJK: 0.96, Symbol: 0.6, MathSymbol: 2.68, URLDelim: 0.77, AtSign: 2.0, Emoji: 2.12, Newline: 0.95, Space: 0.17,
+	},
+	FamilyUnknown: {
+		Word: 1.02, Number: 1.55, CJK: 0.85, Symbol: 0.4, MathSymbol: 2.68, URLDelim: 1.0, AtSign: 2.0, Emoji: 2.12, Newline: 0.5, Space: 0.42,
+	},
+}
+
+// Estimate returns the estimated token count of text for model, using the
+// weights of the model's family. It never runs a tokenizer; use Count for an
+// exact count where one is available.
+func Estimate(model, text string) int {
+	if text == "" {
+		return 0
+	}
+	return EstimateFamily(ModelFamily(model), text)
+}
+
+// EstimateFamily returns the estimated token count of text for a tokenizer
+// family. Unrecognized families use the FamilyUnknown weights.
+func EstimateFamily(family Family, text string) int {
+	w, ok := familyWeights[family]
+	if !ok {
+		w = familyWeights[FamilyUnknown]
+	}
+	var count float64
+
+	type wordType int
+	const (
+		none wordType = iota
+		latin
+		number
+	)
+	current := none
+
+	for _, r := range text {
+		if unicode.IsSpace(r) {
+			current = none
+			if r == '\n' || r == '\t' {
+				count += w.Newline
+			} else {
+				count += w.Space
+			}
+			continue
+		}
+
+		if isCJK(r) {
+			current = none
+			count += w.CJK
+			continue
+		}
+
+		if isEmoji(r) {
+			current = none
+			count += w.Emoji
+			continue
+		}
+
+		// A run of letters or digits counts once; switching between letters
+		// and digits starts a new run.
+		if unicode.IsLetter(r) || unicode.IsNumber(r) {
+			next := latin
+			if unicode.IsNumber(r) {
+				next = number
+			}
+			if current != next {
+				if next == number {
+					count += w.Number
+				} else {
+					count += w.Word
+				}
+				current = next
+			}
+			continue
+		}
+
+		current = none
+		switch {
+		case isMathSymbol(r):
+			count += w.MathSymbol
+		case r == '@':
+			count += w.AtSign
+		case isURLDelim(r):
+			count += w.URLDelim
+		default:
+			count += w.Symbol
+		}
+	}
+
+	return int(math.Ceil(count))
+}
+
+func isCJK(r rune) bool {
+	return unicode.Is(unicode.Han, r) ||
+		(r >= 0x3040 && r <= 0x30FF) || // Japanese kana
+		(r >= 0xAC00 && r <= 0xD7A3) // Korean Hangul syllables
+}
+
+func isEmoji(r rune) bool {
+	return (r >= 0x1F300 && r <= 0x1F9FF) || // pictographs, emoticons, supplemental symbols
+		(r >= 0x2600 && r <= 0x26FF) || // miscellaneous symbols
+		(r >= 0x2700 && r <= 0x27BF) || // dingbats
+		(r >= 0x1FA00 && r <= 0x1FAFF) // symbols and pictographs extended-A
+}
+
+func isMathSymbol(r rune) bool {
+	if strings.ContainsRune("∑∫∂√∞≤≥≠≈±×÷∈∉∋∌⊂⊃⊆⊇∪∩∧∨¬∀∃∄∅∆∇∝∟∠∡∢°′″‴⁺⁻⁼⁽⁾ⁿ₀₁₂₃₄₅₆₇₈₉₊₋₌₍₎²³¹⁴⁵⁶⁷⁸⁹⁰", r) {
+		return true
+	}
+	return (r >= 0x2200 && r <= 0x22FF) || // mathematical operators
+		(r >= 0x2A00 && r <= 0x2AFF) || // supplemental mathematical operators
+		(r >= 0x1D400 && r <= 0x1D7FF) // mathematical alphanumeric symbols
+}
+
+func isURLDelim(r rune) bool {
+	return strings.ContainsRune("/:?&=;#%", r)
+}

tokenkit/tokenkit_test.go

@@ -0,0 +1,83 @@
+package tokenkit
+
+import (
+	"math/rand"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/tiktoken-go/tokenizer/codec"
+)
+
+func TestModelFamily(t *testing.T) {
+	for _, tc := range []struct {
+		model  string
+		family Family
+	}{
+		{"claude-3-5-sonnet-20240620", FamilyClaude},
+		{"claude-sonnet-4-20250514", FamilyClaude},
+		{"claude-sonnet-4-5-20250929", FamilyClaude},
+		{"claude-haiku-4-5", FamilyClaude},
+		{"claude-sonnet-4-6", FamilyClaude},
+		// Claude Opus 4.7 introduced the tokenizer every later Claude model uses.
+		{"claude-opus-4-7", FamilyClaude47},
+		{"claude-opus-4-7@20260416", FamilyClaude47},
+		{"us.anthropic.claude-opus-4-7-v1:0", FamilyClaude47},
+		{"claude-sonnet-5", FamilyClaude47},
+		{"claude-fable-5-1", FamilyClaude47},
+		{"Claude-Opus-5-5", FamilyClaude47},
+		{"claude-mythos-preview", FamilyClaude47},
+		{"gemini-3.7-flash", FamilyGemini},
+		{"gpt-5.6-sol", FamilyOpenAI},
+		{"gpt-6-astra", FamilyOpenAI},
+		{"o3-mini", FamilyOpenAI},
+		{"deepseek-v4", FamilyUnknown},
+	} {
+		assert.Equal(t, tc.family, ModelFamily(tc.model), tc.model)
+	}
+}
+
+func TestEstimateUsesFamilyWeights(t *testing.T) {
+	const text = "func main() {\n\tfmt.Println(\"统计 token\", 42)\n}\nSee https://example.com/a?b=1"
+	require.NotEqual(t, EstimateFamily(FamilyClaude, text), EstimateFamily(FamilyClaude47, text))
+
+	assert.Equal(t, EstimateFamily(FamilyClaude, text), Estimate("claude-haiku-4-5", text))
+	assert.Equal(t, EstimateFamily(FamilyClaude47, text), Estimate("claude-sonnet-5", text))
+	assert.Equal(t, EstimateFamily(FamilyUnknown, text), Estimate("deepseek-v4", text))
+	assert.Equal(t, EstimateFamily(FamilyUnknown, text), EstimateFamily("other", text))
+	assert.Zero(t, Estimate("claude-sonnet-5", ""))
+}
+
+func TestCountUsesTiktokenForOpenAIModels(t *testing.T) {
+	const text = "大语言模型按 token 计费；Large language models bill by token. 🙂"
+	o200k, _ := codec.NewO200kBase().Count(text)
+	cl100k, _ := codec.NewCl100kBase().Count(text)
+	require.NotEqual(t, o200k, cl100k)
+
+	assert.Equal(t, o200k, Count("gpt-5.6-sol", text))
+	assert.Equal(t, o200k, Count("gpt-6-astra", text))
+	assert.Equal(t, o200k, Count("gpt-4o-mini", text))
+	assert.Equal(t, cl100k, Count("gpt-4", text))
+	assert.Equal(t, cl100k, Count("gpt-3.5-turbo", text))
+	assert.Equal(t, Estimate("claude-sonnet-5", text), Count("claude-sonnet-5", text))
+}
+
+// Long runs without separators are cut before encoding so BPE stays linear;
+// ordinary text is counted exactly and cut runs stay within 0.5%.
+func TestCountCutsOnlyLongRuns(t *testing.T) {
+	encoder := codec.NewO200kBase()
+	ordinary := strings.Repeat("func main() {\n\tfmt.Println(\"统计 token，按模型计费。\", 42)\n}\n| a | b |\n|---|---|\n", 200)
+	want, _ := encoder.Count(ordinary)
+	assert.Equal(t, want, Count("gpt-5.6-sol", ordinary))
+
+	rng := rand.New(rand.NewSource(1))
+	var cjk strings.Builder
+	for range 5000 {
+		cjk.WriteRune(rune(0x4E00 + rng.Intn(0x9FA5-0x4E00)))
+	}
+	for _, text := range []string{cjk.String(), strings.Repeat("token", 2000), "a" + strings.Repeat(" ", 10000) + "b", strings.Repeat("=", 10000)} {
+		exact, _ := encoder.Count(text)
+		assert.InEpsilon(t, exact, Count("gpt-5.6-sol", text), 0.005)
+	}
+}