feat: publish sanitized custom.97 source
This commit is contained in:
1022 files changed
+69903
-19207
No files matched your search
@@ -0,0 +1,245 @@
|
||||
// Package tokenkit estimates LLM token counts for text and images without
|
||||
// calling an upstream API.
|
||||
//
|
||||
// Text is counted exactly with tiktoken for OpenAI GPT and o-series models and
|
||||
// estimated with per-tokenizer weights for Claude, Gemini, and other models.
|
||||
// Image costs follow each provider's published sizing rules. The weights were
|
||||
// calibrated against upstream usage on real agent traffic (Claude Code and
|
||||
// Codex sessions).
|
||||
package tokenkit
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Family is a tokenizer family. Models in one family tokenize text alike.
|
||||
type Family string
|
||||
|
||||
const (
|
||||
FamilyOpenAI Family = "openai" // GPT and o-series models
|
||||
FamilyClaude Family = "claude" // Claude models before Claude Opus 4.7
|
||||
FamilyClaude47 Family = "claude4.7" // Claude Opus 4.7 and later, including Claude 5, Fable, and Mythos
|
||||
FamilyGemini Family = "gemini" // Gemini models
|
||||
FamilyUnknown Family = "unknown" // everything else
|
||||
)
|
||||
|
||||
// ModelFamily classifies a model name, case-insensitively. Vendor prefixes and
|
||||
// date suffixes are allowed (for example us.anthropic.claude-opus-4-7-v1:0).
|
||||
func ModelFamily(model string) Family {
|
||||
model = strings.ToLower(model)
|
||||
switch {
|
||||
case strings.Contains(model, "gemini"):
|
||||
return FamilyGemini
|
||||
case strings.Contains(model, "claude"):
|
||||
if claudeUsesOpus47Tokenizer(model) {
|
||||
return FamilyClaude47
|
||||
}
|
||||
return FamilyClaude
|
||||
case isOpenAITextModel(model):
|
||||
return FamilyOpenAI
|
||||
default:
|
||||
return FamilyUnknown
|
||||
}
|
||||
}
|
||||
|
||||
func isOpenAITextModel(model string) bool {
|
||||
for _, marker := range []string{"gpt-", "o1", "o3", "o4", "chatgpt"} {
|
||||
if strings.Contains(model, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// claudeUsesOpus47Tokenizer reports whether a lower-cased Claude model name uses
|
||||
// the tokenizer and high-resolution image tier introduced with Claude Opus 4.7.
|
||||
// Claude 4.7 and later use them; names without a version are treated as new.
|
||||
func claudeUsesOpus47Tokenizer(model string) bool {
|
||||
major, minor, ok := modelVersion(model, "claude-")
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
return major > 4 || (major == 4 && minor >= 7)
|
||||
}
|
||||
|
||||
// modelVersion reads the first one- or two-digit version numbers after prefix,
|
||||
// skipping family names (claude-opus-4-7, claude-3-5-sonnet, gemini-3.7-flash).
|
||||
// A date such as 20250929 ends the version. ok is false when no version is found.
|
||||
func modelVersion(model, prefix string) (major, minor int, ok bool) {
|
||||
_, name, found := strings.Cut(model, prefix)
|
||||
if !found {
|
||||
return 0, 0, false
|
||||
}
|
||||
version := make([]int, 0, 2)
|
||||
for part := range strings.FieldsFuncSeq(name, func(r rune) bool { return r == '-' || r == '.' }) {
|
||||
digits := len(part) - len(strings.TrimLeft(part, "0123456789"))
|
||||
if digits == 0 || digits > 2 {
|
||||
if len(version) > 0 {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
n, _ := strconv.Atoi(part[:digits])
|
||||
version = append(version, n)
|
||||
if len(version) == 2 || digits < len(part) {
|
||||
break
|
||||
}
|
||||
}
|
||||
switch len(version) {
|
||||
case 0:
|
||||
return 0, 0, false
|
||||
case 1:
|
||||
return version[0], 0, true
|
||||
default:
|
||||
return version[0], version[1], true
|
||||
}
|
||||
}
|
||||
|
||||
// weights are the estimated tokens per text feature.
|
||||
type weights struct {
|
||||
Word float64 // Latin word (each run of letters)
|
||||
Number float64 // each run of digits
|
||||
CJK float64 // each Chinese, Japanese, or Korean character
|
||||
Symbol float64 // each ordinary punctuation mark
|
||||
MathSymbol float64 // each math symbol (∑, ∫, ∂, √, ...)
|
||||
URLDelim float64 // each URL delimiter (/ : ? & = ; # %)
|
||||
AtSign float64 // each @, which splits words
|
||||
Emoji float64 // each emoji
|
||||
Newline float64 // each newline or tab
|
||||
Space float64 // each other whitespace character
|
||||
}
|
||||
|
||||
// Word, Number, CJK, Symbol, URLDelim, Newline, and Space are calibrated
|
||||
// against upstream counts on agent traffic. MathSymbol, AtSign, and Emoji make
|
||||
// up under 0.3% of that traffic and keep their earlier values. FamilyUnknown
|
||||
// keeps the earlier OpenAI weights; no other vendor was calibrated.
|
||||
var familyWeights = map[Family]weights{
|
||||
FamilyGemini: {
|
||||
Word: 0.89, Number: 2.42, CJK: 0.93, Symbol: 0.65, MathSymbol: 1.05, URLDelim: 1.3, AtSign: 2.5, Emoji: 1.08, Newline: 1.28, Space: 0,
|
||||
},
|
||||
FamilyClaude: {
|
||||
Word: 1.24, Number: 1.53, CJK: 1.38, Symbol: 0.66, MathSymbol: 4.52, URLDelim: 0.74, AtSign: 2.82, Emoji: 2.6, Newline: 1.19, Space: 0.14,
|
||||
},
|
||||
FamilyClaude47: {
|
||||
Word: 1.52, Number: 1.77, CJK: 1.4, Symbol: 0.89, MathSymbol: 4.52, URLDelim: 1.45, AtSign: 2.82, Emoji: 2.6, Newline: 1.38, Space: 0.21,
|
||||
},
|
||||
FamilyOpenAI: {
|
||||
Word: 0.75, Number: 1.86, CJK: 0.96, Symbol: 0.6, MathSymbol: 2.68, URLDelim: 0.77, AtSign: 2.0, Emoji: 2.12, Newline: 0.95, Space: 0.17,
|
||||
},
|
||||
FamilyUnknown: {
|
||||
Word: 1.02, Number: 1.55, CJK: 0.85, Symbol: 0.4, MathSymbol: 2.68, URLDelim: 1.0, AtSign: 2.0, Emoji: 2.12, Newline: 0.5, Space: 0.42,
|
||||
},
|
||||
}
|
||||
|
||||
// Estimate returns the estimated token count of text for model, using the
|
||||
// weights of the model's family. It never runs a tokenizer; use Count for an
|
||||
// exact count where one is available.
|
||||
func Estimate(model, text string) int {
|
||||
if text == "" {
|
||||
return 0
|
||||
}
|
||||
return EstimateFamily(ModelFamily(model), text)
|
||||
}
|
||||
|
||||
// EstimateFamily returns the estimated token count of text for a tokenizer
|
||||
// family. Unrecognized families use the FamilyUnknown weights.
|
||||
func EstimateFamily(family Family, text string) int {
|
||||
w, ok := familyWeights[family]
|
||||
if !ok {
|
||||
w = familyWeights[FamilyUnknown]
|
||||
}
|
||||
var count float64
|
||||
|
||||
type wordType int
|
||||
const (
|
||||
none wordType = iota
|
||||
latin
|
||||
number
|
||||
)
|
||||
current := none
|
||||
|
||||
for _, r := range text {
|
||||
if unicode.IsSpace(r) {
|
||||
current = none
|
||||
if r == '\n' || r == '\t' {
|
||||
count += w.Newline
|
||||
} else {
|
||||
count += w.Space
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if isCJK(r) {
|
||||
current = none
|
||||
count += w.CJK
|
||||
continue
|
||||
}
|
||||
|
||||
if isEmoji(r) {
|
||||
current = none
|
||||
count += w.Emoji
|
||||
continue
|
||||
}
|
||||
|
||||
// A run of letters or digits counts once; switching between letters
|
||||
// and digits starts a new run.
|
||||
if unicode.IsLetter(r) || unicode.IsNumber(r) {
|
||||
next := latin
|
||||
if unicode.IsNumber(r) {
|
||||
next = number
|
||||
}
|
||||
if current != next {
|
||||
if next == number {
|
||||
count += w.Number
|
||||
} else {
|
||||
count += w.Word
|
||||
}
|
||||
current = next
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
current = none
|
||||
switch {
|
||||
case isMathSymbol(r):
|
||||
count += w.MathSymbol
|
||||
case r == '@':
|
||||
count += w.AtSign
|
||||
case isURLDelim(r):
|
||||
count += w.URLDelim
|
||||
default:
|
||||
count += w.Symbol
|
||||
}
|
||||
}
|
||||
|
||||
return int(math.Ceil(count))
|
||||
}
|
||||
|
||||
func isCJK(r rune) bool {
|
||||
return unicode.Is(unicode.Han, r) ||
|
||||
(r >= 0x3040 && r <= 0x30FF) || // Japanese kana
|
||||
(r >= 0xAC00 && r <= 0xD7A3) // Korean Hangul syllables
|
||||
}
|
||||
|
||||
func isEmoji(r rune) bool {
|
||||
return (r >= 0x1F300 && r <= 0x1F9FF) || // pictographs, emoticons, supplemental symbols
|
||||
(r >= 0x2600 && r <= 0x26FF) || // miscellaneous symbols
|
||||
(r >= 0x2700 && r <= 0x27BF) || // dingbats
|
||||
(r >= 0x1FA00 && r <= 0x1FAFF) // symbols and pictographs extended-A
|
||||
}
|
||||
|
||||
func isMathSymbol(r rune) bool {
|
||||
if strings.ContainsRune("∑∫∂√∞≤≥≠≈±×÷∈∉∋∌⊂⊃⊆⊇∪∩∧∨¬∀∃∄∅∆∇∝∟∠∡∢°′″‴⁺⁻⁼⁽⁾ⁿ₀₁₂₃₄₅₆₇₈₉₊₋₌₍₎²³¹⁴⁵⁶⁷⁸⁹⁰", r) {
|
||||
return true
|
||||
}
|
||||
return (r >= 0x2200 && r <= 0x22FF) || // mathematical operators
|
||||
(r >= 0x2A00 && r <= 0x2AFF) || // supplemental mathematical operators
|
||||
(r >= 0x1D400 && r <= 0x1D7FF) // mathematical alphanumeric symbols
|
||||
}
|
||||
|
||||
func isURLDelim(r rune) bool {
|
||||
return strings.ContainsRune("/:?&=;#%", r)
|
||||
}
|
||||
Reference in new issue
Block a user