[ { "name": "gpt-6-astra", "samples": [ { "text": "43", "expected_count": 0 }, { "text": "43", "expected_count": 1 }, { "text": "43", "expected_count": 2 }, { "text": "43", "expected_count": 3 }, { "text": "43", "expected_count": 4 }, { "text": "43", "expected_count": 5 }, { "text": "43", "expected_count": 6 }, { "text": "43", "expected_count": 7 }, { "text": "43", "expected_count": 8 }, { "text": "43", "expected_count": 9 }, { "text": "43", "expected_count": 10 }, { "text": "七筒", "expected_count": 11 }, { "text": "七筒", "expected_count": 12 }, { "text": "七筒", "expected_count": 13 }, { "text": "七筒", "expected_count": 14 }, { "text": "七筒", "expected_count": 15 }, { "text": "七筒", "expected_count": 16 }, { "text": "七筒", "expected_count": 17 }, { "text": "七筒", "expected_count": 18 }, { "text": "七筒", "expected_count": 19 }, { "text": "七筒", "expected_count": 20 }, { "text": "七筒", "expected_count": 21 }, { "text": "哗啦", "expected_count": 22 }, { "text": "哗啦", "expected_count": 23 }, { "text": "哗啦", "expected_count": 24 }, { "text": "哗啦", "expected_count": 25 }, { "text": "哗啦", "expected_count": 26 }, { "text": "哗啦", "expected_count": 27 }, { "text": "哗啦", "expected_count": 28 }, { "text": "哗啦", "expected_count": 29 }, { "text": "哗啦", "expected_count": 30 }, { "text": "哗啦", "expected_count": 31 } ], "expected": { "scoring_version": "meow-fingerprint-v3-predictive", "verdict": "match", "color": "green", "model": "gpt-6-astra", "claimed_model": "gpt-6-astra", "reasons": [], "matches": { "gpt-6-astra": 0.7198267203992741, "gpt-6-sol": 0.12720127373863774, "gpt-5.6-terra": 0.18239419374550428, "gpt-6-luna": 0.17098911498775238, "other_known_external": 0.2801732796007259 }, "scores": { "gpt-6-astra": -0.33326644869550826, "gpt-6-sol": -2.2592007593183085, "gpt-5.6-terra": -1.833476525105067, "gpt-6-luna": -1.9118998346026106, "other_known_external": -1.2768686981548583 }, "thresholds": { "gpt-6-astra": 0.56729343469469, "gpt-6-sol": 0.5, "gpt-5.6-terra": 0.5, "gpt-6-luna": 0.5, "other_known_external": 0.5610207135285652 }, "cells": { "gpt__screen023": { "planned": 11, "minimum": 7, "completed": 11, "valid": 11, "counts": { "43": 11 }, "weight": 0.34375 }, "gpt__screen106": { "planned": 11, "minimum": 7, "completed": 11, "valid": 11, "counts": { "七筒": 11 }, "weight": 0.34375 }, "gpt__screen114": { "planned": 10, "minimum": 6, "completed": 10, "valid": 10, "counts": { "哗啦": 10 }, "weight": 0.3125 } }, "families": {}, "valid_samples": 32, "planned_samples": 32, "sample_policy": { "version": "60-percent-v1", "overall_ratio": 0.6, "per_cell_ratio": 0.6 }, "partial_samples": false, "quality_status": "sufficient" } }, { "name": "gpt-6-sol", "samples": [ { "text": "47", "expected_count": 0 }, { "text": "47", "expected_count": 1 }, { "text": "47", "expected_count": 2 }, { "text": "47", "expected_count": 3 }, { "text": "47", "expected_count": 4 }, { "text": "47", "expected_count": 5 }, { "text": "47", "expected_count": 6 }, { "text": "47", "expected_count": 7 }, { "text": "47", "expected_count": 8 }, { "text": "47", "expected_count": 9 }, { "text": "47", "expected_count": 10 }, { "text": "一筒", "expected_count": 11 }, { "text": "一筒", "expected_count": 12 }, { "text": "一筒", "expected_count": 13 }, { "text": "一筒", "expected_count": 14 }, { "text": "一筒", "expected_count": 15 }, { "text": "一筒", "expected_count": 16 }, { "text": "一筒", "expected_count": 17 }, { "text": "一筒", "expected_count": 18 }, { "text": "一筒", "expected_count": 19 }, { "text": "一筒", "expected_count": 20 }, { "text": "一筒", "expected_count": 21 }, { "text": "喵呜", "expected_count": 22 }, { "text": "喵呜", "expected_count": 23 }, { "text": "喵呜", "expected_count": 24 }, { "text": "喵呜", "expected_count": 25 }, { "text": "喵呜", "expected_count": 26 }, { "text": "喵呜", "expected_count": 27 }, { "text": "喵呜", "expected_count": 28 }, { "text": "喵呜", "expected_count": 29 }, { "text": "喵呜", "expected_count": 30 }, { "text": "喵呜", "expected_count": 31 } ], "expected": { "scoring_version": "meow-fingerprint-v3-predictive", "verdict": "mismatch", "color": "red", "model": "gpt-6-sol", "claimed_model": "gpt-6-astra", "reasons": [], "matches": { "gpt-6-astra": 0.16433376702935898, "gpt-6-sol": 0.678802755307823, "gpt-5.6-terra": 0.2284423124177911, "gpt-6-luna": 0.21009386636554536, "other_known_external": 0.32119724469217703 }, "scores": { "gpt-6-astra": -2.3324679246962168, "gpt-6-sol": -0.7061381586832349, "gpt-5.6-terra": -1.9232658845729893, "gpt-6-luna": -2.0304978653570918, "other_known_external": -1.4544133472544305 }, "thresholds": { "gpt-6-astra": 0.56729343469469, "gpt-6-sol": 0.5, "gpt-5.6-terra": 0.5, "gpt-6-luna": 0.5, "other_known_external": 0.5610207135285652 }, "cells": { "gpt__screen023": { "planned": 11, "minimum": 7, "completed": 11, "valid": 11, "counts": { "47": 11 }, "weight": 0.34375 }, "gpt__screen106": { "planned": 11, "minimum": 7, "completed": 11, "valid": 11, "counts": { "一筒": 11 }, "weight": 0.34375 }, "gpt__screen114": { "planned": 10, "minimum": 6, "completed": 10, "valid": 10, "counts": { "喵呜": 10 }, "weight": 0.3125 } }, "families": {}, "valid_samples": 32, "planned_samples": 32, "sample_policy": { "version": "60-percent-v1", "overall_ratio": 0.6, "per_cell_ratio": 0.6 }, "partial_samples": false, "quality_status": "sufficient" } }, { "name": "gpt-5.6-terra", "samples": [ { "text": "47", "expected_count": 0 }, { "text": "47", "expected_count": 1 }, { "text": "47", "expected_count": 2 }, { "text": "47", "expected_count": 3 }, { "text": "47", "expected_count": 4 }, { "text": "47", "expected_count": 5 }, { "text": "47", "expected_count": 6 }, { "text": "47", "expected_count": 7 }, { "text": "47", "expected_count": 8 }, { "text": "47", "expected_count": 9 }, { "text": "47", "expected_count": 10 }, { "text": "五万", "expected_count": 11 }, { "text": "五万", "expected_count": 12 }, { "text": "五万", "expected_count": 13 }, { "text": "五万", "expected_count": 14 }, { "text": "五万", "expected_count": 15 }, { "text": "五万", "expected_count": 16 }, { "text": "五万", "expected_count": 17 }, { "text": "五万", "expected_count": 18 }, { "text": "五万", "expected_count": 19 }, { "text": "五万", "expected_count": 20 }, { "text": "五万", "expected_count": 21 }, { "text": "啪", "expected_count": 22 }, { "text": "啪", "expected_count": 23 }, { "text": "啪", "expected_count": 24 }, { "text": "啪", "expected_count": 25 }, { "text": "啪", "expected_count": 26 }, { "text": "啪", "expected_count": 27 }, { "text": "啪", "expected_count": 28 }, { "text": "啪", "expected_count": 29 }, { "text": "啪", "expected_count": 30 }, { "text": "啪", "expected_count": 31 } ], "expected": { "scoring_version": "meow-fingerprint-v3-predictive", "verdict": "mismatch", "color": "red", "model": "gpt-5.6-terra", "claimed_model": "gpt-6-astra", "reasons": [], "matches": { "gpt-6-astra": 0.16949642783135352, "gpt-6-sol": 0.262768570572442, "gpt-5.6-terra": 0.6996240485385832, "gpt-6-luna": 0.16949642783135352, "other_known_external": 0.3003759514614168 }, "scores": { "gpt-6-astra": -2.3324679246962168, "gpt-6-sol": -1.7748957196997543, "gpt-5.6-terra": -0.7432675462827238, "gpt-6-luna": -2.3324679246962168, "other_known_external": -1.5887758019039786 }, "thresholds": { "gpt-6-astra": 0.56729343469469, "gpt-6-sol": 0.5, "gpt-5.6-terra": 0.5, "gpt-6-luna": 0.5, "other_known_external": 0.5610207135285652 }, "cells": { "gpt__screen023": { "planned": 11, "minimum": 7, "completed": 11, "valid": 11, "counts": { "47": 11 }, "weight": 0.34375 }, "gpt__screen106": { "planned": 11, "minimum": 7, "completed": 11, "valid": 11, "counts": { "五万": 11 }, "weight": 0.34375 }, "gpt__screen114": { "planned": 10, "minimum": 6, "completed": 10, "valid": 10, "counts": { "啪": 10 }, "weight": 0.3125 } }, "families": {}, "valid_samples": 32, "planned_samples": 32, "sample_policy": { "version": "60-percent-v1", "overall_ratio": 0.6, "per_cell_ratio": 0.6 }, "partial_samples": false, "quality_status": "sufficient" } }, { "name": "gpt-6-luna", "samples": [ { "text": "42", "expected_count": 0 }, { "text": "42", "expected_count": 1 }, { "text": "42", "expected_count": 2 }, { "text": "42", "expected_count": 3 }, { "text": "42", "expected_count": 4 }, { "text": "42", "expected_count": 5 }, { "text": "42", "expected_count": 6 }, { "text": "42", "expected_count": 7 }, { "text": "42", "expected_count": 8 }, { "text": "42", "expected_count": 9 }, { "text": "42", "expected_count": 10 }, { "text": "红中", "expected_count": 11 }, { "text": "红中", "expected_count": 12 }, { "text": "红中", "expected_count": 13 }, { "text": "红中", "expected_count": 14 }, { "text": "红中", "expected_count": 15 }, { "text": "红中", "expected_count": 16 }, { "text": "红中", "expected_count": 17 }, { "text": "红中", "expected_count": 18 }, { "text": "红中", "expected_count": 19 }, { "text": "红中", "expected_count": 20 }, { "text": "红中", "expected_count": 21 }, { "text": "喵喵", "expected_count": 22 }, { "text": "喵喵", "expected_count": 23 }, { "text": "喵喵", "expected_count": 24 }, { "text": "喵喵", "expected_count": 25 }, { "text": "喵喵", "expected_count": 26 }, { "text": "喵喵", "expected_count": 27 }, { "text": "喵喵", "expected_count": 28 }, { "text": "喵喵", "expected_count": 29 }, { "text": "喵喵", "expected_count": 30 }, { "text": "喵喵", "expected_count": 31 } ], "expected": { "scoring_version": "meow-fingerprint-v3-predictive", "verdict": "mismatch", "color": "red", "model": "gpt-6-luna", "claimed_model": "gpt-6-astra", "reasons": [], "matches": { "gpt-6-astra": 0.11034768854830239, "gpt-6-sol": 0.18262928514091403, "gpt-5.6-terra": 0.17249846553639775, "gpt-6-luna": 0.6626074593360759, "other_known_external": 0.3373925406639241 }, "scores": { "gpt-6-astra": -2.463864569052184, "gpt-6-sol": -1.8753044392335134, "gpt-5.6-terra": -1.9446926504739512, "gpt-6-luna": -0.37667002987845055, "other_known_external": -1.0516057157764647 }, "thresholds": { "gpt-6-astra": 0.56729343469469, "gpt-6-sol": 0.5, "gpt-5.6-terra": 0.5, "gpt-6-luna": 0.5, "other_known_external": 0.5610207135285652 }, "cells": { "gpt__screen023": { "planned": 11, "minimum": 7, "completed": 11, "valid": 11, "counts": { "42": 11 }, "weight": 0.34375 }, "gpt__screen106": { "planned": 11, "minimum": 7, "completed": 11, "valid": 11, "counts": { "红中": 11 }, "weight": 0.34375 }, "gpt__screen114": { "planned": 10, "minimum": 6, "completed": 10, "valid": 10, "counts": { "喵喵": 10 }, "weight": 0.3125 } }, "families": {}, "valid_samples": 32, "planned_samples": 32, "sample_policy": { "version": "60-percent-v1", "overall_ratio": 0.6, "per_cell_ratio": 0.6 }, "partial_samples": false, "quality_status": "sufficient" } }, { "name": "partial", "samples": [ { "text": "43", "expected_count": 0 }, { "text": "43", "expected_count": 1 }, { "text": "43", "expected_count": 2 }, { "text": "43", "expected_count": 3 }, { "text": "43", "expected_count": 4 }, { "text": "43", "expected_count": 5 }, { "text": "43", "expected_count": 6 }, { "text": "43", "expected_count": 7 } ], "expected": { "scoring_version": "meow-fingerprint-v3-predictive", "verdict": "insufficient", "color": "yellow", "model": null, "claimed_model": "gpt-6-astra", "reasons": [ "samples_incomplete" ], "matches": { "gpt-6-astra": 0.6637577509332276, "gpt-6-sol": 0.13628981670052215, "gpt-5.6-terra": 0.2236014336853945, "gpt-6-luna": 0.10869590616949418, "other_known_external": 0.3362422490667724 }, "scores": { "gpt-6-astra": -0.05486073970997829, "gpt-6-sol": -1.9013143927307334, "gpt-5.6-terra": -1.2996615913309375, "gpt-6-luna": -2.1589922722528767, "other_known_external": -0.734946110138968 }, "thresholds": { "gpt-6-astra": 0.56729343469469, "gpt-6-sol": 0.5, "gpt-5.6-terra": 0.5, "gpt-6-luna": 0.5, "other_known_external": 0.5610207135285652 }, "cells": { "gpt__screen023": { "planned": 11, "minimum": 7, "completed": 8, "valid": 8, "counts": { "43": 8 }, "weight": 1.0 }, "gpt__screen106": { "planned": 11, "minimum": 7, "completed": 0, "valid": 0, "counts": {}, "weight": 0.0 }, "gpt__screen114": { "planned": 10, "minimum": 6, "completed": 0, "valid": 0, "counts": {}, "weight": 0.0 } }, "families": {}, "valid_samples": 8, "planned_samples": 32, "sample_policy": { "version": "60-percent-v1", "overall_ratio": 0.6, "per_cell_ratio": 0.6 }, "partial_samples": true, "quality_status": "insufficient_valid_samples" } }, { "name": "empty", "samples": [], "expected": { "scoring_version": "meow-fingerprint-v3-predictive", "verdict": "insufficient", "color": "yellow", "model": null, "claimed_model": "gpt-6-astra", "reasons": [ "no_valid_samples", "samples_incomplete" ], "matches": { "gpt-6-astra": 0.5, "gpt-6-sol": 0.5, "gpt-5.6-terra": 0.5, "gpt-6-luna": 0.5, "other_known_external": 0.5 }, "scores": { "gpt-6-astra": 0.0, "gpt-6-sol": 0.0, "gpt-5.6-terra": 0.0, "gpt-6-luna": 0.0, "other_known_external": 0.0 }, "thresholds": { "gpt-6-astra": 0.56729343469469, "gpt-6-sol": 0.5, "gpt-5.6-terra": 0.5, "gpt-6-luna": 0.5, "other_known_external": 0.5610207135285652 }, "cells": { "gpt__screen023": { "planned": 11, "minimum": 7, "completed": 0, "valid": 0, "counts": {}, "weight": 0.0 }, "gpt__screen106": { "planned": 11, "minimum": 7, "completed": 0, "valid": 0, "counts": {}, "weight": 0.0 }, "gpt__screen114": { "planned": 10, "minimum": 6, "completed": 0, "valid": 0, "counts": {}, "weight": 0.0 } }, "families": {}, "valid_samples": 0, "planned_samples": 32, "sample_policy": { "version": "60-percent-v1", "overall_ratio": 0.6, "per_cell_ratio": 0.6 }, "partial_samples": true, "quality_status": "insufficient_valid_samples" } }, { "name": "unseen", "samples": [ { "text": "UNSEEN fixture answer", "expected_count": 0 }, { "text": "UNSEEN fixture answer", "expected_count": 1 }, { "text": "UNSEEN fixture answer", "expected_count": 2 }, { "text": "UNSEEN fixture answer", "expected_count": 3 }, { "text": "UNSEEN fixture answer", "expected_count": 4 }, { "text": "UNSEEN fixture answer", "expected_count": 5 }, { "text": "UNSEEN fixture answer", "expected_count": 6 }, { "text": "UNSEEN fixture answer", "expected_count": 7 }, { "text": "UNSEEN fixture answer", "expected_count": 8 }, { "text": "UNSEEN fixture answer", "expected_count": 9 }, { "text": "UNSEEN fixture answer", "expected_count": 10 }, { "text": "UNSEEN fixture answer", "expected_count": 11 }, { "text": "UNSEEN fixture answer", "expected_count": 12 }, { "text": "UNSEEN fixture answer", "expected_count": 13 }, { "text": "UNSEEN fixture answer", "expected_count": 14 }, { "text": "UNSEEN fixture answer", "expected_count": 15 }, { "text": "UNSEEN fixture answer", "expected_count": 16 }, { "text": "UNSEEN fixture answer", "expected_count": 17 }, { "text": "UNSEEN fixture answer", "expected_count": 18 }, { "text": "UNSEEN fixture answer", "expected_count": 19 }, { "text": "UNSEEN fixture answer", "expected_count": 20 }, { "text": "UNSEEN fixture answer", "expected_count": 21 }, { "text": "UNSEEN fixture answer", "expected_count": 22 }, { "text": "UNSEEN fixture answer", "expected_count": 23 }, { "text": "UNSEEN fixture answer", "expected_count": 24 }, { "text": "UNSEEN fixture answer", "expected_count": 25 }, { "text": "UNSEEN fixture answer", "expected_count": 26 }, { "text": "UNSEEN fixture answer", "expected_count": 27 }, { "text": "UNSEEN fixture answer", "expected_count": 28 }, { "text": "UNSEEN fixture answer", "expected_count": 29 }, { "text": "UNSEEN fixture answer", "expected_count": 30 }, { "text": "UNSEEN fixture answer", "expected_count": 31 } ], "expected": { "scoring_version": "meow-fingerprint-v3-predictive", "verdict": "mismatch", "color": "red", "model": "other_known_external", "claimed_model": "gpt-6-astra", "reasons": [], "matches": { "gpt-6-astra": 0.3905375565158004, "gpt-6-sol": 0.39053755651580024, "gpt-5.6-terra": 0.3905375565158004, "gpt-6-luna": 0.39053755651580024, "other_known_external": 0.6094624434841995 }, "scores": { "gpt-6-astra": -2.463864569052184, "gpt-6-sol": -2.463864569052185, "gpt-5.6-terra": -2.463864569052184, "gpt-6-luna": -2.463864569052185, "other_known_external": -2.0188113807068433 }, "thresholds": { "gpt-6-astra": 0.56729343469469, "gpt-6-sol": 0.5, "gpt-5.6-terra": 0.5, "gpt-6-luna": 0.5, "other_known_external": 0.5610207135285652 }, "cells": { "gpt__screen023": { "planned": 11, "minimum": 7, "completed": 11, "valid": 11, "counts": { "__UNSEEN_IN_TRAINING__": 11 }, "weight": 0.34375 }, "gpt__screen106": { "planned": 11, "minimum": 7, "completed": 11, "valid": 11, "counts": { "__UNSEEN_IN_TRAINING__": 11 }, "weight": 0.34375 }, "gpt__screen114": { "planned": 10, "minimum": 6, "completed": 10, "valid": 10, "counts": { "__UNSEEN_IN_TRAINING__": 10 }, "weight": 0.3125 } }, "families": {}, "valid_samples": 32, "planned_samples": 32, "sample_policy": { "version": "60-percent-v1", "overall_ratio": 0.6, "per_cell_ratio": 0.6 }, "partial_samples": false, "quality_status": "sufficient" } } ]