1200 lines
25 KiB
JSON
1200 lines
25 KiB
JSON
[
|
|
{
|
|
"name": "gpt-6-astra",
|
|
"samples": [
|
|
{
|
|
"text": "43",
|
|
"expected_count": 0
|
|
},
|
|
{
|
|
"text": "43",
|
|
"expected_count": 1
|
|
},
|
|
{
|
|
"text": "43",
|
|
"expected_count": 2
|
|
},
|
|
{
|
|
"text": "43",
|
|
"expected_count": 3
|
|
},
|
|
{
|
|
"text": "43",
|
|
"expected_count": 4
|
|
},
|
|
{
|
|
"text": "43",
|
|
"expected_count": 5
|
|
},
|
|
{
|
|
"text": "43",
|
|
"expected_count": 6
|
|
},
|
|
{
|
|
"text": "43",
|
|
"expected_count": 7
|
|
},
|
|
{
|
|
"text": "43",
|
|
"expected_count": 8
|
|
},
|
|
{
|
|
"text": "43",
|
|
"expected_count": 9
|
|
},
|
|
{
|
|
"text": "43",
|
|
"expected_count": 10
|
|
},
|
|
{
|
|
"text": "七筒",
|
|
"expected_count": 11
|
|
},
|
|
{
|
|
"text": "七筒",
|
|
"expected_count": 12
|
|
},
|
|
{
|
|
"text": "七筒",
|
|
"expected_count": 13
|
|
},
|
|
{
|
|
"text": "七筒",
|
|
"expected_count": 14
|
|
},
|
|
{
|
|
"text": "七筒",
|
|
"expected_count": 15
|
|
},
|
|
{
|
|
"text": "七筒",
|
|
"expected_count": 16
|
|
},
|
|
{
|
|
"text": "七筒",
|
|
"expected_count": 17
|
|
},
|
|
{
|
|
"text": "七筒",
|
|
"expected_count": 18
|
|
},
|
|
{
|
|
"text": "七筒",
|
|
"expected_count": 19
|
|
},
|
|
{
|
|
"text": "七筒",
|
|
"expected_count": 20
|
|
},
|
|
{
|
|
"text": "七筒",
|
|
"expected_count": 21
|
|
},
|
|
{
|
|
"text": "哗啦",
|
|
"expected_count": 22
|
|
},
|
|
{
|
|
"text": "哗啦",
|
|
"expected_count": 23
|
|
},
|
|
{
|
|
"text": "哗啦",
|
|
"expected_count": 24
|
|
},
|
|
{
|
|
"text": "哗啦",
|
|
"expected_count": 25
|
|
},
|
|
{
|
|
"text": "哗啦",
|
|
"expected_count": 26
|
|
},
|
|
{
|
|
"text": "哗啦",
|
|
"expected_count": 27
|
|
},
|
|
{
|
|
"text": "哗啦",
|
|
"expected_count": 28
|
|
},
|
|
{
|
|
"text": "哗啦",
|
|
"expected_count": 29
|
|
},
|
|
{
|
|
"text": "哗啦",
|
|
"expected_count": 30
|
|
},
|
|
{
|
|
"text": "哗啦",
|
|
"expected_count": 31
|
|
}
|
|
],
|
|
"expected": {
|
|
"scoring_version": "meow-fingerprint-v3-predictive",
|
|
"verdict": "match",
|
|
"color": "green",
|
|
"model": "gpt-6-astra",
|
|
"claimed_model": "gpt-6-astra",
|
|
"reasons": [],
|
|
"matches": {
|
|
"gpt-6-astra": 0.7198267203992741,
|
|
"gpt-6-sol": 0.12720127373863774,
|
|
"gpt-5.6-terra": 0.18239419374550428,
|
|
"gpt-6-luna": 0.17098911498775238,
|
|
"other_known_external": 0.2801732796007259
|
|
},
|
|
"scores": {
|
|
"gpt-6-astra": -0.33326644869550826,
|
|
"gpt-6-sol": -2.2592007593183085,
|
|
"gpt-5.6-terra": -1.833476525105067,
|
|
"gpt-6-luna": -1.9118998346026106,
|
|
"other_known_external": -1.2768686981548583
|
|
},
|
|
"thresholds": {
|
|
"gpt-6-astra": 0.56729343469469,
|
|
"gpt-6-sol": 0.5,
|
|
"gpt-5.6-terra": 0.5,
|
|
"gpt-6-luna": 0.5,
|
|
"other_known_external": 0.5610207135285652
|
|
},
|
|
"cells": {
|
|
"gpt__screen023": {
|
|
"planned": 11,
|
|
"minimum": 7,
|
|
"completed": 11,
|
|
"valid": 11,
|
|
"counts": {
|
|
"43": 11
|
|
},
|
|
"weight": 0.34375
|
|
},
|
|
"gpt__screen106": {
|
|
"planned": 11,
|
|
"minimum": 7,
|
|
"completed": 11,
|
|
"valid": 11,
|
|
"counts": {
|
|
"七筒": 11
|
|
},
|
|
"weight": 0.34375
|
|
},
|
|
"gpt__screen114": {
|
|
"planned": 10,
|
|
"minimum": 6,
|
|
"completed": 10,
|
|
"valid": 10,
|
|
"counts": {
|
|
"哗啦": 10
|
|
},
|
|
"weight": 0.3125
|
|
}
|
|
},
|
|
"families": {},
|
|
"valid_samples": 32,
|
|
"planned_samples": 32,
|
|
"sample_policy": {
|
|
"version": "60-percent-v1",
|
|
"overall_ratio": 0.6,
|
|
"per_cell_ratio": 0.6
|
|
},
|
|
"partial_samples": false,
|
|
"quality_status": "sufficient"
|
|
}
|
|
},
|
|
{
|
|
"name": "gpt-6-sol",
|
|
"samples": [
|
|
{
|
|
"text": "47",
|
|
"expected_count": 0
|
|
},
|
|
{
|
|
"text": "47",
|
|
"expected_count": 1
|
|
},
|
|
{
|
|
"text": "47",
|
|
"expected_count": 2
|
|
},
|
|
{
|
|
"text": "47",
|
|
"expected_count": 3
|
|
},
|
|
{
|
|
"text": "47",
|
|
"expected_count": 4
|
|
},
|
|
{
|
|
"text": "47",
|
|
"expected_count": 5
|
|
},
|
|
{
|
|
"text": "47",
|
|
"expected_count": 6
|
|
},
|
|
{
|
|
"text": "47",
|
|
"expected_count": 7
|
|
},
|
|
{
|
|
"text": "47",
|
|
"expected_count": 8
|
|
},
|
|
{
|
|
"text": "47",
|
|
"expected_count": 9
|
|
},
|
|
{
|
|
"text": "47",
|
|
"expected_count": 10
|
|
},
|
|
{
|
|
"text": "一筒",
|
|
"expected_count": 11
|
|
},
|
|
{
|
|
"text": "一筒",
|
|
"expected_count": 12
|
|
},
|
|
{
|
|
"text": "一筒",
|
|
"expected_count": 13
|
|
},
|
|
{
|
|
"text": "一筒",
|
|
"expected_count": 14
|
|
},
|
|
{
|
|
"text": "一筒",
|
|
"expected_count": 15
|
|
},
|
|
{
|
|
"text": "一筒",
|
|
"expected_count": 16
|
|
},
|
|
{
|
|
"text": "一筒",
|
|
"expected_count": 17
|
|
},
|
|
{
|
|
"text": "一筒",
|
|
"expected_count": 18
|
|
},
|
|
{
|
|
"text": "一筒",
|
|
"expected_count": 19
|
|
},
|
|
{
|
|
"text": "一筒",
|
|
"expected_count": 20
|
|
},
|
|
{
|
|
"text": "一筒",
|
|
"expected_count": 21
|
|
},
|
|
{
|
|
"text": "喵呜",
|
|
"expected_count": 22
|
|
},
|
|
{
|
|
"text": "喵呜",
|
|
"expected_count": 23
|
|
},
|
|
{
|
|
"text": "喵呜",
|
|
"expected_count": 24
|
|
},
|
|
{
|
|
"text": "喵呜",
|
|
"expected_count": 25
|
|
},
|
|
{
|
|
"text": "喵呜",
|
|
"expected_count": 26
|
|
},
|
|
{
|
|
"text": "喵呜",
|
|
"expected_count": 27
|
|
},
|
|
{
|
|
"text": "喵呜",
|
|
"expected_count": 28
|
|
},
|
|
{
|
|
"text": "喵呜",
|
|
"expected_count": 29
|
|
},
|
|
{
|
|
"text": "喵呜",
|
|
"expected_count": 30
|
|
},
|
|
{
|
|
"text": "喵呜",
|
|
"expected_count": 31
|
|
}
|
|
],
|
|
"expected": {
|
|
"scoring_version": "meow-fingerprint-v3-predictive",
|
|
"verdict": "mismatch",
|
|
"color": "red",
|
|
"model": "gpt-6-sol",
|
|
"claimed_model": "gpt-6-astra",
|
|
"reasons": [],
|
|
"matches": {
|
|
"gpt-6-astra": 0.16433376702935898,
|
|
"gpt-6-sol": 0.678802755307823,
|
|
"gpt-5.6-terra": 0.2284423124177911,
|
|
"gpt-6-luna": 0.21009386636554536,
|
|
"other_known_external": 0.32119724469217703
|
|
},
|
|
"scores": {
|
|
"gpt-6-astra": -2.3324679246962168,
|
|
"gpt-6-sol": -0.7061381586832349,
|
|
"gpt-5.6-terra": -1.9232658845729893,
|
|
"gpt-6-luna": -2.0304978653570918,
|
|
"other_known_external": -1.4544133472544305
|
|
},
|
|
"thresholds": {
|
|
"gpt-6-astra": 0.56729343469469,
|
|
"gpt-6-sol": 0.5,
|
|
"gpt-5.6-terra": 0.5,
|
|
"gpt-6-luna": 0.5,
|
|
"other_known_external": 0.5610207135285652
|
|
},
|
|
"cells": {
|
|
"gpt__screen023": {
|
|
"planned": 11,
|
|
"minimum": 7,
|
|
"completed": 11,
|
|
"valid": 11,
|
|
"counts": {
|
|
"47": 11
|
|
},
|
|
"weight": 0.34375
|
|
},
|
|
"gpt__screen106": {
|
|
"planned": 11,
|
|
"minimum": 7,
|
|
"completed": 11,
|
|
"valid": 11,
|
|
"counts": {
|
|
"一筒": 11
|
|
},
|
|
"weight": 0.34375
|
|
},
|
|
"gpt__screen114": {
|
|
"planned": 10,
|
|
"minimum": 6,
|
|
"completed": 10,
|
|
"valid": 10,
|
|
"counts": {
|
|
"喵呜": 10
|
|
},
|
|
"weight": 0.3125
|
|
}
|
|
},
|
|
"families": {},
|
|
"valid_samples": 32,
|
|
"planned_samples": 32,
|
|
"sample_policy": {
|
|
"version": "60-percent-v1",
|
|
"overall_ratio": 0.6,
|
|
"per_cell_ratio": 0.6
|
|
},
|
|
"partial_samples": false,
|
|
"quality_status": "sufficient"
|
|
}
|
|
},
|
|
{
|
|
"name": "gpt-5.6-terra",
|
|
"samples": [
|
|
{
|
|
"text": "47",
|
|
"expected_count": 0
|
|
},
|
|
{
|
|
"text": "47",
|
|
"expected_count": 1
|
|
},
|
|
{
|
|
"text": "47",
|
|
"expected_count": 2
|
|
},
|
|
{
|
|
"text": "47",
|
|
"expected_count": 3
|
|
},
|
|
{
|
|
"text": "47",
|
|
"expected_count": 4
|
|
},
|
|
{
|
|
"text": "47",
|
|
"expected_count": 5
|
|
},
|
|
{
|
|
"text": "47",
|
|
"expected_count": 6
|
|
},
|
|
{
|
|
"text": "47",
|
|
"expected_count": 7
|
|
},
|
|
{
|
|
"text": "47",
|
|
"expected_count": 8
|
|
},
|
|
{
|
|
"text": "47",
|
|
"expected_count": 9
|
|
},
|
|
{
|
|
"text": "47",
|
|
"expected_count": 10
|
|
},
|
|
{
|
|
"text": "五万",
|
|
"expected_count": 11
|
|
},
|
|
{
|
|
"text": "五万",
|
|
"expected_count": 12
|
|
},
|
|
{
|
|
"text": "五万",
|
|
"expected_count": 13
|
|
},
|
|
{
|
|
"text": "五万",
|
|
"expected_count": 14
|
|
},
|
|
{
|
|
"text": "五万",
|
|
"expected_count": 15
|
|
},
|
|
{
|
|
"text": "五万",
|
|
"expected_count": 16
|
|
},
|
|
{
|
|
"text": "五万",
|
|
"expected_count": 17
|
|
},
|
|
{
|
|
"text": "五万",
|
|
"expected_count": 18
|
|
},
|
|
{
|
|
"text": "五万",
|
|
"expected_count": 19
|
|
},
|
|
{
|
|
"text": "五万",
|
|
"expected_count": 20
|
|
},
|
|
{
|
|
"text": "五万",
|
|
"expected_count": 21
|
|
},
|
|
{
|
|
"text": "啪",
|
|
"expected_count": 22
|
|
},
|
|
{
|
|
"text": "啪",
|
|
"expected_count": 23
|
|
},
|
|
{
|
|
"text": "啪",
|
|
"expected_count": 24
|
|
},
|
|
{
|
|
"text": "啪",
|
|
"expected_count": 25
|
|
},
|
|
{
|
|
"text": "啪",
|
|
"expected_count": 26
|
|
},
|
|
{
|
|
"text": "啪",
|
|
"expected_count": 27
|
|
},
|
|
{
|
|
"text": "啪",
|
|
"expected_count": 28
|
|
},
|
|
{
|
|
"text": "啪",
|
|
"expected_count": 29
|
|
},
|
|
{
|
|
"text": "啪",
|
|
"expected_count": 30
|
|
},
|
|
{
|
|
"text": "啪",
|
|
"expected_count": 31
|
|
}
|
|
],
|
|
"expected": {
|
|
"scoring_version": "meow-fingerprint-v3-predictive",
|
|
"verdict": "mismatch",
|
|
"color": "red",
|
|
"model": "gpt-5.6-terra",
|
|
"claimed_model": "gpt-6-astra",
|
|
"reasons": [],
|
|
"matches": {
|
|
"gpt-6-astra": 0.16949642783135352,
|
|
"gpt-6-sol": 0.262768570572442,
|
|
"gpt-5.6-terra": 0.6996240485385832,
|
|
"gpt-6-luna": 0.16949642783135352,
|
|
"other_known_external": 0.3003759514614168
|
|
},
|
|
"scores": {
|
|
"gpt-6-astra": -2.3324679246962168,
|
|
"gpt-6-sol": -1.7748957196997543,
|
|
"gpt-5.6-terra": -0.7432675462827238,
|
|
"gpt-6-luna": -2.3324679246962168,
|
|
"other_known_external": -1.5887758019039786
|
|
},
|
|
"thresholds": {
|
|
"gpt-6-astra": 0.56729343469469,
|
|
"gpt-6-sol": 0.5,
|
|
"gpt-5.6-terra": 0.5,
|
|
"gpt-6-luna": 0.5,
|
|
"other_known_external": 0.5610207135285652
|
|
},
|
|
"cells": {
|
|
"gpt__screen023": {
|
|
"planned": 11,
|
|
"minimum": 7,
|
|
"completed": 11,
|
|
"valid": 11,
|
|
"counts": {
|
|
"47": 11
|
|
},
|
|
"weight": 0.34375
|
|
},
|
|
"gpt__screen106": {
|
|
"planned": 11,
|
|
"minimum": 7,
|
|
"completed": 11,
|
|
"valid": 11,
|
|
"counts": {
|
|
"五万": 11
|
|
},
|
|
"weight": 0.34375
|
|
},
|
|
"gpt__screen114": {
|
|
"planned": 10,
|
|
"minimum": 6,
|
|
"completed": 10,
|
|
"valid": 10,
|
|
"counts": {
|
|
"啪": 10
|
|
},
|
|
"weight": 0.3125
|
|
}
|
|
},
|
|
"families": {},
|
|
"valid_samples": 32,
|
|
"planned_samples": 32,
|
|
"sample_policy": {
|
|
"version": "60-percent-v1",
|
|
"overall_ratio": 0.6,
|
|
"per_cell_ratio": 0.6
|
|
},
|
|
"partial_samples": false,
|
|
"quality_status": "sufficient"
|
|
}
|
|
},
|
|
{
|
|
"name": "gpt-6-luna",
|
|
"samples": [
|
|
{
|
|
"text": "42",
|
|
"expected_count": 0
|
|
},
|
|
{
|
|
"text": "42",
|
|
"expected_count": 1
|
|
},
|
|
{
|
|
"text": "42",
|
|
"expected_count": 2
|
|
},
|
|
{
|
|
"text": "42",
|
|
"expected_count": 3
|
|
},
|
|
{
|
|
"text": "42",
|
|
"expected_count": 4
|
|
},
|
|
{
|
|
"text": "42",
|
|
"expected_count": 5
|
|
},
|
|
{
|
|
"text": "42",
|
|
"expected_count": 6
|
|
},
|
|
{
|
|
"text": "42",
|
|
"expected_count": 7
|
|
},
|
|
{
|
|
"text": "42",
|
|
"expected_count": 8
|
|
},
|
|
{
|
|
"text": "42",
|
|
"expected_count": 9
|
|
},
|
|
{
|
|
"text": "42",
|
|
"expected_count": 10
|
|
},
|
|
{
|
|
"text": "红中",
|
|
"expected_count": 11
|
|
},
|
|
{
|
|
"text": "红中",
|
|
"expected_count": 12
|
|
},
|
|
{
|
|
"text": "红中",
|
|
"expected_count": 13
|
|
},
|
|
{
|
|
"text": "红中",
|
|
"expected_count": 14
|
|
},
|
|
{
|
|
"text": "红中",
|
|
"expected_count": 15
|
|
},
|
|
{
|
|
"text": "红中",
|
|
"expected_count": 16
|
|
},
|
|
{
|
|
"text": "红中",
|
|
"expected_count": 17
|
|
},
|
|
{
|
|
"text": "红中",
|
|
"expected_count": 18
|
|
},
|
|
{
|
|
"text": "红中",
|
|
"expected_count": 19
|
|
},
|
|
{
|
|
"text": "红中",
|
|
"expected_count": 20
|
|
},
|
|
{
|
|
"text": "红中",
|
|
"expected_count": 21
|
|
},
|
|
{
|
|
"text": "喵喵",
|
|
"expected_count": 22
|
|
},
|
|
{
|
|
"text": "喵喵",
|
|
"expected_count": 23
|
|
},
|
|
{
|
|
"text": "喵喵",
|
|
"expected_count": 24
|
|
},
|
|
{
|
|
"text": "喵喵",
|
|
"expected_count": 25
|
|
},
|
|
{
|
|
"text": "喵喵",
|
|
"expected_count": 26
|
|
},
|
|
{
|
|
"text": "喵喵",
|
|
"expected_count": 27
|
|
},
|
|
{
|
|
"text": "喵喵",
|
|
"expected_count": 28
|
|
},
|
|
{
|
|
"text": "喵喵",
|
|
"expected_count": 29
|
|
},
|
|
{
|
|
"text": "喵喵",
|
|
"expected_count": 30
|
|
},
|
|
{
|
|
"text": "喵喵",
|
|
"expected_count": 31
|
|
}
|
|
],
|
|
"expected": {
|
|
"scoring_version": "meow-fingerprint-v3-predictive",
|
|
"verdict": "mismatch",
|
|
"color": "red",
|
|
"model": "gpt-6-luna",
|
|
"claimed_model": "gpt-6-astra",
|
|
"reasons": [],
|
|
"matches": {
|
|
"gpt-6-astra": 0.11034768854830239,
|
|
"gpt-6-sol": 0.18262928514091403,
|
|
"gpt-5.6-terra": 0.17249846553639775,
|
|
"gpt-6-luna": 0.6626074593360759,
|
|
"other_known_external": 0.3373925406639241
|
|
},
|
|
"scores": {
|
|
"gpt-6-astra": -2.463864569052184,
|
|
"gpt-6-sol": -1.8753044392335134,
|
|
"gpt-5.6-terra": -1.9446926504739512,
|
|
"gpt-6-luna": -0.37667002987845055,
|
|
"other_known_external": -1.0516057157764647
|
|
},
|
|
"thresholds": {
|
|
"gpt-6-astra": 0.56729343469469,
|
|
"gpt-6-sol": 0.5,
|
|
"gpt-5.6-terra": 0.5,
|
|
"gpt-6-luna": 0.5,
|
|
"other_known_external": 0.5610207135285652
|
|
},
|
|
"cells": {
|
|
"gpt__screen023": {
|
|
"planned": 11,
|
|
"minimum": 7,
|
|
"completed": 11,
|
|
"valid": 11,
|
|
"counts": {
|
|
"42": 11
|
|
},
|
|
"weight": 0.34375
|
|
},
|
|
"gpt__screen106": {
|
|
"planned": 11,
|
|
"minimum": 7,
|
|
"completed": 11,
|
|
"valid": 11,
|
|
"counts": {
|
|
"红中": 11
|
|
},
|
|
"weight": 0.34375
|
|
},
|
|
"gpt__screen114": {
|
|
"planned": 10,
|
|
"minimum": 6,
|
|
"completed": 10,
|
|
"valid": 10,
|
|
"counts": {
|
|
"喵喵": 10
|
|
},
|
|
"weight": 0.3125
|
|
}
|
|
},
|
|
"families": {},
|
|
"valid_samples": 32,
|
|
"planned_samples": 32,
|
|
"sample_policy": {
|
|
"version": "60-percent-v1",
|
|
"overall_ratio": 0.6,
|
|
"per_cell_ratio": 0.6
|
|
},
|
|
"partial_samples": false,
|
|
"quality_status": "sufficient"
|
|
}
|
|
},
|
|
{
|
|
"name": "partial",
|
|
"samples": [
|
|
{
|
|
"text": "43",
|
|
"expected_count": 0
|
|
},
|
|
{
|
|
"text": "43",
|
|
"expected_count": 1
|
|
},
|
|
{
|
|
"text": "43",
|
|
"expected_count": 2
|
|
},
|
|
{
|
|
"text": "43",
|
|
"expected_count": 3
|
|
},
|
|
{
|
|
"text": "43",
|
|
"expected_count": 4
|
|
},
|
|
{
|
|
"text": "43",
|
|
"expected_count": 5
|
|
},
|
|
{
|
|
"text": "43",
|
|
"expected_count": 6
|
|
},
|
|
{
|
|
"text": "43",
|
|
"expected_count": 7
|
|
}
|
|
],
|
|
"expected": {
|
|
"scoring_version": "meow-fingerprint-v3-predictive",
|
|
"verdict": "insufficient",
|
|
"color": "yellow",
|
|
"model": null,
|
|
"claimed_model": "gpt-6-astra",
|
|
"reasons": [
|
|
"samples_incomplete"
|
|
],
|
|
"matches": {
|
|
"gpt-6-astra": 0.6637577509332276,
|
|
"gpt-6-sol": 0.13628981670052215,
|
|
"gpt-5.6-terra": 0.2236014336853945,
|
|
"gpt-6-luna": 0.10869590616949418,
|
|
"other_known_external": 0.3362422490667724
|
|
},
|
|
"scores": {
|
|
"gpt-6-astra": -0.05486073970997829,
|
|
"gpt-6-sol": -1.9013143927307334,
|
|
"gpt-5.6-terra": -1.2996615913309375,
|
|
"gpt-6-luna": -2.1589922722528767,
|
|
"other_known_external": -0.734946110138968
|
|
},
|
|
"thresholds": {
|
|
"gpt-6-astra": 0.56729343469469,
|
|
"gpt-6-sol": 0.5,
|
|
"gpt-5.6-terra": 0.5,
|
|
"gpt-6-luna": 0.5,
|
|
"other_known_external": 0.5610207135285652
|
|
},
|
|
"cells": {
|
|
"gpt__screen023": {
|
|
"planned": 11,
|
|
"minimum": 7,
|
|
"completed": 8,
|
|
"valid": 8,
|
|
"counts": {
|
|
"43": 8
|
|
},
|
|
"weight": 1.0
|
|
},
|
|
"gpt__screen106": {
|
|
"planned": 11,
|
|
"minimum": 7,
|
|
"completed": 0,
|
|
"valid": 0,
|
|
"counts": {},
|
|
"weight": 0.0
|
|
},
|
|
"gpt__screen114": {
|
|
"planned": 10,
|
|
"minimum": 6,
|
|
"completed": 0,
|
|
"valid": 0,
|
|
"counts": {},
|
|
"weight": 0.0
|
|
}
|
|
},
|
|
"families": {},
|
|
"valid_samples": 8,
|
|
"planned_samples": 32,
|
|
"sample_policy": {
|
|
"version": "60-percent-v1",
|
|
"overall_ratio": 0.6,
|
|
"per_cell_ratio": 0.6
|
|
},
|
|
"partial_samples": true,
|
|
"quality_status": "insufficient_valid_samples"
|
|
}
|
|
},
|
|
{
|
|
"name": "empty",
|
|
"samples": [],
|
|
"expected": {
|
|
"scoring_version": "meow-fingerprint-v3-predictive",
|
|
"verdict": "insufficient",
|
|
"color": "yellow",
|
|
"model": null,
|
|
"claimed_model": "gpt-6-astra",
|
|
"reasons": [
|
|
"no_valid_samples",
|
|
"samples_incomplete"
|
|
],
|
|
"matches": {
|
|
"gpt-6-astra": 0.5,
|
|
"gpt-6-sol": 0.5,
|
|
"gpt-5.6-terra": 0.5,
|
|
"gpt-6-luna": 0.5,
|
|
"other_known_external": 0.5
|
|
},
|
|
"scores": {
|
|
"gpt-6-astra": 0.0,
|
|
"gpt-6-sol": 0.0,
|
|
"gpt-5.6-terra": 0.0,
|
|
"gpt-6-luna": 0.0,
|
|
"other_known_external": 0.0
|
|
},
|
|
"thresholds": {
|
|
"gpt-6-astra": 0.56729343469469,
|
|
"gpt-6-sol": 0.5,
|
|
"gpt-5.6-terra": 0.5,
|
|
"gpt-6-luna": 0.5,
|
|
"other_known_external": 0.5610207135285652
|
|
},
|
|
"cells": {
|
|
"gpt__screen023": {
|
|
"planned": 11,
|
|
"minimum": 7,
|
|
"completed": 0,
|
|
"valid": 0,
|
|
"counts": {},
|
|
"weight": 0.0
|
|
},
|
|
"gpt__screen106": {
|
|
"planned": 11,
|
|
"minimum": 7,
|
|
"completed": 0,
|
|
"valid": 0,
|
|
"counts": {},
|
|
"weight": 0.0
|
|
},
|
|
"gpt__screen114": {
|
|
"planned": 10,
|
|
"minimum": 6,
|
|
"completed": 0,
|
|
"valid": 0,
|
|
"counts": {},
|
|
"weight": 0.0
|
|
}
|
|
},
|
|
"families": {},
|
|
"valid_samples": 0,
|
|
"planned_samples": 32,
|
|
"sample_policy": {
|
|
"version": "60-percent-v1",
|
|
"overall_ratio": 0.6,
|
|
"per_cell_ratio": 0.6
|
|
},
|
|
"partial_samples": true,
|
|
"quality_status": "insufficient_valid_samples"
|
|
}
|
|
},
|
|
{
|
|
"name": "unseen",
|
|
"samples": [
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 0
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 1
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 2
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 3
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 4
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 5
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 6
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 7
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 8
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 9
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 10
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 11
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 12
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 13
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 14
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 15
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 16
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 17
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 18
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 19
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 20
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 21
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 22
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 23
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 24
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 25
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 26
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 27
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 28
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 29
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 30
|
|
},
|
|
{
|
|
"text": "UNSEEN fixture answer",
|
|
"expected_count": 31
|
|
}
|
|
],
|
|
"expected": {
|
|
"scoring_version": "meow-fingerprint-v3-predictive",
|
|
"verdict": "mismatch",
|
|
"color": "red",
|
|
"model": "other_known_external",
|
|
"claimed_model": "gpt-6-astra",
|
|
"reasons": [],
|
|
"matches": {
|
|
"gpt-6-astra": 0.3905375565158004,
|
|
"gpt-6-sol": 0.39053755651580024,
|
|
"gpt-5.6-terra": 0.3905375565158004,
|
|
"gpt-6-luna": 0.39053755651580024,
|
|
"other_known_external": 0.6094624434841995
|
|
},
|
|
"scores": {
|
|
"gpt-6-astra": -2.463864569052184,
|
|
"gpt-6-sol": -2.463864569052185,
|
|
"gpt-5.6-terra": -2.463864569052184,
|
|
"gpt-6-luna": -2.463864569052185,
|
|
"other_known_external": -2.0188113807068433
|
|
},
|
|
"thresholds": {
|
|
"gpt-6-astra": 0.56729343469469,
|
|
"gpt-6-sol": 0.5,
|
|
"gpt-5.6-terra": 0.5,
|
|
"gpt-6-luna": 0.5,
|
|
"other_known_external": 0.5610207135285652
|
|
},
|
|
"cells": {
|
|
"gpt__screen023": {
|
|
"planned": 11,
|
|
"minimum": 7,
|
|
"completed": 11,
|
|
"valid": 11,
|
|
"counts": {
|
|
"__UNSEEN_IN_TRAINING__": 11
|
|
},
|
|
"weight": 0.34375
|
|
},
|
|
"gpt__screen106": {
|
|
"planned": 11,
|
|
"minimum": 7,
|
|
"completed": 11,
|
|
"valid": 11,
|
|
"counts": {
|
|
"__UNSEEN_IN_TRAINING__": 11
|
|
},
|
|
"weight": 0.34375
|
|
},
|
|
"gpt__screen114": {
|
|
"planned": 10,
|
|
"minimum": 6,
|
|
"completed": 10,
|
|
"valid": 10,
|
|
"counts": {
|
|
"__UNSEEN_IN_TRAINING__": 10
|
|
},
|
|
"weight": 0.3125
|
|
}
|
|
},
|
|
"families": {},
|
|
"valid_samples": 32,
|
|
"planned_samples": 32,
|
|
"sample_policy": {
|
|
"version": "60-percent-v1",
|
|
"overall_ratio": 0.6,
|
|
"per_cell_ratio": 0.6
|
|
},
|
|
"partial_samples": false,
|
|
"quality_status": "sufficient"
|
|
}
|
|
}
|
|
] |