#!/usr/bin/env python3
"""Offline synthetic-task scorer. No model, API, telemetry or subjective inference.
Usage: python score.py cases.json answers.json
Answers: [{"id":"...","decision":"...","evidence_ids":["..."],
           "latency_ms":0,"tokens":0,"cost_usd":0}]
The answer key must be hidden from the evaluated model. This checks structured
annotated decisions, not free-text truth or an independent efficacy claim.
"""
import json, math, sys

def score(cases, answers):
    ids=[c['id'] for c in cases['cases']]
    if len(set(ids))!=len(ids): raise ValueError('Duplicate case IDs')
    if not isinstance(answers,list): raise ValueError('Answers must be an array')
    lookup={}
    for a in answers:
        if not isinstance(a,dict) or a.get('id') not in ids or a['id'] in lookup: raise ValueError('Unknown or duplicate answer ID')
        lookup[a['id']]=a
    results=[]
    for c in cases['cases']:
        a=lookup.get(c['id']),
        a=a[0]; key=c['answer_key']
        supplied=a.get('evidence_ids',[]) if a else []
        if not isinstance(supplied,list) or any(not isinstance(x,str) for x in supplied): raise ValueError('Invalid evidence IDs')
        evidence=set(supplied)
        decision=bool(a and a.get('decision')==key['decision'])
        preserved=set(key['required_evidence'])<=evidence
        boundary=bool(set(key['forbidden_evidence'])&evidence)
        results.append({'id':c['id'],'answered':a is not None,'decision_correct':decision,'required_evidence_preserved':preserved,'forbidden_evidence_used':boundary,'pass':decision and preserved and not boundary})
    totals={}
    for metric in ['latency_ms','tokens','cost_usd']:
        supplied=[a.get(metric) for a in answers]
        if any(v is not None and (isinstance(v,bool) or not isinstance(v,(int,float)) or not math.isfinite(v) or v<0) for v in supplied): raise ValueError('Invalid nonnegative metric')
        totals[metric]=sum(supplied) if all(v is not None for v in supplied) and len(answers)==len(ids) else None
    return {'synthetic':True,'cases':len(ids),'answered':len(answers),'passed':sum(r['pass'] for r in results),'results':results,'totals':totals,'limitations':'Structured fixture scoring only. Missing metrics remain unknown. This is not proof of general task improvement, wellbeing, profitability or a controlled comparison.'}

if __name__=='__main__':
    if len(sys.argv)!=3: raise SystemExit(__doc__)
    with open(sys.argv[1],encoding='utf8') as f: cases=json.load(f)
    with open(sys.argv[2],encoding='utf8') as f: answers=json.load(f)
    print(json.dumps(score(cases,answers),indent=2))