"""Retrieval eval: 6 labelled questions, does top-1 retrieval pick the right runbook?"""
import json, subprocess, sys, math, urllib.request
sys.argv = ['x']; src = open('rag.py').read().split("q = sys.argv")[0]; exec(src)
CASES = [('container killed, exit code 137', 'oomkilled.md'), ('pod never gets scheduled', 'pending.md'), ('df says 100% but du finds nothing', 'disk-full.md'),
         ('ErrImagePull after deploy', 'imagepull.md'), ('cannot resolve external api from pod', 'dns.md'), ('undo the release that broke checkout', 'rollback.md')]
E = {k: post('/api/embeddings', {'model': 'nomic-embed-text', 'prompt': v})['embedding'] for k, v in DOCS.items()}
ok = 0
for q, want in CASES:
    qe = post('/api/embeddings', {'model': 'nomic-embed-text', 'prompt': q})['embedding']
    got = max(E, key=lambda k: sum(x * y for x, y in zip(qe, E[k])) / (math.sqrt(sum(x * x for x in qe)) * math.sqrt(sum(y * y for y in E[k]))))
    ok += got == want; print('%-4s %-42s -> %s' % ('PASS' if got == want else 'FAIL', q, got))
print('retrieval accuracy: %d/%d' % (ok, len(CASES)))