#!/usr/bin/env python3
"""Held-out benchmark for jsonshim.

The rules this file plays by, stated before the numbers:

  * The cases in jsonshim.py CASES were used while writing the repairs. Anything
    measured on them is in-sample and worth nothing as a claim.
  * The cases below were written AFTER jsonshim.py was finished and were run
    exactly once. Whatever they gave is what is published, failures included.
  * "Recovered" means the returned value EQUALS the value a human would have
    written down - not merely that something parsed. A parser that returns the
    wrong dict has failed, not passed.
  * REFUSE cases have no recoverable value. Returning anything there is a
    failure, because a wrong object is worse than an error.

Run: python3 bench.py
"""
import json
import sys

from jsonshim import extract

HOLDOUT = [
    ("tool call, fence + prose",
     'I will search for that.\n\n```json\n{"name": "web_search", "arguments": {"query": "python 3.13 release date", "top_k": 5}}\n```\nLet me know if you want more.',
     {"name": "web_search", "arguments": {"query": "python 3.13 release date", "top_k": 5}}),
    ("two fences, first is python",
     '```python\nprint("hi")\n```\nand the config:\n```json\n{"temperature": 0.7}\n```',
     {"temperature": 0.7}),
    ("markdown bold before",
     '**Result:**\n{"ok": true, "count": 0}',
     {"ok": True, "count": 0}),
    ("negative and exponent numbers",
     '{"delta": -0.25, "scale": 1e-3}',
     {"delta": -0.25, "scale": 1e-3}),
    ("nested trailing commas both levels",
     '{"a": [1, 2,], "b": {"c": 3,},}',
     {"a": [1, 2], "b": {"c": 3}}),
    ("single quotes with a comma inside",
     "{'note': 'one, two, three'}",
     {"note": "one, two, three"}),
    ("mixed quote styles",
     '{"a": \'x\', \'b\': "y"}',
     {"a": "x", "b": "y"}),
    ("unquoted key with underscore and digit",
     '{max_tokens_2: 100}',
     {"max_tokens_2": 100}),
    ("comment after every line",
     '{\n "a": 1, // first\n "b": 2 // second\n}',
     {"a": 1, "b": 2}),
    ("block comment spanning lines",
     '{\n/* the model\n   explains itself */\n"a": 1}',
     {"a": 1}),
    ("truncated inside a nested key",
     '{"plan": {"steps": ["one", "two"], "note": "we sho',
     {"plan": {"steps": ["one", "two"], "note": "we sho"}}),
    ("truncated right after an opening brace",
     '{"a": 1, "b": {',
     {"a": 1, "b": {}}),
    ("truncated inside an escape",
     '{"a": "line\\',
     {"a": "line"}),
    ("array of tool calls, truncated",
     '[{"tool": "a"}, {"tool": "b"}, {"tool":',
     [{"tool": "a"}, {"tool": "b"}]),
    ("escaped braces in a string value",
     '{"template": "hello {name}, bye {other}"}',
     {"template": "hello {name}, bye {other}"}),
    ("backslash path in a string",
     '{"path": "C:\\\\Users\\\\x"}',
     {"path": "C:\\Users\\x"}),
    ("empty object and empty array",
     'here: {"a": {}, "b": []}',
     {"a": {}, "b": []}),
    ("deeply nested clean",
     '{"a":{"b":{"c":{"d":[1,{"e":2}]}}}}',
     {"a": {"b": {"c": {"d": [1, {"e": 2}]}}}}),
    ("unicode text, no escapes",
     '{"msg": "café über naïve"}',
     {"msg": "café über naïve"}),
    ("json inside a sentence with a period after",
     'The answer is {"x": 1}.',
     {"x": 1}),
    ("boolean-looking string stays a string",
     '{"a": "true"}',
     {"a": "true"}),
    ("leading zeros are not valid json",
     '{"a": 007}',
     {"a": "007"}),
    ("plus-prefixed number",
     '{"a": +1}',
     {"a": "+1"}),
    ("array at top level with prose",
     'Ranked:\n1. stuff\n["a", "b", "c"]',
     ["a", "b", "c"]),
    ("object with a null value spelled None inside an array",
     '[None, True, 1]',
     [None, True, 1]),
    ("crlf line endings",
     '{\r\n  "a": 1\r\n}',
     {"a": 1}),
    ("tab indentation and a trailing comma",
     '{\n\t"a": 1,\n}',
     {"a": 1}),
    ("very long string value",
     '{"a": "' + "x" * 5000 + '"}',
     {"a": "x" * 5000}),
    ("duplicate keys, last wins as in json",
     '{"a": 1, "a": 2}',
     {"a": 2}),
    ("colon inside an unquoted-looking url value",
     '{"u": http://a.b}',
     {"u": "http://a.b"}),
]

REFUSE = [
    ("apology only", "I'm sorry, I can't help with that."),
    ("code with braces but no json", "def f(x):\n    return {y for y in x}"),
    ("whitespace only", "   \n\t  "),
]


def run():
    base_ok = shim_ok = 0
    fails = []
    for name, src, want in HOLDOUT:
        try:
            if json.loads(src) == want:
                base_ok += 1
        except Exception:
            pass
        r = extract(src)
        if r.ok and r.value == want:
            shim_ok += 1
        else:
            fails.append((name, r.value if r.ok else "REFUSED(%s)" % r.error))

    invented = []
    for name, src in REFUSE:
        r = extract(src)
        if r.ok and r.value not in ({}, []):
            invented.append((name, r.value))

    n = len(HOLDOUT)
    print("held-out cases: %d   (written after the tool, run once, never tuned on)" % n)
    print("  json.loads baseline exact-match : %2d/%d  (%.1f%%)"
          % (base_ok, n, 100.0 * base_ok / n))
    print("  jsonshim exact-match            : %2d/%d  (%.1f%%)"
          % (shim_ok, n, 100.0 * shim_ok / n))
    print()
    print("refuse cases: %d   (any value returned here is a failure)" % len(REFUSE))
    print("  jsonshim invented a value       : %d" % len(invented))
    for name, val in invented:
        print("      %-34s -> %r" % (name, val))
    if fails:
        print()
        print("held-out failures, listed because hiding them would make the number a lie:")
        for name, got in fails:
            print("  %-38s -> %r" % (name, got))
    print()
    print("in-sample self-test (worth nothing as a claim, run it anyway):")
    import jsonshim
    print("  %s" % ("PASS" if jsonshim.selftest(verbose=False) else "FAIL"))
    return 0


if __name__ == "__main__":
    sys.exit(run())