#!/usr/bin/env python3
"""Re-probe the 33 hosts probe2 graded BROKEN, this time filling in the path
parameters the sellers documented.

probe2 was itself the correction of probe1, whose four bugs all made the
ecosystem look worse than it is. I wrote two thousand words about that this
morning, published the instrument, and invited people to check me. Then I went
to extract the list of broken hosts so operators could find themselves on it,
looked at the first two rows, and found this:

    404 POST  https://image.gedx402.com/v1/image/:model
    400 GET   https://intel.twzrd.xyz/v1/intel/trust/:pubkey

Those are route templates. I had requested a literal `:model`. The servers
returned 404 for a path that does not exist, which is the correct behaviour, and
one of them said so in the body: {"ok":false,"documented":true,
"content_kind":"route_template"...}. 27 of the 33 have an unfilled `:param` in
the URL.

And the sellers had told me what to put there. The listing carries
`input.pathParams` next to `input.method` and `input.queryParams`:

    {"method":"GET","pathParams":{"symbol":"AAPL"},"queryParams":{"symbol":"AAPL"}}

probe2 read method, body and queryParams from that object. It never read
pathParams. So this is the fifth instrument bug, it is the same bug as the third
one -- ignoring metadata the seller published -- and like all the others it made
the ecosystem look worse rather than better. Five for five in the same
direction is not bad luck. It is what building a measurement while already
holding the story does to the measurement.
"""
import json, re, sys, concurrent.futures as cf, urllib.request, urllib.error, urllib.parse

sys.path.insert(0, "/home/agent/x402survey")
from probe2 import terms_from, UA, TIMEOUT, MAXBODY  # same instrument, one field wiser

HOSTS = "/home/agent/x402survey/probe2_hosts.json"
RAW = "/home/agent/x402survey/raw.json"


def fill(url, path_params):
    """Substitute documented example values for :name and {name} segments."""
    used = {}
    for k, v in (path_params or {}).items():
        for pat in (f":{k}", "{" + k + "}"):
            if pat in url:
                url = url.replace(pat, urllib.parse.quote(str(v), safe=""))
                used[k] = v
    return url, used


def probe(rec):
    url = rec["url"]
    req = urllib.request.Request(url, method=rec["method"])
    req.add_header("user-agent", UA)
    req.add_header("accept", "application/json, */*")
    data = None
    if rec.get("body") is not None and rec["method"] in ("POST", "PUT", "PATCH"):
        data = json.dumps(rec["body"]).encode()
        req.add_header("content-type", "application/json")
    try:
        with urllib.request.urlopen(req, data=data, timeout=TIMEOUT) as r:
            code, body, hdrs = r.status, r.read(MAXBODY), dict(r.headers.items())
    except urllib.error.HTTPError as e:
        code, body, hdrs = e.code, e.read(MAXBODY), dict(e.headers.items())
    except Exception as e:
        return dict(rec, code=None, grade="DEAD", why=f"{type(e).__name__}: {e}"[:120])
    hl = {k.lower(): v for k, v in hdrs.items()}
    txt = body.decode("utf-8", "replace")
    if code == 402:
        t = terms_from(txt, hl)
        return dict(rec, code=code, grade="LIVE" if t else "OPAQUE",
                    why="402 with terms" if t else "402 without terms")
    if code == 200:
        return dict(rec, code=code, grade="OPEN", why="200 without payment")
    return dict(rec, code=code, grade="BROKEN", why=f"HTTP {code}", body=txt[:200])


def main():
    raw = json.load(open(RAW))
    by_res = {}
    for r in raw:
        u = r.get("resource")
        if isinstance(u, str):
            by_res.setdefault(u, r)

    hosts = json.load(open(HOSTS))
    broken = [h for h in hosts if h.get("grade") == "BROKEN"]
    jobs = []
    for h in broken:
        rec = by_res.get(h["resource"], {})
        info = (rec.get("extensions") or {}).get("bazaar", {}).get("info", {})
        inp = info.get("input") or {}
        url, used = fill(h["resource"], inp.get("pathParams"))
        qp = inp.get("queryParams") or {}
        if qp and "?" not in url:
            url += "?" + urllib.parse.urlencode({k: v for k, v in qp.items()
                                                 if isinstance(v, (str, int, float))})
        jobs.append({"resource": h["resource"], "url": url, "host": h["host"],
                     "method": (inp.get("method") or h.get("method") or "GET").upper(),
                     "body": inp.get("body"), "filled": used,
                     "old_code": h.get("code"), "old_grade": h["grade"]})

    with cf.ThreadPoolExecutor(max_workers=12) as ex:
        out = list(ex.map(probe, jobs))

    json.dump(out, open("/home/agent/x402survey/probe3_broken.json", "w"), indent=1)
    import collections
    print(collections.Counter(o["grade"] for o in out))
    for o in sorted(out, key=lambda x: x["grade"]):
        mark = "fixed-by-params" if o["filled"] and o["grade"] != "BROKEN" else ""
        print(f'  {str(o["old_code"]):>4} -> {str(o["code"]):>4} {o["grade"]:7}'
              f' {o["resource"][:62]} {mark}')


if __name__ == "__main__":
    import urllib.parse
    main()