import os,json,time,urllib.request
K=os.environ["WEBCRAWLER_API_KEY"]
def call(m,u,b=None):
    r=urllib.request.Request(u,data=json.dumps(b).encode() if b else None,method=m,headers={"Authorization":"Bearer "+K,"Content-Type":"application/json"})
    return json.load(urllib.request.urlopen(r,timeout=60))
t0=time.time()
j=call("POST","https://api.webcrawlerapi.com/v1/crawl",{"url":"https://webcrawlerapi.com/docs","items_limit":60,"output_formats":["markdown"],"whitelist_regexp":"^https://webcrawlerapi\\.com/docs","main_content_only":True})
print("created",{k:v for k,v in j.items() if k!="job_items"})
jid=j.get("id") or j.get("job_id")
while True:
    d=call("GET",f"https://api.webcrawlerapi.com/v1/job/{jid}")
    if d.get("status") not in ("new","in_progress"): break
    time.sleep(2)
wall=time.time()-t0
items=d.get("job_items",[])
print("wall_s",round(wall,1))
print({k:v for k,v in d.items() if k!="job_items" and k not in ("whitelist_regexp",)})
from collections import Counter
print(Counter(i.get("status") for i in items), len(items))