import time
from collections import Counter

from common import connect, TABLE, read_chunks, to_record

rows, model, dim = read_chunks()
print(f"{len(rows)} chunks, model={model}, dim={dim}")

db = connect()
t0 = time.time()
table = db.create_table(TABLE, data=[to_record(r) for r in rows], mode="overwrite")
print(f"created table in {time.time() - t0:.2f}s")
print(table.schema)
print("count:", table.count_rows())
assert table.count_rows() == len(rows)

sections = Counter(r["section"] for r in table.search().select(["section"]).limit(10000).to_list())
print(f"{len(sections)} sections, total {sum(sections.values())}")
for s, n in sections.most_common():
    print(f"  {s:20s} {n}")