"""Build the browser's compact GloVe vocabulary; Python standard library only.

Downloads public pretrained vectors, never game data. Run from the project root.
"""
import gzip
import hashlib
import json
import math
import re
import urllib.request
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
URL = "https://github.com/piskvorky/gensim-data/releases/download/glove-wiki-gigaword-50/glove-wiki-gigaword-50.gz"
EXPECTED_MD5 = "c289bc5d7f2f02c6dc9f2f9b67641813"  # Published gensim-data artifact checksum.


def main():
    cache = ROOT / ".data" / "prompt-model" / "glove-wiki-gigaword-50.gz"
    cache.parent.mkdir(parents=True, exist_ok=True)
    if not cache.exists():
        print("Downloading 69 MB of public word vectors (one-time build input).", flush=True)
        urllib.request.urlretrieve(URL, cache)
    digest = hashlib.md5(cache.read_bytes(), usedforsecurity=False).hexdigest()
    if digest != EXPECTED_MD5:
        raise ValueError("Source checksum mismatch; cached source was not used.")
    words = {}
    with gzip.open(cache, "rt", encoding="utf-8") as source:
        next(source)
        for line in source:
            word, *values = line.split()
            if not re.fullmatch(r"[a-z]+", word) or len(values) != 50:
                continue
            vector = [float(value) for value in values]
            norm = math.sqrt(sum(value * value for value in vector))
            words[word] = [round(127 * value / norm) for value in vector]
            if len(words) == 20000:
                break
    model = {"version": 1, "name": "glove-wiki-gigaword-50", "dimensions": 50,
             "source": URL, "sourceMd5": digest,
             "sourceSha256": hashlib.sha256(cache.read_bytes()).hexdigest(),
             "license": "https://opendatacommons.org/licenses/pddl/1-0/",
             "selection": "First 20000 alphabetic lowercase tokens in source order; normalized and int8 quantized.",
             "words": words}
    destination = ROOT / "public" / "models" / "prompt-glove-50.json"
    destination.parent.mkdir(parents=True, exist_ok=True)
    destination.write_text(json.dumps(model, separators=(",", ":")) + "\n", encoding="utf-8")
    print(json.dumps({"path": str(destination), "words": len(words), "bytes": destination.stat().st_size,
                      "sha256": hashlib.sha256(destination.read_bytes()).hexdigest()}))


if __name__ == "__main__":
    main()