wordle/raw/generate-banks-1000.py

#!/usr/bin/env python3
"""Generate word-bank.json and word-bank-ru.json with 1000 targets each.

Targets are the first 1000 words from the frequency-sorted guessable list
(most common words first). IDs are sequential 1..1000.

Uses:
  - Datamuse API + WordNet for English synonyms
  - Abramov's Russian synonym dictionary for Russian synonyms
"""

import json
import sys
import time
from pathlib import Path

RAW_DIR = Path(__file__).parent
DATA_DIR = RAW_DIR.parent / "data"

TARGET_COUNT = 1000

# ---- English hint overrides (curated for quality) ----
EN_HINT_OVERRIDES: dict[str, str] = {
    "about": "around", "there": "present", "which": "that",
    "other": "different", "would": "could", "these": "those",
    "while": "during", "being": "living", "where": "location",
    "still": "quiet", "never": "not ever", "every": "each",
    "since": "because", "those": "these", "among": "between",
    "money": "cash", "local": "nearby", "least": "fewest",
    "month": "april", "river": "stream", "david": "giant",
    "steve": "stephen", "didnt": "did not", "three": "trio",
    "years": "decade", "going": "moving",
    "state": "condition", "games": "sport", "found": "located",
    "often": "frequent", "south": "southern", "small": "little",
    "north": "northern", "third": "tertiary", "party": "group",
    "began": "started", "young": "youth", "final": "last",
    "given": "provided", "using": "employ", "field": "area",
    "teams": "squad", "thing": "object", "white": "pale",
    "march": "parade", "black": "dark", "built": "constructed",
    "moved": "shifted", "added": "extra", "needs": "requires",
    "short": "brief", "front": "forward", "seven": "seventh",
    "title": "name", "comes": "arrives", "hours": "time",
    "eight": "eighth", "green": "color", "round": "circular",
    "union": "group", "lives": "exists", "radio": "broadcast",
    "words": "phrase", "model": "design", "terms": "words",
    "score": "points", "ready": "prepared", "grand": "large",
}


# ---- Russian hint overrides (curated for the most common words) ----
RU_HINT_OVERRIDES: dict[str, str] = {
    "жизнь": "бытие", "слово": "речь", "часть": "доля", "город": "посёлок",
    "земля": "почва", "право": "закон", "образ": "облик", "закон": "правило",
    "война": "битва", "голос": "звук", "книга": "роман", "число": "цифра",
    "народ": "люди", "форма": "фигура", "улица": "дорога", "вечер": "закат",
    "мысль": "идея", "месяц": "луна", "школа": "учёба", "театр": "сцена",
    "смысл": "суть", "рынок": "базар", "центр": "середина", "ответ": "отклик",
    "автор": "писатель", "совет": "наказ", "глава": "лидер", "наука": "знание",
    "плечо": "рука", "точка": "пятно", "палец": "рука", "метод": "способ",
    "фильм": "кино", "гость": "визитёр", "район": "зона", "кровь": "рана",
    "армия": "войско", "класс": "группа", "герой": "смельчак", "спина": "тело",
    "берег": "пляж", "фирма": "компания", "завод": "фабрика", "повод": "причина",
    "выход": "уход", "текст": "письмо", "пункт": "место", "линия": "черта",
    "среда": "окружение", "волос": "прядь", "ветер": "буря", "огонь": "пламя",
    "страх": "ужас", "сфера": "область", "немец": "германец", "выбор": "отбор",
    "масса": "вес", "кухня": "готовка", "товар": "продукт", "вывод": "итог",
    "норма": "правило", "рамка": "граница", "прием": "встреча", "режим": "порядок",
    "целое": "полное", "доход": "прибыль", "карта": "схема", "акция": "дело",
    "фраза": "выражение", "толпа": "скопление", "птица": "пернатый", "запах": "аромат",
    "поезд": "состав", "адрес": "жильё", "лидер": "вождь", "весна": "сезон",
    "музей": "галерея", "сутки": "день", "еврей": "иудей", "сотня": "сто",
    "труба": "дымоход", "масло": "жир", "экран": "дисплей", "вагон": "поезд",
    "сезон": "пора", "длина": "размер", "доска": "дерево", "лодка": "судно",
    "серия": "ряд", "кулак": "рука", "нефть": "топливо", "кость": "скелет",
    "взрыв": "удар", "почва": "земля", "хвост": "конец", "строй": "порядок",
    "отчет": "доклад", "забор": "стена", "пожар": "огонь", "ножка": "нога",
}


def load_dictionary(path: Path) -> list[str]:
    """Read frequency-sorted word list (one word per line)."""
    return [line.strip() for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]


def load_abramov_dict() -> dict[str, str]:
    """Load Abramov's Russian synonym dictionary, returning {word: best_synonym}.
    Downloads from GitHub if not already cached locally."""
    path = RAW_DIR / "abramov-ru-synonyms.json"
    if not path.exists():
        print("  Downloading Abramov Russian synonym dictionary...")
        import requests
        url = "https://raw.githubusercontent.com/egorkaru/synonym_dictionary/master/dictionary.json"
        resp = requests.get(url, timeout=30)
        resp.raise_for_status()
        path.write_text(resp.text, encoding="utf-8")
    data = json.loads(path.read_text(encoding="utf-8"))
    result: dict[str, str] = {}
    for entry in data["wordlist"]:
        name = entry["name"].lower()
        synonyms = entry.get("synonyms", [])
        if not synonyms:
            continue
        # Pick the best single-word synonym
        for syn in synonyms:
            # Clean: remove multi-word alternatives (split by ";"), pick first single word
            parts = syn.replace(";", ",").split(",")
            for part in parts:
                part = part.strip().lower()
                # Skip multi-word phrases, the word itself, and single letters
                if " " in part or part == name or len(part) <= 1:
                    continue
                result[name] = part
                break
            if name in result:
                break
    print(f"  Loaded {len(result)} Russian synonyms from Abramov dictionary")
    return result


def get_english_synonyms_datamuse(word: str) -> list[str]:
    """Get related words from Datamuse API."""
    import requests
    try:
        resp = requests.get(
            "https://api.datamuse.com/words",
            params={"ml": word, "max": 8},
            timeout=5,
        )
        resp.raise_for_status()
        results = resp.json()
        return [r["word"].lower() for r in results if " " not in r["word"] and r["word"].lower() != word]
    except Exception as e:
        print(f"    Datamuse error for '{word}': {e}", file=sys.stderr)
        return []


def get_english_synonyms_wordnet(word: str) -> list[str]:
    """Get synonyms from WordNet."""
    from nltk.corpus import wordnet as wn
    syns = wn.synsets(word)
    if not syns:
        return []
    lemmas = []
    for s in syns[:10]:
        for l in s.lemmas():
            name = l.name().lower()
            if "_" not in name and name != word and name not in lemmas:
                lemmas.append(name)
    return lemmas


def pick_best_synonym(word: str, candidates: list[str]) -> str | None:
    """Pick the best synonym from candidates."""
    good = [c for c in candidates if len(c) <= 10 and "_" not in c]
    if not good:
        return None
    good.sort(key=len)
    return good[0]


def generate_english_hints(targets: list[dict]) -> int:
    """Add hints to English targets. Returns count of missing hints."""
    missing = 0
    for i, target in enumerate(targets):
        word = target["word"]

        # Check manual override first
        if word in EN_HINT_OVERRIDES:
            target["hint"] = EN_HINT_OVERRIDES[word]
            continue

        # Try Datamuse
        candidates = get_english_synonyms_datamuse(word)
        if not candidates:
            candidates = get_english_synonyms_wordnet(word)

        hint = pick_best_synonym(word, candidates)
        if hint:
            target["hint"] = hint
        else:
            missing += 1
            if missing <= 10:
                print(f"  MISSING: {word} (index={i})")

        if (i + 1) % 50 == 0:
            print(f"  Progress: {i + 1}/{len(targets)}")
            time.sleep(0.5)

    return missing


def generate_russian_hints(targets: list[dict], abramov: dict[str, str]) -> int:
    """Add hints to Russian targets from Abramov dictionary, with curated overrides."""
    missing = 0
    missing_words: list[str] = []

    for target in targets:
        word = target["word"]
        # Check curated override first
        if word in RU_HINT_OVERRIDES:
            target["hint"] = RU_HINT_OVERRIDES[word]
            continue
        hint = abramov.get(word)
        if hint:
            target["hint"] = hint
        else:
            missing_words.append(word)
            missing += 1
            if missing <= 5:
                idx = targets.index(target)
                print(f"  MISSING: {word} (index={idx})")

    return missing


def generate_russian_hints_translation(targets: list[dict], missing_words: list[str]) -> int:
    """Fallback: translate RU→EN, find English synonym, translate back to RU."""
    from deep_translator import GoogleTranslator
    from nltk.corpus import wordnet as wn
    import requests as req

    target_map = {t["word"]: t for t in targets}
    still_missing = 0

    for i, word in enumerate(missing_words):
        try:
            # Step 1: Translate RU → EN
            en = GoogleTranslator(source='ru', target='en').translate(word).lower()
            time.sleep(0.15)

            # Step 2: Find English synonym (WordNet primary synset first, then Datamuse)
            synonym = None
            syns = wn.synsets(en)
            if syns:
                primary = syns[0]
                for l in primary.lemmas():
                    name = l.name().lower()
                    if name != en and '_' not in name:
                        synonym = name
                        break

            if not synonym:
                # Try Datamuse as fallback
                try:
                    resp = req.get("https://api.datamuse.com/words",
                                   params={"ml": en, "max": 5}, timeout=5)
                    for r in resp.json():
                        dm_word = r["word"]
                        if " " not in dm_word and dm_word != en:
                            synonym = dm_word
                            break
                except Exception:
                    pass

            if synonym:
                # Step 3: Translate back EN → RU
                ru_syn = GoogleTranslator(source='en', target='ru').translate(synonym).lower()
                time.sleep(0.15)

                # Quality filters: skip if same as input, multi-word, or too long
                if ru_syn != word and ' ' not in ru_syn and len(ru_syn) <= 15 and ru_syn != en:
                    target_map[word]["hint"] = ru_syn
                else:
                    still_missing += 1
            else:
                still_missing += 1
        except Exception:
            still_missing += 1

        if (i + 1) % 50 == 0:
            print(f"  Translation progress: {i + 1}/{len(missing_words)} (missing: {still_missing})")

    return still_missing


def build_bank(dict_path: Path, output_path: Path, language: str, abramov: dict[str, str] | None = None) -> None:
    """Build a word bank with 1000 targets and hints."""
    guessable = load_dictionary(dict_path)
    if len(guessable) < TARGET_COUNT:
        print(f"  WARNING: {dict_path} has only {len(guessable)} words, using all")
        count = len(guessable)
    else:
        count = TARGET_COUNT

    targets = [{"word": guessable[i]} for i in range(count)]

    print(f"  Generating hints for {len(targets)} targets ({language})...")
    if language == "en":
        missing = generate_english_hints(targets)
    else:
        missing = generate_russian_hints(targets, abramov or {})

    bank = {"guessable": guessable, "targets": targets}
    output_path.write_text(
        json.dumps(bank, ensure_ascii=False, indent=2) + "\n",
        encoding="utf-8",
    )

    print(f"  Wrote {output_path}: {len(guessable)} guessable, {len(targets)} targets, {len(targets) - missing} with hints, {missing} missing")


def main():
    print("Loading Abramov Russian synonym dictionary...")
    abramov = load_abramov_dict()

    print("\n=== English word bank ===")
    build_bank(
        RAW_DIR / "english-5letter-freq-sortfreq-wordsonly.txt",
        DATA_DIR / "word-bank.json",
        "en",
    )

    print("\n=== Russian word bank ===")
    build_bank(
        RAW_DIR / "russian-5letter-freq-sortfreq-wordsonly.txt",
        DATA_DIR / "word-bank-ru.json",
        "ru",
        abramov,
    )

    print("\nDone. Run 'npm run solve' to validate solvability.")


if __name__ == "__main__":
    main()