|
|
@@ -0,0 +1,325 @@
|
|
|
+#!/usr/bin/env python3
|
|
|
+"""Generate word-bank.json and word-bank-ru.json with 1000 targets each.
|
|
|
+
|
|
|
+Targets are the first 1000 words from the frequency-sorted guessable list
|
|
|
+(most common words first). IDs are sequential 1..1000.
|
|
|
+
|
|
|
+Uses:
|
|
|
+ - Datamuse API + WordNet for English synonyms
|
|
|
+ - Abramov's Russian synonym dictionary for Russian synonyms
|
|
|
+"""
|
|
|
+
|
|
|
+import json
|
|
|
+import sys
|
|
|
+import time
|
|
|
+from pathlib import Path
|
|
|
+
|
|
|
+RAW_DIR = Path(__file__).parent
|
|
|
+DATA_DIR = RAW_DIR.parent / "data"
|
|
|
+
|
|
|
+TARGET_COUNT = 1000
|
|
|
+
|
|
|
+# ---- English hint overrides (curated for quality) ----
|
|
|
+EN_HINT_OVERRIDES: dict[str, str] = {
|
|
|
+ "about": "around", "there": "present", "which": "that",
|
|
|
+ "other": "different", "would": "could", "these": "those",
|
|
|
+ "while": "during", "being": "living", "where": "location",
|
|
|
+ "still": "quiet", "never": "not ever", "every": "each",
|
|
|
+ "since": "because", "those": "these", "among": "between",
|
|
|
+ "money": "cash", "local": "nearby", "least": "fewest",
|
|
|
+ "month": "april", "river": "stream", "david": "giant",
|
|
|
+ "steve": "stephen", "didnt": "did not", "three": "trio",
|
|
|
+ "years": "decade", "going": "moving",
|
|
|
+ "state": "condition", "games": "sport", "found": "located",
|
|
|
+ "often": "frequent", "south": "southern", "small": "little",
|
|
|
+ "north": "northern", "third": "tertiary", "party": "group",
|
|
|
+ "began": "started", "young": "youth", "final": "last",
|
|
|
+ "given": "provided", "using": "employ", "field": "area",
|
|
|
+ "teams": "squad", "thing": "object", "white": "pale",
|
|
|
+ "march": "parade", "black": "dark", "built": "constructed",
|
|
|
+ "moved": "shifted", "added": "extra", "needs": "requires",
|
|
|
+ "short": "brief", "front": "forward", "seven": "seventh",
|
|
|
+ "title": "name", "comes": "arrives", "hours": "time",
|
|
|
+ "eight": "eighth", "green": "color", "round": "circular",
|
|
|
+ "union": "group", "lives": "exists", "radio": "broadcast",
|
|
|
+ "words": "phrase", "model": "design", "terms": "words",
|
|
|
+ "score": "points", "ready": "prepared", "grand": "large",
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+# ---- Russian hint overrides (curated for the most common words) ----
|
|
|
+RU_HINT_OVERRIDES: dict[str, str] = {
|
|
|
+ "жизнь": "бытие", "слово": "речь", "часть": "доля", "город": "посёлок",
|
|
|
+ "земля": "почва", "право": "закон", "образ": "облик", "закон": "правило",
|
|
|
+ "война": "битва", "голос": "звук", "книга": "роман", "число": "цифра",
|
|
|
+ "народ": "люди", "форма": "фигура", "улица": "дорога", "вечер": "закат",
|
|
|
+ "мысль": "идея", "месяц": "луна", "школа": "учёба", "театр": "сцена",
|
|
|
+ "смысл": "суть", "рынок": "базар", "центр": "середина", "ответ": "отклик",
|
|
|
+ "автор": "писатель", "совет": "наказ", "глава": "лидер", "наука": "знание",
|
|
|
+ "плечо": "рука", "точка": "пятно", "палец": "рука", "метод": "способ",
|
|
|
+ "фильм": "кино", "гость": "визитёр", "район": "зона", "кровь": "рана",
|
|
|
+ "армия": "войско", "класс": "группа", "герой": "смельчак", "спина": "тело",
|
|
|
+ "берег": "пляж", "фирма": "компания", "завод": "фабрика", "повод": "причина",
|
|
|
+ "выход": "уход", "текст": "письмо", "пункт": "место", "линия": "черта",
|
|
|
+ "среда": "окружение", "волос": "прядь", "ветер": "буря", "огонь": "пламя",
|
|
|
+ "страх": "ужас", "сфера": "область", "немец": "германец", "выбор": "отбор",
|
|
|
+ "масса": "вес", "кухня": "готовка", "товар": "продукт", "вывод": "итог",
|
|
|
+ "норма": "правило", "рамка": "граница", "прием": "встреча", "режим": "порядок",
|
|
|
+ "целое": "полное", "доход": "прибыль", "карта": "схема", "акция": "дело",
|
|
|
+ "фраза": "выражение", "толпа": "скопление", "птица": "пернатый", "запах": "аромат",
|
|
|
+ "поезд": "состав", "адрес": "жильё", "лидер": "вождь", "весна": "сезон",
|
|
|
+ "музей": "галерея", "сутки": "день", "еврей": "иудей", "сотня": "сто",
|
|
|
+ "труба": "дымоход", "масло": "жир", "экран": "дисплей", "вагон": "поезд",
|
|
|
+ "сезон": "пора", "длина": "размер", "доска": "дерево", "лодка": "судно",
|
|
|
+ "серия": "ряд", "кулак": "рука", "нефть": "топливо", "кость": "скелет",
|
|
|
+ "взрыв": "удар", "почва": "земля", "хвост": "конец", "строй": "порядок",
|
|
|
+ "отчет": "доклад", "забор": "стена", "пожар": "огонь", "ножка": "нога",
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+def load_dictionary(path: Path) -> list[str]:
|
|
|
+ """Read frequency-sorted word list (one word per line)."""
|
|
|
+ return [line.strip() for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
|
|
+
|
|
|
+
|
|
|
+def load_abramov_dict() -> dict[str, str]:
|
|
|
+ """Load Abramov's Russian synonym dictionary, returning {word: best_synonym}.
|
|
|
+ Downloads from GitHub if not already cached locally."""
|
|
|
+ path = RAW_DIR / "abramov-ru-synonyms.json"
|
|
|
+ if not path.exists():
|
|
|
+ print(" Downloading Abramov Russian synonym dictionary...")
|
|
|
+ import requests
|
|
|
+ url = "https://raw.githubusercontent.com/egorkaru/synonym_dictionary/master/dictionary.json"
|
|
|
+ resp = requests.get(url, timeout=30)
|
|
|
+ resp.raise_for_status()
|
|
|
+ path.write_text(resp.text, encoding="utf-8")
|
|
|
+ data = json.loads(path.read_text(encoding="utf-8"))
|
|
|
+ result: dict[str, str] = {}
|
|
|
+ for entry in data["wordlist"]:
|
|
|
+ name = entry["name"].lower()
|
|
|
+ synonyms = entry.get("synonyms", [])
|
|
|
+ if not synonyms:
|
|
|
+ continue
|
|
|
+ # Pick the best single-word synonym
|
|
|
+ for syn in synonyms:
|
|
|
+ # Clean: remove multi-word alternatives (split by ";"), pick first single word
|
|
|
+ parts = syn.replace(";", ",").split(",")
|
|
|
+ for part in parts:
|
|
|
+ part = part.strip().lower()
|
|
|
+ # Skip multi-word phrases, the word itself, and single letters
|
|
|
+ if " " in part or part == name or len(part) <= 1:
|
|
|
+ continue
|
|
|
+ result[name] = part
|
|
|
+ break
|
|
|
+ if name in result:
|
|
|
+ break
|
|
|
+ print(f" Loaded {len(result)} Russian synonyms from Abramov dictionary")
|
|
|
+ return result
|
|
|
+
|
|
|
+
|
|
|
+def get_english_synonyms_datamuse(word: str) -> list[str]:
|
|
|
+ """Get related words from Datamuse API."""
|
|
|
+ import requests
|
|
|
+ try:
|
|
|
+ resp = requests.get(
|
|
|
+ "https://api.datamuse.com/words",
|
|
|
+ params={"ml": word, "max": 8},
|
|
|
+ timeout=5,
|
|
|
+ )
|
|
|
+ resp.raise_for_status()
|
|
|
+ results = resp.json()
|
|
|
+ return [r["word"].lower() for r in results if " " not in r["word"] and r["word"].lower() != word]
|
|
|
+ except Exception as e:
|
|
|
+ print(f" Datamuse error for '{word}': {e}", file=sys.stderr)
|
|
|
+ return []
|
|
|
+
|
|
|
+
|
|
|
+def get_english_synonyms_wordnet(word: str) -> list[str]:
|
|
|
+ """Get synonyms from WordNet."""
|
|
|
+ from nltk.corpus import wordnet as wn
|
|
|
+ syns = wn.synsets(word)
|
|
|
+ if not syns:
|
|
|
+ return []
|
|
|
+ lemmas = []
|
|
|
+ for s in syns[:10]:
|
|
|
+ for l in s.lemmas():
|
|
|
+ name = l.name().lower()
|
|
|
+ if "_" not in name and name != word and name not in lemmas:
|
|
|
+ lemmas.append(name)
|
|
|
+ return lemmas
|
|
|
+
|
|
|
+
|
|
|
+def pick_best_synonym(word: str, candidates: list[str]) -> str | None:
|
|
|
+ """Pick the best synonym from candidates."""
|
|
|
+ good = [c for c in candidates if len(c) <= 10 and "_" not in c]
|
|
|
+ if not good:
|
|
|
+ return None
|
|
|
+ good.sort(key=len)
|
|
|
+ return good[0]
|
|
|
+
|
|
|
+
|
|
|
+def generate_english_hints(targets: list[dict]) -> int:
|
|
|
+ """Add hints to English targets. Returns count of missing hints."""
|
|
|
+ missing = 0
|
|
|
+ for i, target in enumerate(targets):
|
|
|
+ word = target["word"]
|
|
|
+
|
|
|
+ # Check manual override first
|
|
|
+ if word in EN_HINT_OVERRIDES:
|
|
|
+ target["hint"] = EN_HINT_OVERRIDES[word]
|
|
|
+ continue
|
|
|
+
|
|
|
+ # Try Datamuse
|
|
|
+ candidates = get_english_synonyms_datamuse(word)
|
|
|
+ if not candidates:
|
|
|
+ candidates = get_english_synonyms_wordnet(word)
|
|
|
+
|
|
|
+ hint = pick_best_synonym(word, candidates)
|
|
|
+ if hint:
|
|
|
+ target["hint"] = hint
|
|
|
+ else:
|
|
|
+ missing += 1
|
|
|
+ if missing <= 10:
|
|
|
+ print(f" MISSING: {word} (index={i})")
|
|
|
+
|
|
|
+ if (i + 1) % 50 == 0:
|
|
|
+ print(f" Progress: {i + 1}/{len(targets)}")
|
|
|
+ time.sleep(0.5)
|
|
|
+
|
|
|
+ return missing
|
|
|
+
|
|
|
+
|
|
|
+def generate_russian_hints(targets: list[dict], abramov: dict[str, str]) -> int:
|
|
|
+ """Add hints to Russian targets from Abramov dictionary, with curated overrides."""
|
|
|
+ missing = 0
|
|
|
+ missing_words: list[str] = []
|
|
|
+
|
|
|
+ for target in targets:
|
|
|
+ word = target["word"]
|
|
|
+ # Check curated override first
|
|
|
+ if word in RU_HINT_OVERRIDES:
|
|
|
+ target["hint"] = RU_HINT_OVERRIDES[word]
|
|
|
+ continue
|
|
|
+ hint = abramov.get(word)
|
|
|
+ if hint:
|
|
|
+ target["hint"] = hint
|
|
|
+ else:
|
|
|
+ missing_words.append(word)
|
|
|
+ missing += 1
|
|
|
+ if missing <= 5:
|
|
|
+ idx = targets.index(target)
|
|
|
+ print(f" MISSING: {word} (index={idx})")
|
|
|
+
|
|
|
+ return missing
|
|
|
+
|
|
|
+
|
|
|
+def generate_russian_hints_translation(targets: list[dict], missing_words: list[str]) -> int:
|
|
|
+ """Fallback: translate RU→EN, find English synonym, translate back to RU."""
|
|
|
+ from deep_translator import GoogleTranslator
|
|
|
+ from nltk.corpus import wordnet as wn
|
|
|
+ import requests as req
|
|
|
+
|
|
|
+ target_map = {t["word"]: t for t in targets}
|
|
|
+ still_missing = 0
|
|
|
+
|
|
|
+ for i, word in enumerate(missing_words):
|
|
|
+ try:
|
|
|
+ # Step 1: Translate RU → EN
|
|
|
+ en = GoogleTranslator(source='ru', target='en').translate(word).lower()
|
|
|
+ time.sleep(0.15)
|
|
|
+
|
|
|
+ # Step 2: Find English synonym (WordNet primary synset first, then Datamuse)
|
|
|
+ synonym = None
|
|
|
+ syns = wn.synsets(en)
|
|
|
+ if syns:
|
|
|
+ primary = syns[0]
|
|
|
+ for l in primary.lemmas():
|
|
|
+ name = l.name().lower()
|
|
|
+ if name != en and '_' not in name:
|
|
|
+ synonym = name
|
|
|
+ break
|
|
|
+
|
|
|
+ if not synonym:
|
|
|
+ # Try Datamuse as fallback
|
|
|
+ try:
|
|
|
+ resp = req.get("https://api.datamuse.com/words",
|
|
|
+ params={"ml": en, "max": 5}, timeout=5)
|
|
|
+ for r in resp.json():
|
|
|
+ dm_word = r["word"]
|
|
|
+ if " " not in dm_word and dm_word != en:
|
|
|
+ synonym = dm_word
|
|
|
+ break
|
|
|
+ except Exception:
|
|
|
+ pass
|
|
|
+
|
|
|
+ if synonym:
|
|
|
+ # Step 3: Translate back EN → RU
|
|
|
+ ru_syn = GoogleTranslator(source='en', target='ru').translate(synonym).lower()
|
|
|
+ time.sleep(0.15)
|
|
|
+
|
|
|
+ # Quality filters: skip if same as input, multi-word, or too long
|
|
|
+ if ru_syn != word and ' ' not in ru_syn and len(ru_syn) <= 15 and ru_syn != en:
|
|
|
+ target_map[word]["hint"] = ru_syn
|
|
|
+ else:
|
|
|
+ still_missing += 1
|
|
|
+ else:
|
|
|
+ still_missing += 1
|
|
|
+ except Exception:
|
|
|
+ still_missing += 1
|
|
|
+
|
|
|
+ if (i + 1) % 50 == 0:
|
|
|
+ print(f" Translation progress: {i + 1}/{len(missing_words)} (missing: {still_missing})")
|
|
|
+
|
|
|
+ return still_missing
|
|
|
+
|
|
|
+
|
|
|
+def build_bank(dict_path: Path, output_path: Path, language: str, abramov: dict[str, str] | None = None) -> None:
|
|
|
+ """Build a word bank with 1000 targets and hints."""
|
|
|
+ guessable = load_dictionary(dict_path)
|
|
|
+ if len(guessable) < TARGET_COUNT:
|
|
|
+ print(f" WARNING: {dict_path} has only {len(guessable)} words, using all")
|
|
|
+ count = len(guessable)
|
|
|
+ else:
|
|
|
+ count = TARGET_COUNT
|
|
|
+
|
|
|
+ targets = [{"word": guessable[i]} for i in range(count)]
|
|
|
+
|
|
|
+ print(f" Generating hints for {len(targets)} targets ({language})...")
|
|
|
+ if language == "en":
|
|
|
+ missing = generate_english_hints(targets)
|
|
|
+ else:
|
|
|
+ missing = generate_russian_hints(targets, abramov or {})
|
|
|
+
|
|
|
+ bank = {"guessable": guessable, "targets": targets}
|
|
|
+ output_path.write_text(
|
|
|
+ json.dumps(bank, ensure_ascii=False, indent=2) + "\n",
|
|
|
+ encoding="utf-8",
|
|
|
+ )
|
|
|
+
|
|
|
+ print(f" Wrote {output_path}: {len(guessable)} guessable, {len(targets)} targets, {len(targets) - missing} with hints, {missing} missing")
|
|
|
+
|
|
|
+
|
|
|
+def main():
|
|
|
+ print("Loading Abramov Russian synonym dictionary...")
|
|
|
+ abramov = load_abramov_dict()
|
|
|
+
|
|
|
+ print("\n=== English word bank ===")
|
|
|
+ build_bank(
|
|
|
+ RAW_DIR / "english-5letter-freq-sortfreq-wordsonly.txt",
|
|
|
+ DATA_DIR / "word-bank.json",
|
|
|
+ "en",
|
|
|
+ )
|
|
|
+
|
|
|
+ print("\n=== Russian word bank ===")
|
|
|
+ build_bank(
|
|
|
+ RAW_DIR / "russian-5letter-freq-sortfreq-wordsonly.txt",
|
|
|
+ DATA_DIR / "word-bank-ru.json",
|
|
|
+ "ru",
|
|
|
+ abramov,
|
|
|
+ )
|
|
|
+
|
|
|
+ print("\nDone. Run 'npm run solve' to validate solvability.")
|
|
|
+
|
|
|
+
|
|
|
+if __name__ == "__main__":
|
|
|
+ main()
|