|
|
@@ -0,0 +1,77 @@
|
|
|
+#!/usr/bin/env python3
|
|
|
+"""Generate word-bank.json and word-bank-ru.json with Gaussian-distributed targets.
|
|
|
+
|
|
|
+Reads frequency-sorted word lists (most common first) from raw/ and produces
|
|
|
+word banks with 100 Gaussian-picked targets per language.
|
|
|
+
|
|
|
+Gaussian parameters:
|
|
|
+ mean = 0 (index of the most frequent word)
|
|
|
+ sigma = 100 (index 100 is at 1σ → ~68% of picks land in the first 100 words)
|
|
|
+
|
|
|
+Target IDs are assigned in ascending index order, so ID 1 is the most common
|
|
|
+selected word and ID 100 is the least common.
|
|
|
+"""
|
|
|
+
|
|
|
+import json
|
|
|
+from pathlib import Path
|
|
|
+import numpy as np
|
|
|
+
|
|
|
+RAW_DIR = Path(__file__).parent
|
|
|
+DATA_DIR = RAW_DIR.parent / "data"
|
|
|
+SEED = 42
|
|
|
+TARGET_COUNT = 100
|
|
|
+SIGMA = 100
|
|
|
+
|
|
|
+def load_dictionary(path: Path) -> list[str]:
|
|
|
+ """Read frequency-sorted word list (one word per line). Preserves order."""
|
|
|
+ words = [line.strip() for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
|
|
+ return words
|
|
|
+
|
|
|
+def select_target_indices(rng: np.random.Generator, dict_len: int, count: int) -> list[int]:
|
|
|
+ """Gaussian sample indices, de-duplicating via rejection.
|
|
|
+ Sorted ascending so ID 1 = most common word."""
|
|
|
+ selected: set[int] = set()
|
|
|
+ while len(selected) < count:
|
|
|
+ samples = np.abs(np.floor(rng.normal(loc=0, scale=SIGMA, size=count * 2)))
|
|
|
+ for s in samples:
|
|
|
+ idx = int(s)
|
|
|
+ if 0 <= idx < dict_len:
|
|
|
+ selected.add(idx)
|
|
|
+ if len(selected) >= count:
|
|
|
+ break
|
|
|
+ return sorted(selected)
|
|
|
+
|
|
|
+def generate_bank(dict_path: Path, output_path: Path) -> None:
|
|
|
+ rng = np.random.default_rng(SEED)
|
|
|
+ guessable = load_dictionary(dict_path)
|
|
|
+ indices = select_target_indices(rng, len(guessable), TARGET_COUNT)
|
|
|
+ targets = [{"id": i + 1, "word": guessable[idx]} for i, idx in enumerate(indices)]
|
|
|
+
|
|
|
+ bank = {"guessable": guessable, "targets": targets}
|
|
|
+ output_path.write_text(
|
|
|
+ json.dumps(bank, ensure_ascii=False, indent=2) + "\n",
|
|
|
+ encoding="utf-8",
|
|
|
+ )
|
|
|
+
|
|
|
+ print(f"Wrote {output_path}: {len(guessable)} guessable, {len(targets)} targets")
|
|
|
+
|
|
|
+ # Sanity checks
|
|
|
+ target_words = [t["word"] for t in targets]
|
|
|
+ assert all(w in guessable for w in target_words), "Target word not in guessable!"
|
|
|
+ assert len(set(target_words)) == TARGET_COUNT, f"Duplicate target words ({len(set(target_words))} unique)!"
|
|
|
+ assert [t["id"] for t in targets] == list(range(1, TARGET_COUNT + 1)), "IDs not contiguous 1..100!"
|
|
|
+ print(f" Top 5: {[targets[i]['word'] for i in range(5)]}")
|
|
|
+
|
|
|
+def main():
|
|
|
+ generate_bank(
|
|
|
+ RAW_DIR / "english-5letter-freq-sortfreq-wordsonly.txt",
|
|
|
+ DATA_DIR / "word-bank.json",
|
|
|
+ )
|
|
|
+ generate_bank(
|
|
|
+ RAW_DIR / "russian-5letter-freq-sortfreq-wordsonly.txt",
|
|
|
+ DATA_DIR / "word-bank-ru.json",
|
|
|
+ )
|
|
|
+ print("Done. Run 'npm run solve' to validate solvability.")
|
|
|
+
|
|
|
+if __name__ == "__main__":
|
|
|
+ main()
|