test_lint_spine.py 9.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270
  1. # /// script
  2. # requires-python = ">=3.10"
  3. # dependencies = ["pytest>=8.0"]
  4. # ///
  5. """Tests for lint_spine.py. Run: uv run --with pytest pytest scripts/tests/test_lint_spine.py
  6. The spine under test: a clean spine lints empty; the linter catches exactly the
  7. mechanical defects a prompt is unreliable at — literal placeholders, AD-n id breakage,
  8. AD-n blocks missing required fields, and unpinned Stack versions.
  9. """
  10. import importlib.util
  11. import json
  12. import re
  13. import sys
  14. from pathlib import Path
  15. import pytest
  16. _SPEC = importlib.util.spec_from_file_location(
  17. "lint_spine", Path(__file__).resolve().parent.parent / "lint_spine.py"
  18. )
  19. lint_spine = importlib.util.module_from_spec(_SPEC)
  20. sys.modules["lint_spine"] = lint_spine
  21. _SPEC.loader.exec_module(lint_spine)
  22. CLEAN = """---
  23. name: 'Demo'
  24. ---
  25. ## Invariants & Rules
  26. ### AD-1 — single write path
  27. - **Binds:** all
  28. - **Prevents:** divergent mutation
  29. - **Rule:** state changes only through the command bus
  30. ### AD-2 — layered deps `[ADOPTED]`
  31. - **Binds:** all
  32. - **Prevents:** import cycles
  33. - **Rule:** ui -> app -> domain, never backward
  34. ```mermaid
  35. flowchart LR
  36. A --> B{decision}
  37. ```
  38. ## Stack
  39. | Name | Version |
  40. | --- | --- |
  41. | fastapi | 0.115 |
  42. | pydantic | 2.9 |
  43. """
  44. def cats(result):
  45. return sorted(f["category"] for f in result["findings"])
  46. def test_clean_spine_passes():
  47. result = lint_spine.lint(CLEAN)
  48. assert result["ok"] is True
  49. assert result["total_findings"] == 0
  50. def test_mermaid_braces_not_flagged():
  51. # the {decision} node lives in a fenced block and must not read as a template token
  52. result = lint_spine.lint(CLEAN)
  53. assert "placeholder" not in cats(result)
  54. def test_placeholder_markers_caught():
  55. text = CLEAN.replace("the command bus", "TBD")
  56. result = lint_spine.lint(text)
  57. assert "placeholder" in cats(result)
  58. def test_similar_to_caught():
  59. text = CLEAN.replace("import cycles", "similar to AD-1")
  60. result = lint_spine.lint(text)
  61. assert any("cross-reference" in f["detail"] for f in result["findings"])
  62. def test_unfilled_template_token_caught():
  63. text = CLEAN.replace("single write path", "{decision}")
  64. result = lint_spine.lint(text)
  65. assert any(f["category"] == "placeholder" for f in result["findings"])
  66. def test_duplicate_ad_id_caught():
  67. text = CLEAN.replace("### AD-2 — layered deps `[ADOPTED]`", "### AD-1 — layered deps")
  68. result = lint_spine.lint(text)
  69. assert "ad_id" in cats(result)
  70. def test_non_monotonic_ad_id_caught():
  71. text = CLEAN.replace("### AD-2 — layered deps `[ADOPTED]`", "### AD-5 — layered deps").replace(
  72. "### AD-1 — single write path", "### AD-9 — single write path"
  73. )
  74. result = lint_spine.lint(text)
  75. assert any("non-monotonic" in f["detail"] for f in result["findings"])
  76. def test_missing_field_caught():
  77. text = CLEAN.replace("- **Rule:** state changes only through the command bus\n", "")
  78. result = lint_spine.lint(text)
  79. assert any(f["category"] == "ad_fields" and "rule" in f["detail"] for f in result["findings"])
  80. def test_unpinned_dep_caught():
  81. text = CLEAN.replace("| fastapi | 0.115 |", "| fastapi | |")
  82. result = lint_spine.lint(text)
  83. assert "version_pin" in cats(result)
  84. def test_placeholder_version_caught():
  85. text = CLEAN.replace("| fastapi | 0.115 |", "| fastapi | {pin} |")
  86. result = lint_spine.lint(text)
  87. assert any(f["category"] == "version_pin" and "fastapi" in f["detail"] for f in result["findings"])
  88. def test_no_stack_section_ok():
  89. text = CLEAN.split("## Stack")[0]
  90. result = lint_spine.lint(text)
  91. assert "version_pin" not in cats(result)
  92. def test_stack_skeleton_row_not_version_pinned():
  93. # a leftover {token} name is the placeholder pass's job, not a double-reported version_pin
  94. text = CLEAN.replace("| fastapi | 0.115 |", "| {language / framework} | {pinned version} |")
  95. result = lint_spine.lint(text)
  96. assert "version_pin" not in cats(result)
  97. def test_stack_html_comment_not_parsed_as_row():
  98. text = CLEAN.replace("## Stack\n", "## Stack\n\n<!-- SEED — verified current 2026-06 -->\n")
  99. result = lint_spine.lint(text)
  100. assert "version_pin" not in cats(result)
  101. def test_template_token_is_low_severity():
  102. # a bare {token} can be legitimate brace prose; it is flagged, but low (not high) so the
  103. # mechanical pass stays near-zero false-positive
  104. text = CLEAN.replace("single write path", "{decision}")
  105. result = lint_spine.lint(text)
  106. toks = [f for f in result["findings"] if f["category"] == "placeholder" and "template token" in f["detail"]]
  107. assert toks and all(f["severity"] == "low" for f in toks)
  108. def test_no_frontmatter_body_still_scanned():
  109. text = "## Invariants\n\n### AD-1 — x\n\n- **Binds:** all\n- **Prevents:** drift\n- **Rule:** TBD\n"
  110. result = lint_spine.lint(text)
  111. assert "placeholder" in cats(result) # TBD caught even with no frontmatter
  112. def test_frontmatter_value_with_dashes_not_truncated():
  113. # a value containing '---' must not be read as the closing fence (line-exact close)
  114. text = ("---\nname: 'x'\nscope: 'phase 1 --- phase 2'\n---\n\n"
  115. "## Stack\n\n| Name | Version |\n| --- | --- |\n| fastapi | |\n")
  116. result = lint_spine.lint(text)
  117. assert any(f["category"] == "version_pin" for f in result["findings"]) # read past the inline ---
  118. def test_ad_heading_in_fence_not_counted():
  119. text = (
  120. "---\nname: 'x'\n---\n\n"
  121. "### AD-1 — real\n\n- **Binds:** all\n- **Prevents:** drift\n- **Rule:** do x\n\n"
  122. "## Docs\n\n```text\n### AD-2 — illustrative only, no fields\n```\n"
  123. )
  124. result = lint_spine.lint(text)
  125. assert result["ok"] is True # the fenced AD-2 is not a live AD → no ad_fields/ad_id finding
  126. def test_stack_table_flags_only_the_unpinned_row():
  127. text = ("---\nname: 'x'\n---\n\n## Stack\n\n| Name | Version |\n| --- | --- |\n"
  128. "| fastapi | 0.115 |\n| redis | |\n")
  129. result = lint_spine.lint(text)
  130. pins = [f for f in result["findings"] if f["category"] == "version_pin"]
  131. assert len(pins) == 1 and "redis" in pins[0]["detail"]
  132. def test_stack_table_all_pinned_ok():
  133. text = ("---\nname: 'x'\n---\n\n## Stack\n\n| Name | Version |\n| --- | --- |\n"
  134. "| fastapi | 0.115 |\n")
  135. result = lint_spine.lint(text)
  136. assert "version_pin" not in cats(result)
  137. def test_fenced_stack_rows_not_parsed():
  138. # an illustrative fenced table under ## Stack must not be read as live rows (fences are
  139. # blanked first, like every other pass) — a blank-version row inside a fence is not a finding
  140. text = ("---\nname: 'x'\n---\n\n## Stack\n\n| Name | Version |\n| --- | --- |\n"
  141. "| fastapi | 0.115 |\n\n```text\n| example | |\n```\n")
  142. result = lint_spine.lint(text)
  143. assert "version_pin" not in cats(result)
  144. def test_fenced_stack_heading_not_live():
  145. # a `## Stack` heading shown inside a code fence is not the live Stack section
  146. text = ("---\nname: 'x'\n---\n\n## Docs\n\n```md\n## Stack\n\n| foo | |\n```\n")
  147. result = lint_spine.lint(text)
  148. assert "version_pin" not in cats(result)
  149. def test_renamed_stack_heading_still_scanned():
  150. # the heading match is word-boundary, so a varied `## Stack` heading still counts
  151. text = ("---\nname: 'x'\n---\n\n## Stack & Versions\n\n| Name | Version |\n| --- | --- |\n"
  152. "| redis | |\n")
  153. result = lint_spine.lint(text)
  154. pins = [f for f in result["findings"] if f["category"] == "version_pin"]
  155. assert len(pins) == 1 and "redis" in pins[0]["detail"]
  156. def test_reordered_columns_pair_name_to_version():
  157. # Version-then-Name header: the unpinned row must still be flagged by its real name
  158. text = ("---\nname: 'x'\n---\n\n## Stack\n\n| Version | Name |\n| --- | --- |\n"
  159. "| 0.115 | fastapi |\n| | redis |\n")
  160. result = lint_spine.lint(text)
  161. pins = [f for f in result["findings"] if f["category"] == "version_pin"]
  162. assert len(pins) == 1 and "redis" in pins[0]["detail"]
  163. def test_placeholder_line_number_is_absolute():
  164. # a TBD after a multi-line fence reports its real file line (fence blanked, not collapsed)
  165. text = (
  166. "---\nname: 'x'\n---\n\n"
  167. "## A\n\n"
  168. "```text\nf1\nf2\nf3\n```\n\n"
  169. "TBD here\n"
  170. )
  171. result = lint_spine.lint(text)
  172. ph = next(f for f in result["findings"] if "TBD" in f["detail"])
  173. n = int(re.search(r"line (\d+)", ph["location"]).group(1))
  174. assert n == 13
  175. def test_missing_spine_file_reports_error(tmp_path, capsys):
  176. rc = lint_spine.main(["--workspace", str(tmp_path)])
  177. out = json.loads(capsys.readouterr().out)
  178. assert rc == 0 and out["ok"] is False and "not found" in out["error"]
  179. def test_frontmatter_unfilled_token_caught():
  180. # an unfilled {scope}/{paradigm}/{date} in frontmatter is part of the contract and must lint
  181. text = "---\nname: 'x'\nscope: '{what this spine governs}'\n---\n\n## Invariants\n"
  182. result = lint_spine.lint(text)
  183. fm = [f for f in result["findings"] if f["category"] == "placeholder" and "frontmatter" in f["detail"]]
  184. assert fm and any("template token" in f["detail"] for f in fm)
  185. def test_frontmatter_tbd_caught():
  186. text = "---\nname: 'x'\nstatus: TBD\n---\n\n## Invariants\n"
  187. result = lint_spine.lint(text)
  188. assert any(f["category"] == "placeholder" and "frontmatter" in f["detail"] and "TBD" in f["detail"]
  189. for f in result["findings"])
  190. def test_unreadable_spine_returns_error_not_crash(tmp_path, capsys):
  191. # a spine that exists but can't be UTF-8 decoded must yield error JSON + exit 0, not a traceback
  192. (tmp_path / lint_spine.SPINE).write_bytes(b"\xff\xfe bad bytes not utf-8")
  193. rc = lint_spine.main(["--workspace", str(tmp_path)])
  194. out = json.loads(capsys.readouterr().out)
  195. assert rc == 0 and out["ok"] is False and "could not read" in out["error"]
  196. if __name__ == "__main__":
  197. sys.exit(pytest.main([__file__, "-q"]))