#!/usr/bin/env python3 """ Spec Validator - Validates a feature specification for completeness and quality. Checks that a spec document contains all required sections, uses RFC 2119 keywords correctly, has acceptance criteria in Given/When/Then format, and scores overall completeness from 0-100. Sections checked: - Context, Functional Requirements, Non-Functional Requirements - Acceptance Criteria, Edge Cases, API Contracts, Data Models, Out of Scope Exit codes: 0 = pass, 1 = warnings, 2 = critical (or --strict with score < 80) No external dependencies - uses only Python standard library. """ import argparse import json import re import sys from pathlib import Path from typing import Dict, List, Any, Tuple # Section definitions: (key, display_name, required_header_patterns, weight) SECTIONS = [ ("context", "Context", [r"^##\s+Context"], 10), ("functional_requirements", "Functional Requirements", [r"^##\s+Functional\s+Requirements"], 15), ("non_functional_requirements", "Non-Functional Requirements", [r"^##\s+Non-Functional\s+Requirements"], 10), ("acceptance_criteria", "Acceptance Criteria", [r"^##\s+Acceptance\s+Criteria"], 20), ("edge_cases", "Edge Cases", [r"^##\s+Edge\s+Cases"], 10), ("api_contracts", "API Contracts", [r"^##\s+API\s+Contracts"], 10), ("data_models", "Data Models", [r"^##\s+Data\s+Models"], 10), ("out_of_scope", "Out of Scope", [r"^##\s+Out\s+of\s+Scope"], 10), ("metadata", "Metadata (Author/Date/Status)", [r"\*\*Author:\*\*", r"\*\*Date:\*\*", r"\*\*Status:\*\*"], 5), ] RFC_KEYWORDS = ["MUST", "MUST NOT", "SHOULD", "SHOULD NOT", "MAY"] # Patterns that indicate placeholder/unfilled content PLACEHOLDER_PATTERNS = [ r"\[your\s+name\]", r"\[list\s+reviewers\]", r"\[describe\s+", r"\[input/condition\]", r"\[precondition\]", r"\[action\]", r"\[expected\s+result\]", r"\[feature/capability\]", r"\[operation\]", r"\[threshold\]", r"\[UI\s+component\]", r"\[service\]", r"\[percentage\]", r"\[number\]", r"\[METHOD\]", r"\[endpoint\]", r"\[Name\]", r"\[Entity\s+Name\]", r"\[type\]", r"\[constraints\]", r"\[field\]", r"\[reason\]", ] class SpecValidator: """Validates a spec document for completeness and quality.""" def __init__(self, content: str, file_path: str = ""): self.content = content self.file_path = file_path self.lines = content.split("\n") self.findings: List[Dict[str, Any]] = [] self.section_scores: Dict[str, Dict[str, Any]] = {} def validate(self) -> Dict[str, Any]: """Run all validation checks and return results.""" self._check_sections_present() self._check_functional_requirements() self._check_acceptance_criteria() self._check_edge_cases() self._check_rfc_keywords() self._check_api_contracts() self._check_data_models() self._check_out_of_scope() self._check_placeholders() self._check_traceability() total_score = self._calculate_score() return { "file": self.file_path, "score": total_score, "grade": self._score_to_grade(total_score), "sections": self.section_scores, "findings": self.findings, "summary": self._build_summary(total_score), } def _add_finding(self, severity: str, section: str, message: str): """Record a validation finding.""" self.findings.append({ "severity": severity, # "error", "warning", "info" "section": section, "message": message, }) def _find_section_content(self, header_pattern: str) -> str: """Extract content between a section header and the next ## header.""" in_section = False section_lines = [] for line in self.lines: if re.match(header_pattern, line, re.IGNORECASE): in_section = True continue if in_section and re.match(r"^##\s+", line): break if in_section: section_lines.append(line) return "\n".join(section_lines) def _check_sections_present(self): """Check that all required sections exist.""" for key, name, patterns, weight in SECTIONS: found = False for pattern in patterns: for line in self.lines: if re.search(pattern, line, re.IGNORECASE): found = True break if found: break if found: self.section_scores[key] = {"name": name, "present": True, "score": weight, "max": weight} else: self.section_scores[key] = {"name": name, "present": False, "score": 0, "max": weight} self._add_finding("error", key, f"Missing section: {name}") def _check_functional_requirements(self): """Validate functional requirements format and content.""" content = self._find_section_content(r"^##\s+Functional\s+Requirements") if not content.strip(): return fr_pattern = re.compile(r"-\s+FR-(\d+):") matches = fr_pattern.findall(content) if not matches: self._add_finding("error", "functional_requirements", "No numbered requirements found (expected FR-N: format)") if "functional_requirements" in self.section_scores: self.section_scores["functional_requirements"]["score"] = max( 0, self.section_scores["functional_requirements"]["score"] - 10 ) return fr_count = len(matches) if fr_count < 3: self._add_finding("warning", "functional_requirements", f"Only {fr_count} requirements found. Most features need 3+.") # Check for RFC keywords has_keyword = False for kw in RFC_KEYWORDS: if kw in content: has_keyword = True break if not has_keyword: self._add_finding("warning", "functional_requirements", "No RFC 2119 keywords (MUST/SHOULD/MAY) found.") def _check_acceptance_criteria(self): """Validate acceptance criteria use Given/When/Then format.""" content = self._find_section_content(r"^##\s+Acceptance\s+Criteria") if not content.strip(): return ac_pattern = re.compile(r"###\s+AC-(\d+):") matches = ac_pattern.findall(content) if not matches: self._add_finding("error", "acceptance_criteria", "No numbered acceptance criteria found (expected ### AC-N: format)") if "acceptance_criteria" in self.section_scores: self.section_scores["acceptance_criteria"]["score"] = max( 0, self.section_scores["acceptance_criteria"]["score"] - 15 ) return ac_count = len(matches) # Check Given/When/Then given_count = len(re.findall(r"(?i)\bgiven\b", content)) when_count = len(re.findall(r"(?i)\bwhen\b", content)) then_count = len(re.findall(r"(?i)\bthen\b", content)) if given_count < ac_count: self._add_finding("warning", "acceptance_criteria", f"Found {ac_count} criteria but only {given_count} 'Given' clauses. Each AC needs Given/When/Then.") if when_count < ac_count: self._add_finding("warning", "acceptance_criteria", f"Found {ac_count} criteria but only {when_count} 'When' clauses.") if then_count < ac_count: self._add_finding("warning", "acceptance_criteria", f"Found {ac_count} criteria but only {then_count} 'Then' clauses.") # Check for FR references fr_refs = re.findall(r"\(FR-\d+", content) if not fr_refs: self._add_finding("warning", "acceptance_criteria", "No acceptance criteria reference functional requirements (expected (FR-N) in title).") def _check_edge_cases(self): """Validate edge cases section.""" content = self._find_section_content(r"^##\s+Edge\s+Cases") if not content.strip(): return ec_pattern = re.compile(r"-\s+EC-(\d+):") matches = ec_pattern.findall(content) if not matches: self._add_finding("warning", "edge_cases", "No numbered edge cases found (expected EC-N: format)") elif len(matches) < 3: self._add_finding("warning", "edge_cases", f"Only {len(matches)} edge cases. Consider failure modes for each external dependency.") def _check_rfc_keywords(self): """Check RFC 2119 keywords are used consistently (capitalized).""" # Look for lowercase must/should/may that might be intended as RFC keywords context_content = self._find_section_content(r"^##\s+Functional\s+Requirements") context_content += self._find_section_content(r"^##\s+Non-Functional\s+Requirements") for kw in ["must", "should", "may"]: # Find lowercase usage in requirement-like sentences pattern = rf"(?:system|service|API|endpoint)\s+{kw}\s+" if re.search(pattern, context_content): self._add_finding("warning", "rfc_keywords", f"Found lowercase '{kw}' in requirements. RFC 2119 keywords should be UPPERCASE: {kw.upper()}") def _check_api_contracts(self): """Validate API contracts section.""" content = self._find_section_content(r"^##\s+API\s+Contracts") if not content.strip(): return # Check for at least one endpoint definition has_endpoint = bool(re.search(r"(GET|POST|PUT|PATCH|DELETE)\s+/", content)) if not has_endpoint: self._add_finding("warning", "api_contracts", "No HTTP method + path found (expected e.g., POST /api/endpoint)") # Check for request/response definitions has_interface = bool(re.search(r"interface\s+\w+", content)) if not has_interface: self._add_finding("info", "api_contracts", "No TypeScript interfaces found. Consider defining request/response shapes.") def _check_data_models(self): """Validate data models section.""" content = self._find_section_content(r"^##\s+Data\s+Models") if not content.strip(): return # Check for table format has_table = bool(re.search(r"\|.*\|.*\|", content)) if not has_table: self._add_finding("warning", "data_models", "No table-formatted data models found. Use | Field | Type | Constraints | format.") def _check_out_of_scope(self): """Validate out of scope section.""" content = self._find_section_content(r"^##\s+Out\s+of\s+Scope") if not content.strip(): return os_pattern = re.compile(r"-\s+OS-(\d+):") matches = os_pattern.findall(content) if not matches: self._add_finding("warning", "out_of_scope", "No numbered exclusions found (expected OS-N: format)") elif len(matches) < 2: self._add_finding("info", "out_of_scope", "Only 1 exclusion listed. Consider what was deliberately left out.") def _check_placeholders(self): """Check for unfilled placeholder text.""" placeholder_count = 0 for pattern in PLACEHOLDER_PATTERNS: matches = re.findall(pattern, self.content, re.IGNORECASE) placeholder_count += len(matches) if placeholder_count > 0: self._add_finding("warning", "placeholders", f"Found {placeholder_count} placeholder(s) that need to be filled in (e.g., [your name], [describe ...]).") # Deduct from overall score proportionally for key in self.section_scores: if self.section_scores[key]["present"]: deduction = min(3, self.section_scores[key]["score"]) self.section_scores[key]["score"] = max(0, self.section_scores[key]["score"] - deduction) def _check_traceability(self): """Check that acceptance criteria reference functional requirements.""" ac_content = self._find_section_content(r"^##\s+Acceptance\s+Criteria") fr_content = self._find_section_content(r"^##\s+Functional\s+Requirements") if not ac_content.strip() or not fr_content.strip(): return # Extract FR IDs fr_ids = set(re.findall(r"FR-(\d+)", fr_content)) # Extract FR references from AC ac_fr_refs = set(re.findall(r"FR-(\d+)", ac_content)) unreferenced = fr_ids - ac_fr_refs if unreferenced: unreferenced_list = ", ".join(f"FR-{i}" for i in sorted(unreferenced)) self._add_finding("warning", "traceability", f"Functional requirements without acceptance criteria: {unreferenced_list}") def _calculate_score(self) -> int: """Calculate the total completeness score.""" total = sum(s["score"] for s in self.section_scores.values()) maximum = sum(s["max"] for s in self.section_scores.values()) if maximum == 0: return 0 # Apply finding-based deductions error_count = sum(1 for f in self.findings if f["severity"] == "error") warning_count = sum(1 for f in self.findings if f["severity"] == "warning") base_score = round((total / maximum) * 100) deduction = (error_count * 5) + (warning_count * 2) return max(0, min(100, base_score - deduction)) @staticmethod def _score_to_grade(score: int) -> str: """Convert score to letter grade.""" if score >= 90: return "A" if score >= 80: return "B" if score >= 70: return "C" if score >= 60: return "D" return "F" def _build_summary(self, score: int) -> str: """Build human-readable summary.""" errors = [f for f in self.findings if f["severity"] == "error"] warnings = [f for f in self.findings if f["severity"] == "warning"] infos = [f for f in self.findings if f["severity"] == "info"] lines = [ f"Spec Completeness Score: {score}/100 (Grade: {self._score_to_grade(score)})", f"Errors: {len(errors)}, Warnings: {len(warnings)}, Info: {len(infos)}", "", ] if errors: lines.append("ERRORS (must fix):") for e in errors: lines.append(f" [{e['section']}] {e['message']}") lines.append("") if warnings: lines.append("WARNINGS (should fix):") for w in warnings: lines.append(f" [{w['section']}] {w['message']}") lines.append("") if infos: lines.append("INFO:") for i in infos: lines.append(f" [{i['section']}] {i['message']}") lines.append("") # Section breakdown lines.append("Section Breakdown:") for key, data in self.section_scores.items(): status = "PRESENT" if data["present"] else "MISSING" lines.append(f" {data['name']}: {data['score']}/{data['max']} ({status})") return "\n".join(lines) def format_human(result: Dict[str, Any]) -> str: """Format validation result for human reading.""" lines = [ "=" * 60, "SPEC VALIDATION REPORT", "=" * 60, "", ] if result["file"]: lines.append(f"File: {result['file']}") lines.append("") lines.append(result["summary"]) return "\n".join(lines) def main(): parser = argparse.ArgumentParser( description="Validate a feature specification for completeness and quality.", epilog="Example: python spec_validator.py --file spec.md --strict", ) parser.add_argument( "--file", "-f", required=True, help="Path to the spec markdown file", ) parser.add_argument( "--strict", action="store_true", help="Exit with code 2 if score is below 80", ) parser.add_argument( "--json", action="store_true", dest="json_flag", help="Output results as JSON", ) args = parser.parse_args() file_path = Path(args.file) if not file_path.exists(): print(f"Error: File not found: {file_path}", file=sys.stderr) sys.exit(2) content = file_path.read_text(encoding="utf-8") if not content.strip(): print(f"Error: File is empty: {file_path}", file=sys.stderr) sys.exit(2) validator = SpecValidator(content, str(file_path)) result = validator.validate() if args.json_flag: print(json.dumps(result, indent=2)) else: print(format_human(result)) # Determine exit code score = result["score"] has_errors = any(f["severity"] == "error" for f in result["findings"]) has_warnings = any(f["severity"] == "warning" for f in result["findings"]) if args.strict and score < 80: sys.exit(2) elif has_errors: sys.exit(2) elif has_warnings: sys.exit(1) else: sys.exit(0) if __name__ == "__main__": main()