#!/usr/bin/env python3 """Print only the doc sections a task needs. usage: ai-context.py list | [...] | check | --version Topics live in wiki/ai-context.json. A source is a whole file or one exact heading (the section runs to the next heading of the same or higher level). check also fails on orphans, so docs cannot silently drift out of the router: wiki pages no topic loads, history files missing from the router index, and nested AGENTS.md files no topic loads, opted-in indexed pages no instructions name, and relative Markdown links are checked. List bulky trees that hold no instructions under "skipDirs" (exact paths or case-sensitive fnmatch globs) to keep check from walking them; the wiki is always walked. List deliberate routing exceptions under "unrouted" in the config as exact paths or case-sensitive fnmatch globs, each with a reason. Keys match themselves and other names via fnmatch, regardless of existence; use wiki/file[[]1].md to exempt the literal wiki/file[1].md without matching wiki/file1.md. Keys normalize backslashes to /; * also matches /. Copy this file to /scripts/ai-context.py; the repo root is its parent's parent. A symlink outside a scripts/ directory (e.g. on PATH) whose own parent's parent has no wiki/ai-context.json uses its target's parent's parent, unless AI_CONTEXT_CONFIG is set; only runs that would fail on a missing config change. For a shared installation, set AI_CONTEXT_ROOT and optionally AI_CONTEXT_CONFIG (default: /wiki/ai-context.json). Explicit relative overrides resolve from the invocation cwd independently; relative paths inside the config resolve from ROOT. Empty or unset overrides use the defaults. No dependencies beyond the Python 3.8+ standard library. """ import fnmatch, json, os, re, sys from urllib.parse import unquote from html import unescape from html.entities import html5 VERSION = '0.2.1' # x-release-please-version def script_root(): """The script's parent's parent, or its symlink target's when that is the only way to a config.""" path = os.path.abspath(__file__) root = os.path.dirname(os.path.dirname(path)) if (os.path.islink(path) and not os.environ.get('AI_CONTEXT_CONFIG') and os.path.basename(os.path.dirname(path)).lower() != 'scripts' and not os.path.exists(os.path.join(root, 'wiki', 'ai-context.json'))): root = os.path.dirname(os.path.dirname(os.path.realpath(path))) return root ROOT = os.path.realpath(os.environ.get('AI_CONTEXT_ROOT') or script_root()) CFG_PATH = os.path.realpath(os.environ.get('AI_CONTEXT_CONFIG') or os.path.join(ROOT, 'wiki', 'ai-context.json')) HEAD = re.compile(r'^(#{1,6})\s+(.+?)\s*#*\s*$') FENCE = re.compile(r'^\s{0,3}(`{3,}|~{3,})') LINK_FENCE = re.compile(r'^ {0,3}(`{3,}|~{3,})') BLOCK_HTML = re.compile(r'^ {0,3}|$)', re.I) THEMATIC = re.compile(r' {0,3}(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})') LIST_MARKER = re.compile(r'^ {0,3}(?:[-+*]|[0-9]{1,9}[.)])(?:[ \t]+|$)') CONTAINER_LEAD = re.compile(r'(?:[ \t>]|(?:[-+*]|[0-9]{1,9}[.)])(?=[ \t]|$))*') BLOCK_HEADING = re.compile(r'^ {0,3}#{1,6}(?:[ \t]|$)') TITLE = r'(?:"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\'|\((?:\\.|[^)\\])*\))' HTML_ATTRIBUTE = r"[A-Za-z_:][A-Za-z0-9_.:-]*(?:[ \t\r\n]*=[ \t\r\n]*(?:[^ \t\r\n\"'=<>`]+|'[^']*'|\"[^\"]*\"))?" HTML_TAG = re.compile(r'|<[A-Za-z][A-Za-z0-9-]*(?:[ \t\r\n]+' + HTML_ATTRIBUTE + r')*[ \t\r\n]*/?>') PUNCTUATION = '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~' BACKTICKS = re.compile(r'`+') BLANK_LINE = re.compile(r'\n[ \t]*\n') WHITESPACE = re.compile(r'[ \t\r\n]*') INLINE_END = re.compile(r'(?:\s+' + TITLE + r')?\s*\)') AUTOLINK = re.compile(r'<(?:[A-Za-z][A-Za-z0-9+.-]{1,31}:[^<>\x00-\x20]*|[A-Za-z0-9.!#$%&\'*+/=?^_`{|}~-]+@[A-Za-z0-9.-]+)>') INLINE_HTML = re.compile(r')[\s\S])*-->|<\?[\s\S]*?\?>|||' + HTML_TAG.pattern) DEFINITION_START = re.compile(r'^ {0,3}\[((?:\\.|[^\]\\])+)\]:') DEFINITION_END = re.compile(r'(?:(?=[ \t\n])[ \t]*(?:\n[ \t]*)?' + TITLE + r')?[ \t]*(?:\n|$)') def load_cfg(): if not os.path.exists(CFG_PATH): sys.exit(f'missing {CFG_PATH}; copy this script to /scripts/ and create wiki/ai-context.json, or set AI_CONTEXT_ROOT and AI_CONTEXT_CONFIG for a shared installation') with open(CFG_PATH, encoding='utf-8') as f: return json.load(f) def headings(lines): """Yield (index, level, text), skipping headings inside fenced code (e.g. shell comments).""" fence = None for i, ln in enumerate(lines): m = FENCE.match(ln) if m: mark = m.group(1) if fence is None: fence = mark[0] * len(mark) elif mark.startswith(fence): fence = None continue if fence is None and (h := HEAD.match(ln)): yield i, len(h.group(1)), h.group(2) def section(src): path = os.path.realpath(os.path.join(ROOT, src['path'].replace('\\', '/'))) if not path.startswith(os.path.join(ROOT, '')): raise ValueError(f'{src["path"]}: outside repo') with open(path, encoding='utf-8') as f: lines = f.read().split('\n') if 'heading' not in src: return '\n'.join(lines).strip() hs = list(headings(lines)) hits = [(i, lv) for i, lv, t in hs if t == src['heading']] if len(hits) != 1: # fail closed: a renamed or duplicated heading must not load the wrong text raise ValueError(f'{src["path"]}: heading {src["heading"]!r} found {len(hits)}x') start, level = hits[0] end = next((i for i, lv, _ in hs if i > start and lv <= level), len(lines)) return '\n'.join(lines[start:end]).strip() def render(cfg, name): out = [] for s in cfg['topics'][name]['sources']: where = s['path'] + (f' ยง {s["heading"]}' if 'heading' in s else '') out.append(f'\n{section(s)}') return '\n\n'.join(out) def indexed_prefixes(indexed): """The literal text each indexed pattern starts with; every match starts with it.""" return [os.path.normcase(re.split(r'[*?\[]', p, maxsplit=1)[0]) for p in indexed] def may_hold_indexed(rel_dir, prefixes): """Whether a file below repo-relative rel_dir could match an indexed pattern.""" path = os.path.normcase(rel_dir + '/') return any(path.startswith(p) or p.startswith(path) for p in prefixes) def skip_patterns(cfg): """Configured skipDirs as forward-slash patterns, or None when the value is invalid.""" value = cfg.get('skipDirs', []) if not isinstance(value, list) or any(not isinstance(p, str) for p in value): return None return [p.replace('\\', '/').rstrip('/') for p in value] def skipped_dir(rel, patterns, wiki, traversing=False): """Whether a directory is skipped; traversal may retain the path to the wiki.""" if (rel == wiki or rel.startswith(wiki + '/') or (traversing and wiki.startswith(rel + '/'))): return False parts = rel.split('/') return any(path == p or fnmatch.fnmatchcase(path, p) for path in ('/'.join(parts[:i]) for i in range(1, len(parts) + 1)) for p in patterns) def child_dir(base, name): """The repo-relative path of directory name under walk directory base.""" return name if base == '.' else f'{base}/{name}' def orphans(cfg, reference_text=''): """Docs that exist on disk but that no agent can reach through the router.""" indexed = cfg.get('indexed', []) if not isinstance(indexed, list) or any(not isinstance(p, str) for p in indexed): return ['indexed: expected a list of strings'] skip_dirs = skip_patterns(cfg) if skip_dirs is None: return ['skipDirs: expected a list of strings'] wiki = cfg.get('wikiDir', 'wiki') history = cfg.get('historyDir', f'{wiki}/history') router = cfg.get('routerFile', f'{wiki}/README.md') root_file = cfg.get('rootInstructions', 'AGENTS.md') routed = {s['path'].replace('\\', '/') for t in cfg['topics'].values() for s in t['sources']} allowed = {p.replace('\\', '/'): reason for p, reason in cfg.get('unrouted', {}).items()} try: router_text = section({'path': router}) except (ValueError, OSError): router_text = '' # link_problems reports the invalid or unreadable router. reference_text += '\n' + router_text bad = [] skip = {'.git', 'node_modules', 'dist', 'build', 'vendor', '.venv', 'target'} prefixes = indexed_prefixes(indexed) for d, dirs, files in os.walk(ROOT): base = os.path.relpath(d, ROOT).replace(os.sep, '/') dirs[:] = [x for x in dirs if x != '.git' and ((x not in skip and not x.startswith('.')) or may_hold_indexed(child_dir(base, x), prefixes)) and not skipped_dir(child_dir(base, x), skip_dirs, wiki, traversing=True)] if base != '.' and skipped_dir(base, skip_dirs, wiki): continue for f in files: rel = os.path.relpath(os.path.join(d, f), ROOT).replace(os.sep, '/') if not f.endswith('.md'): continue reasons = [reason for pattern, reason in allowed.items() if rel == pattern or fnmatch.fnmatchcase(rel, pattern)] if (any(fnmatch.fnmatch(rel, pattern) for pattern in indexed) and rel not in routed and rel not in reference_text and not any(isinstance(reason, str) and reason.strip() for reason in reasons)): bad.append(f'{rel}: indexed page not named (name it in a topic page, route it, or list it under "unrouted" with a reason)') if (reasons or any(x in skip or x.startswith('.') for x in rel.split('/')[:-1])): continue if rel.startswith(history + '/'): if os.path.relpath(rel, wiki).replace(os.sep, '/') not in router_text and rel not in router_text: bad.append(f'{rel}: history file not indexed in {router}') elif rel.startswith(wiki + '/'): if rel != router and rel not in routed: bad.append(f'{rel}: wiki page no topic loads (map it, or list it under "unrouted" with a reason)') elif f == 'AGENTS.md' and rel != root_file and rel not in routed: bad.append(f'{rel}: scoped instructions no topic loads') return bad MARKDOWN_ESCAPE = re.compile(r'\\([' + re.escape(PUNCTUATION) + r'])|&(?:\#(?:[xX][0-9A-Fa-f]{1,6}|[0-9]{1,7})|[A-Za-z][A-Za-z0-9]{1,31});') def decode_markdown_token(match): """Decode a Markdown escape or a complete, recognized character reference.""" token = match.group() if match.group(1): return match.group(1) return unescape(token) if token.startswith('&#') or token[1:] in html5 else token def markdown_unescape(value): return MARKDOWN_ESCAPE.sub(decode_markdown_token, value) def destination(text, start, end=None): """Read an angle-wrapped or balanced bare Markdown destination.""" end = len(text) if end is None else end start = WHITESPACE.match(text, start, end).end() angle = text[start:start + 1] == '<' pos, depth = start + angle, 0 begin = pos while pos < end: char = text[pos] if char == '\\' and pos + 1 < end and text[pos + 1] in PUNCTUATION: pos += 2 continue if angle: if char == '>': return text[begin:pos], pos + 1 if char in '<\n': break else: if char.isspace() or (char == ')' and depth == 0): return (text[begin:pos], pos) if depth == 0 else None if char == '(': depth += 1 elif char == ')': depth -= 1 pos += 1 if not angle and depth == 0: return text[begin:pos], pos return None def reference_definition(text): """Read one complete definition and its exact consumed line count.""" match = DEFINITION_START.match(text) found = destination(text, match.end()) if match and valid_label(match[1]) else None # Only <> may be empty; a definition needs a bare destination. if not found or (not found[0] and not text[:found[1]].endswith('>')) or BLANK_LINE.search(text, 0, found[1]): return None tail = DEFINITION_END.match(text, found[1], len(text)) if tail: end = tail.end() return match[1], found[0], text.count('\n', 0, end) + (text[end - 1:end] != '\n') return None def indent_prefix(text, minimum=None, start=0): """Return consumed characters and indentation columns without expanding text.""" pos, column = 0, start while pos < len(text) and text[pos] in ' \t' and (minimum is None or column - start < minimum): column += 1 if text[pos] == ' ' else 4 - column % 4 pos += 1 return pos, column - start def list_padding(content, marker, base=0): """Return marker prefix, list indentation, and any partially consumed tab.""" start = re.match(r'^ {0,3}(?:[-+*]|[0-9]{1,9}[.)])', content).end() padding, columns = indent_prefix(content[start:marker.end()], start=base + start) if columns <= 4: return marker.end(), base + start + max(columns, 1), base + start + columns, 0 first_columns = indent_prefix(content[start:start + 1], start=base + start)[1] return start + 1, base + start + 1, base + start + first_columns, first_columns - 1 def list_interrupts(content, marker): """A new list interrupts a paragraph only with content and start number 1.""" if not content[marker.end():].strip(): return False ordered = re.match(r'([0-9]{1,9})[.)]', content.lstrip()) return not ordered or ordered[1].lstrip('0') == '1' def title_continuations(lines): """A title continuation cannot consume a line starting a new block.""" for line in lines: fence = LINK_FENCE.match(line) if (BLOCK_HEADING.match(line) or THEMATIC.fullmatch(line) or LIST_MARKER.match(line) or re.fullmatch(r' {0,3}(?:=+|-+)[ \t]*', line) or (fence and (fence[1][0] == '~' or '`' not in line[fence.end():])) or BLOCK_HTML.match(line) or re.match(r'^ {0,3}<(?:pre|script|style|textarea)(?:\s|>|$)', line, re.I) or re.match(r'^ {0,3}(?:' if content.lstrip().startswith('