diff --git a/Makefile b/Makefile index 56d9f76b6..d446ad184 100644 --- a/Makefile +++ b/Makefile @@ -43,14 +43,6 @@ build: $(VENV)/.installed ## Build the static site and validate it compiles clea $(MKDOCS) build --strict $(ARGS) || \ (echo "\nError: Build failed. Fix the errors above, then re-run: make build\n Tip: run 'make serve' to preview and identify broken references interactively." && exit 1) -.PHONY: gen-cookbook -gen-cookbook: $(VENV)/.installed ## Generate the cookbook index table (pass TARGET=path/to/section) -ifndef TARGET - $(error TARGET is required. Example: make gen-cookbook TARGET=smart-contracts/cookbook) -endif - $(PYTHON) $(SCRIPTS_DIR)/generate-cookbook-indexes.py $(TARGET) || \ - (echo "\nError: Failed to generate cookbook index for '$(TARGET)'.\n Ensure the path exists under polkadot-docs/ and contains a .nav.yml file." && exit 1) - .PHONY: help help: @echo "Please use \`make ' where is one of" @@ -60,4 +52,3 @@ help: @echo " pass extra flags with ARGS: make serve ARGS='--watch-theme'" @echo " build to build the static site and validate it compiles cleanly (mirrors CI)" @echo " pass extra flags with ARGS: make build ARGS='-d site'" - @echo " gen-cookbook to regenerate the cookbook index table (requires TARGET=path/to/section)" diff --git a/hooks/auto_index_table.py b/hooks/auto_index_table.py new file mode 100644 index 000000000..5c76ef020 --- /dev/null +++ b/hooks/auto_index_table.py @@ -0,0 +1,346 @@ +"""MkDocs hook: auto-generate index tables from nav structure. + +Place one or more marker blocks anywhere in a page: + + + + +Each block is replaced with a generated table on every build. Custom content +outside the markers is left untouched, so multiple blocks and hand-written +sections can coexist freely on the same page. + +Optional YAML config in the opening comment: + + + +Config options: + dir - directory to scan, relative to the page (default: page's own dir) + use a leading / to make it relative to docs_dir instead + columns - columns to include and their display order + (default: [title, difficulty, tools, description]) + available: title, difficulty, tools, description + flat - if true, all pages are collected into a single table with no + section headings (default: false); when false, directory entries + are recursively expanded by following .nav.yml files in + subdirectories + extra_rows - list of extra rows appended after the auto-generated ones + each entry is a dict with: title (raw markdown), description, + tools, and optionally difficulty + overrides - dict keyed by filename (e.g. accounts.md) to override any + field on a specific auto-generated row: title, description, + tools, or difficulty + +Fields used per row: + title - taken from .nav.yml key first (icon prefixes stripped), + falls back to frontmatter title; page is skipped if both missing + description - read from frontmatter short_description first, falls back to + description; page is skipped if both missing + tools - frontmatter field; accepts a list or a comma/semicolon-separated + string; shows N/A if absent + difficulty - from page_badges.tutorial_badge frontmatter; shows N/A if absent + +Note: index.md files are always skipped regardless of nav configuration. +""" + +import os +import re +import yaml + +from mkdocs.utils.meta import get_data + +import logging +log = logging.getLogger('mkdocs') + +_ICON_RE = re.compile(r'^:[a-z0-9_+-]+:\s*') + +BLOCK_RE = re.compile( + r'()' + r'.*?' + r'', + re.DOTALL, +) + +END_MARKER = '' + +DEFAULT_COLUMNS = ['title', 'difficulty', 'tools', 'description'] + +COLUMN_HEADERS = { + 'title': 'Title', + 'difficulty': 'Difficulty', + 'tools': 'Tools', + 'description': 'Description', +} + +ACRONYMS = {'API', 'SDK', 'CLI', 'AI', 'ML', 'CPU', 'GPU', 'EVM', 'PVM', 'NFT', 'DApp'} +DIFFICULTY_MAP = { + 'beginner': '🟢 Beginner', + 'intermediate': '🟡 Intermediate', + 'advanced': '🔴 Advanced', +} + +def on_page_markdown(markdown, page, config, files, **kwargs): + if '', opening, re.DOTALL) + raw_config = yaml_match.group(1).strip() if yaml_match else '' + + cfg = {} + if raw_config: + try: + cfg = yaml.safe_load(raw_config) or {} + except Exception as e: + log.warning(f"auto_index: invalid YAML config in {page.file.src_path}: {e} — using defaults") + + columns = cfg.get('columns', DEFAULT_COLUMNS) + if isinstance(columns, str): + columns = [columns] + unknown = [c for c in columns if c not in COLUMN_HEADERS] + if unknown: + log.warning(f"auto_index: unknown column(s) {unknown} in {page.file.src_path} — falling back to defaults") + columns = DEFAULT_COLUMNS + flat = cfg.get('flat', False) + extra_rows = cfg.get('extra_rows') or [] + overrides = cfg.get('overrides') or {} + + dir_config = cfg.get('dir') + if dir_config: + scan_dir = _resolve(page_dir, str(dir_config), docs_dir) + if not scan_dir: + log.warning(f"auto_index: 'dir: {dir_config}' in {page.file.src_path} resolves outside docs_dir — skipping block") + return match.group(0) + if not os.path.isdir(scan_dir): + log.warning(f"auto_index: 'dir: {dir_config}' in {page.file.src_path} does not exist — skipping block") + return match.group(0) + else: + scan_dir = page_dir + + generated = _build_content(scan_dir, docs_dir, columns, flat, extra_rows, overrides) + inner = f"\n\n{generated}\n" if generated else "\n" + return f"{opening}{inner}{END_MARKER}" + + return BLOCK_RE.sub(replace_block, markdown) + + +def _build_content(scan_dir, docs_dir, columns, flat=False, extra_rows=None, overrides=None): + nav_path = os.path.join(scan_dir, '.nav.yml') + if not os.path.exists(nav_path): + log.warning(f"auto_index: no .nav.yml found in {os.path.relpath(scan_dir, docs_dir)} — table will be empty") + return "" + + header = '| ' + ' | '.join(COLUMN_HEADERS[c] for c in columns) + ' |' + separator = '|' + '|'.join(':----------:' if c == 'difficulty' else '-------' for c in columns) + '|' + + nav_items = _load_nav(nav_path) + + if flat: + rows = [] + for item in nav_items: + if not isinstance(item, dict): + continue + for nav_title, path in item.items(): + if os.path.basename(str(path)) == 'index.md': + continue + resolved = _resolve(scan_dir, str(path), docs_dir) + if not resolved: + continue + nt = _strip_icons(nav_title) + if os.path.isfile(resolved) and resolved.endswith('.md'): + row = _make_row(resolved, docs_dir, columns, nav_title=nt, overrides=overrides) + if row: + rows.append(row) + else: + md_path = resolved.rstrip('/') + '.md' + if os.path.isfile(md_path): + row = _make_row(md_path, docs_dir, columns, nav_title=nt, overrides=overrides) + if row: + rows.append(row) + for er in (extra_rows or []): + row = _make_extra_row(er, columns) + if row: + rows.append(row) + if not rows: + return "" + return "\n".join([header, separator] + rows) + + sections = [] + for item in nav_items: + if not isinstance(item, dict): + continue + for title, path in item.items(): + if os.path.basename(str(path)) == 'index.md': + continue + + resolved = _resolve(scan_dir, str(path), docs_dir) + if not resolved: + continue + + rows = [r for r in (_make_row(f, docs_dir, columns, nav_title=nt, overrides=overrides) for f, nt in _collect_files(resolved, docs_dir)) if r] + + if rows: + sections.append(f"## {_strip_icons(title)}\n") + sections.append(header) + sections.append(separator) + sections.extend(rows) + sections.append("") + + if extra_rows: + extra = [_make_extra_row(er, columns) for er in extra_rows] + extra = [r for r in extra if r] + if extra: + sections.append(header) + sections.append(separator) + sections.extend(extra) + sections.append("") + + return "\n".join(sections) + + +def _make_row(md_path, docs_dir, columns, nav_title=None, overrides=None): + fm = _read_frontmatter(md_path) + ov = (overrides or {}).get(os.path.basename(md_path)) or {} + + title = ov.get('title') or nav_title or (fm.get('title') or '').strip() + description = ov.get('description') or (fm.get('short_description') or fm.get('description') or '').strip() + + if not (title and description): + return None + + page_badges = fm.get('page_badges') or {} + tutorial_badge = (page_badges.get('tutorial_badge') or '').strip() if isinstance(page_badges, dict) else '' + difficulty_raw = ov.get('difficulty', '') + + data = { + 'title': f"[{_escape(title)}]({_to_site_path(md_path, docs_dir)})", + 'difficulty': _difficulty(difficulty_raw or tutorial_badge), + 'tools': _escape(_format_tools(ov.get('tools') or fm.get('tools', ''))), + 'description': _escape(description), + } + + return '| ' + ' | '.join(data[c] for c in columns) + ' |' + + +def _make_extra_row(row_data, columns): + if not isinstance(row_data, dict): + return None + title = str(row_data.get('title', '')).replace('|', r'\|').strip() + description = _escape(str(row_data.get('description', ''))) + if not (title and description): + return None + + data = { + 'title': title, + 'difficulty': _difficulty(row_data.get('difficulty', '')), + 'tools': _escape(_format_tools(row_data.get('tools', ''))), + 'description': description, + } + return '| ' + ' | '.join(data[c] for c in columns) + ' |' + + +def _collect_files(path, docs_dir, nav_title=None): + """Recursively collect (md_path, nav_title) from a path, following .nav.yml files.""" + if os.path.isfile(path) and path.endswith('.md'): + return [(path, nav_title)] + + if os.path.isdir(path): + sub_nav = os.path.join(path, '.nav.yml') + if os.path.exists(sub_nav): + files = [] + for item in _load_nav(sub_nav): + if not isinstance(item, dict): + continue + for item_title, sub_path in item.items(): + if os.path.basename(str(sub_path)) == 'index.md': + continue + resolved = _resolve(path, str(sub_path), docs_dir) + if resolved: + files.extend(_collect_files(resolved, docs_dir, _strip_icons(item_title))) + return files + return [ + (os.path.join(path, f), None) + for f in sorted(os.listdir(path)) + if f.endswith('.md') and f != 'index.md' + ] + + md_path = path.rstrip('/') + '.md' + if os.path.isfile(md_path): + return [(md_path, nav_title)] + + return [] + + +def _difficulty(badge): + badge = str(badge).strip() + if not badge: + return 'N/A' + return DIFFICULTY_MAP.get(badge.lower(), f'⚪ {badge.title()}') + + +def _strip_icons(text): + return _ICON_RE.sub('', str(text)).strip() + + +def _read_frontmatter(path): + try: + with open(path, encoding='utf-8-sig') as f: + _, meta = get_data(f.read()) + return meta or {} + except Exception: + return {} + + +def _load_nav(nav_path): + try: + with open(nav_path, encoding='utf-8') as f: + data = yaml.safe_load(f) or {} + return data if isinstance(data, list) else data.get('nav', []) + except Exception: + return [] + + +def _resolve(base_dir, path, docs_dir): + if path.startswith('/'): + resolved = os.path.normpath(os.path.join(docs_dir, path.lstrip('/'))) + else: + resolved = os.path.normpath(os.path.join(base_dir, path)) + if os.path.commonpath([resolved, os.path.normpath(docs_dir)]) != os.path.normpath(docs_dir): + return None + return resolved + + +def _to_site_path(abs_path, docs_dir): + rel = os.path.relpath(abs_path, docs_dir).replace('\\', '/') + if rel.endswith('.md'): + rel = rel[:-3] + if rel.endswith('/index') or rel == 'index': + return '/' + rel[:-len('index')] + return '/' + rel + '/' + return '/' + rel + + +def _format_tools(tools): + if not tools: + return 'N/A' + parts = tools if isinstance(tools, list) else re.split(r'[;,]', str(tools)) + out = [] + for t in (p.strip() for p in parts if str(p).strip()): + if t.upper() in ACRONYMS: + out.append(t.upper()) + elif t.islower() and ' ' not in t: + out.append(t.capitalize()) + else: + out.append(t) + return ', '.join(out) if out else 'N/A' + + +def _escape(text): + return str(text).replace('|', r'\|').replace('`', r'\`').replace('\n', ' ').strip() diff --git a/mkdocs.yml b/mkdocs.yml index 35ac53282..646eca3e3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -111,8 +111,9 @@ markdown_extensions: # Hooks hooks: - - hooks/glossary_abbreviations.py + - hooks/auto_index_table.py - hooks/footer_nav.py + - hooks/glossary_abbreviations.py - hooks/synthesize_ancestors.py # Plugins diff --git a/scripts/generate_index_pages.py b/scripts/generate_index_pages.py deleted file mode 100644 index 087bbaf4c..000000000 --- a/scripts/generate_index_pages.py +++ /dev/null @@ -1,123 +0,0 @@ -import os -import re - -IGNORE_DIRECTORIES = {".", "node_modules", "js", "images"} - -def parse_frontmatter(file_path): - """ - Extracts title and description from the frontmatter of a Markdown file. - """ - with open(file_path, 'r') as f: - content = f.read() - - frontmatter_match = re.match(r'^---\s*(.*?)\s*---', content, re.DOTALL | re.MULTILINE) - if frontmatter_match: - frontmatter_content = frontmatter_match.group(1) - # Extract title and description from the frontmatter - title = re.search(r'^title:\s*(.+)', frontmatter_content, re.MULTILINE) - description = re.search(r'^description:\s*(.+)', frontmatter_content, re.MULTILINE) - return ( - title.group(1).strip() if title else None, - description.group(1).strip() if description else None - ) - return None, None - -def check_content_after_second_frontmatter(file_path): - """ - Checks if there is any content after the second `---` in the file. - """ - with open(file_path, 'r') as f: - content = f.read() - - parts = re.split(r'^---\s*$', content, flags=re.MULTILINE) - return len(parts) > 2 and parts[2].strip() != '' - -def convert_path(file_path): - """ - Converts a full file path (i.e., 'polkadot-docs/develop/blockchains/deployment/generate-chain-specs.md') - to an absolute path (i.e., '/develop/blockchains/deployment/generate-chain-specs/') - """ - # Remove the 'polkadot-docs' prefix - if file_path.startswith('polkadot-docs'): - file_path = file_path[len('polkadot-docs'):] - - # Remove the '.md' extension - if file_path.endswith('.md'): - file_path = file_path[:-3] - - # Ensure it starts with '/' - return file_path + '/' - -def process_directory(directory): - """ - Processes a directory to create or modify `index.md` based on the rules. - """ - index_path = os.path.join(directory, 'index.md') - - if not os.path.isfile(index_path): - return - - if check_content_after_second_frontmatter(index_path): - return - - in_this_section = [] - - for item in os.listdir(directory): - item_path = os.path.join(directory, item) - - if os.path.isdir(item_path) and os.path.basename(item_path) not in IGNORE_DIRECTORIES: - sub_index_path = os.path.join(item_path, 'index.md') - if os.path.isfile(sub_index_path): - title, description = parse_frontmatter(sub_index_path) - if title or description: - in_this_section.append((title, description, sub_index_path)) - - elif item.endswith('.md') and item != 'index.md': - title, description = parse_frontmatter(item_path) - if title or description: - in_this_section.append((title, description, item_path)) - - if in_this_section: - with open(index_path, 'r+') as f: - content = f.read() - parts = re.split(r'^---\s*$', content, flags=re.MULTILINE) - if len(parts) >= 2: - # Add content below the second `---` - new_content = parts[0] + "---" + parts[1] + "---" + os.linesep + os.linesep - title = re.search(r'^title:\s*(.+)', parts[1], re.MULTILINE) - title = title.group(1).strip() if title else None - - description = re.search(r'^description:\s*(.+)', parts[1], re.MULTILINE) - description = description.group(1).strip() if title else None - - print(description) - - # Preserve any content that might exist after the second `---` - existing_content = parts[2] if len(parts) > 2 else "" - - # Append the new content to the existing content - new_content += f'# {title or ""}' + os.linesep + os.linesep - new_content += f'{description or ""}' + os.linesep + os.linesep - new_content += "## In This Section" + os.linesep + os.linesep - new_content += ":::INSERT_IN_THIS_SECTION:::" - - new_content += existing_content - - # Write the modified content back to the file - f.seek(0) - f.write(new_content) - f.truncate() - - -def main(base_directory): - """ - Main function to iterate over directories and process each. - """ - for root, dirs, files in os.walk(base_directory): - # Exclude directories in-place to prevent walking into them - dirs[:] = [d for d in dirs if d not in IGNORE_DIRECTORIES and not d.startswith('.')] - process_directory(root) - -if __name__ == '__main__': - base_directory = "polkadot-docs" # Replace with the path to your polkadot-docs directory - main(base_directory)