1import json2import logging3import os4import re5import shutil6import subprocess7from html.parser import HTMLParser8from http.server import HTTPServer, SimpleHTTPRequestHandler9from multiprocessing import Pool10from pathlib import Path11from typing import Any1213import typer14import yaml15from jinja2 import Template16from ruff.__main__ import find_ruff_bin17from slugify import slugify as py_slugify1819logging.basicConfig(level=logging.INFO)2021SUPPORTED_LANGS = {22    "de",23    "en",24    "es",25    "fr",26    "ja",27    "ko",28    "pt",29    "ru",30    "tr",31    "uk",32    "zh",33    "zh-hant",34}353637app = typer.Typer()3839mkdocs_name = "mkdocs.yml"4041non_translated_sections = (42    f"reference{os.sep}",43    "release-notes.md",44    "fastapi-people.md",45    "external-links.md",46    "newsletter.md",47    "management-tasks.md",48    "management.md",49    "contributing.md",50    "translations.md",51)5253docs_path = Path("docs")54en_docs_path = Path("docs/en")55en_config_path: Path = en_docs_path / mkdocs_name56site_path = Path("site").absolute()57zensical_src_path = Path("site_zensical_src").absolute()5859header_pattern = re.compile(r"^(#{1,6}) (.+?)(?:\s*\{\s*(#.*)\s*\})?\s*$")60header_with_permalink_pattern = re.compile(r"^(#{1,6}) (.+?)(\s*\{\s*#.*\s*\})\s*$")61code_block3_pattern = re.compile(r"^\s*```")62code_block4_pattern = re.compile(r"^\s*````")636465# Pattern to match markdown links: [text](url) → text66md_link_pattern = re.compile(r"\[([^\]]+)\]\([^)]+\)")676869def strip_markdown_links(text: str) -> str:70    """Replace markdown links with just their visible text."""71    return md_link_pattern.sub(r"\1", text)727374class VisibleTextExtractor(HTMLParser):75    """Extract visible text from a string with HTML tags."""7677    def __init__(self):78        super().__init__()79        self.text_parts = []8081    def handle_data(self, data):82        self.text_parts.append(data)8384    def extract_visible_text(self, html: str) -> str:85        self.reset()86        self.text_parts = []87        self.feed(html)88        return "".join(self.text_parts).strip()899091def slugify(text: str) -> str:92    return py_slugify(93        text,94        replacements=[95            ("`", ""),  # `dict`s -> dicts96            ("'s", "s"),  # it's -> its97            ("'t", "t"),  # don't -> dont98            ("**", ""),  # **FastAPI**s -> FastAPIs99        ],100    )101102103def get_en_config() -> dict[str, Any]:104    return yaml.unsafe_load(en_config_path.read_text(encoding="utf-8"))105106107def get_lang_paths() -> list[Path]:108    return sorted(docs_path.iterdir())109110111def lang_callback(lang: str | None) -> str | None:112    if lang is None:113        return None114    lang = lang.lower()115    return lang116117118def complete_existing_lang(incomplete: str):119    lang_path: Path120    for lang_path in get_lang_paths():121        if lang_path.is_dir() and lang_path.name.startswith(incomplete):122            yield lang_path.name123124125@app.callback()126def callback() -> None:127    # For MacOS with Cairo128    os.environ["DYLD_FALLBACK_LIBRARY_PATH"] = "/opt/homebrew/lib"129130131@app.command()132def new_lang(lang: str = typer.Argument(..., callback=lang_callback)):133    """134    Generate a new docs translation directory for the language LANG.135    """136    new_path: Path = Path("docs") / lang137    if new_path.exists():138        typer.echo(f"The language was already created: {lang}")139        raise typer.Abort()140    new_path.mkdir()141    new_llm_prompt_path: Path = new_path / "llm-prompt.md"142    new_llm_prompt_path.write_text("", encoding="utf-8")143    print(f"Successfully initialized: {new_path}")144    update_languages()145146147@app.command()148def build_lang(149    lang: str = typer.Argument(150        ..., callback=lang_callback, autocompletion=complete_existing_lang151    ),152) -> None:153    """154    Build the docs for a language.155    """156    build_zensical_lang_to_stage(lang)157    copy_zensical_stage_to_site(lang)158    typer.secho(f"Successfully built docs for: {lang}", color=typer.colors.GREEN)159160161def split_markdown_header(markdown: str) -> tuple[str, str]:162    prefix = ""163    if markdown.startswith("---\n"):164        front_matter_end = markdown.find("\n---\n", 4)165        if front_matter_end != -1:166            front_matter_end += len("\n---\n")167            prefix = markdown[:front_matter_end]168            markdown = markdown[front_matter_end:]169    if markdown.startswith("#"):170        header, separator, body = markdown.partition("\n\n")171        if separator:172            return f"{prefix}{header}", body173    if prefix:174        return prefix.rstrip("\n"), markdown175    return "", markdown176177178def add_markdown_notice(markdown: str, notice: str) -> str:179    header, body = split_markdown_header(markdown)180    if header:181        return f"{header}\n\n{notice}\n\n{body}"182    return f"{notice}\n\n{body}"183184185def is_non_translated_path(path: Path) -> bool:186    src_path = path.as_posix()187    return any(src_path.startswith(section) for section in non_translated_sections)188189190def get_en_url(path: Path) -> str:191    url_path = path.with_suffix("").as_posix()192    if url_path.endswith("/index"):193        url_path = url_path.removesuffix("index")194    elif url_path != "index":195        url_path = f"{url_path}/"196    else:197        url_path = ""198    return f"https://fastapi.tiangolo.com/{url_path}"199200201def get_zensical_theme_language(lang: str) -> str:202    if lang == "zh-hant":203        return "zh-Hant"204    return lang205206207def stage_zensical_docs(lang: str) -> Path:208    lang_docs_path = docs_path / lang / "docs"209    if not lang_docs_path.is_dir():210        typer.echo(f"The language translation doesn't seem to exist yet: {lang}")211        raise typer.Abort()212213    en_docs_source_path = en_docs_path / "docs"214    staged_docs_src_path = zensical_src_path / "docs_src"215    if not staged_docs_src_path.exists():216        shutil.copytree(Path("docs_src"), staged_docs_src_path, dirs_exist_ok=True)217    lang_stage_path = zensical_src_path / lang218    staged_docs_path = lang_stage_path / "content"219    shutil.rmtree(lang_stage_path, ignore_errors=True)220    shutil.copytree(en_docs_source_path, staged_docs_path)221222    missing_translation = (docs_path / "missing-translation.md").read_text(223        encoding="utf-8"224    )225    translation_banner_path = lang_docs_path / "translation-banner.md"226    if not translation_banner_path.is_file():227        translation_banner_path = en_docs_source_path / "translation-banner.md"228    translation_banner = translation_banner_path.read_text(encoding="utf-8")229230    if lang != "en":231        for staged_file in staged_docs_path.rglob("*.md"):232            relative_path = staged_file.relative_to(staged_docs_path)233            translated_file = lang_docs_path / relative_path234            if translated_file.is_file():235                markdown = translated_file.read_text(encoding="utf-8")236                if relative_path.name == "translation-banner.md":237                    staged_file.write_text(markdown, encoding="utf-8")238                    continue239                en_url = get_en_url(relative_path)240                banner = translation_banner.replace("ENGLISH_VERSION_URL", en_url)241                staged_file.write_text(242                    add_markdown_notice(markdown, banner), encoding="utf-8"243                )244            elif not is_non_translated_path(relative_path):245                markdown = staged_file.read_text(encoding="utf-8")246                staged_file.write_text(247                    add_markdown_notice(markdown, missing_translation),248                    encoding="utf-8",249                )250251    shutil.copytree(en_docs_path / "data", lang_stage_path / "data")252    shutil.copytree(en_docs_path / "overrides", lang_stage_path / "overrides")253254    config = get_updated_config_content()255    config["docs_dir"] = "content"256    config["site_dir"] = "site"257    if lang == "en":258        config["site_url"] = "https://fastapi.tiangolo.com/"259    else:260        config["site_url"] = f"https://fastapi.tiangolo.com/{lang}/"261    config.setdefault("theme", {})262    config["theme"]["language"] = get_zensical_theme_language(lang)263    if lang != "en":264        # The root English build owns shared static assets; translated builds should265        # reference those root paths instead of emitting language-local copies.266        if "logo" in config["theme"]:267            config["theme"]["logo"] = "/" + config["theme"]["logo"].lstrip("/")268        if "favicon" in config["theme"]:269            config["theme"]["favicon"] = "/" + config["theme"]["favicon"].lstrip("/")270        config["extra_css"] = ["/" + path.lstrip("/") for path in config["extra_css"]]271        config["extra_javascript"] = [272            "/" + path.lstrip("/") for path in config["extra_javascript"]273        ]274    config_path = lang_stage_path / mkdocs_name275    config_path.write_text(276        yaml.dump(config, sort_keys=False, width=200, allow_unicode=True),277        encoding="utf-8",278    )279    return config_path280281282def build_zensical_config(config_path: Path) -> None:283    subprocess.run(284        ["zensical", "build", "--config-file", config_path.name],285        check=True,286        cwd=config_path.parent,287    )288289290def build_zensical_lang_to_stage(lang: str) -> Path:291    typer.echo(f"Building Zensical docs for: {lang}")292    config_path = stage_zensical_docs(lang)293    config = yaml.unsafe_load(config_path.read_text(encoding="utf-8"))294    build_site_dist_path = config_path.parent / config["site_dir"]295    shutil.rmtree(build_site_dist_path, ignore_errors=True)296    build_zensical_config(config_path)297    return build_site_dist_path298299300def copy_zensical_stage_to_site(lang: str) -> None:301    build_site_dist_path = zensical_src_path / lang / "site"302    if lang == "en":303        dist_path = site_path304    else:305        dist_path = site_path / lang306        shutil.rmtree(dist_path, ignore_errors=True)307    shutil.copytree(build_site_dist_path, dist_path, dirs_exist_ok=True)308309310index_sponsors_template = """311### Keystone Sponsor312313{% for sponsor in sponsors.keystone -%}314<a href="{{ sponsor.url }}" target="_blank" title="{{ sponsor.title }}"><img src="{{ sponsor.img }}"></a>315{% endfor %}316### Gold Sponsors317318{% for sponsor in sponsors.gold -%}319<a href="{{ sponsor.url }}" target="_blank" title="{{ sponsor.title }}"><img src="{{ sponsor.img }}"></a>320{% endfor %}321### Silver Sponsors322323{% for sponsor in sponsors.silver -%}324<a href="{{ sponsor.url }}" target="_blank" title="{{ sponsor.title }}"><img src="{{ sponsor.img }}"></a>325{% endfor %}326327"""328329330def remove_header_permalinks(content: str):331    lines: list[str] = []332    for line in content.split("\n"):333        match = header_with_permalink_pattern.match(line)334        if match:335            hashes, title, *_ = match.groups()336            line = f"{hashes} {title}"337        lines.append(line)338    return "\n".join(lines)339340341def generate_readme_content() -> str:342    en_index = en_docs_path / "docs" / "index.md"343    content = en_index.read_text("utf-8")344    content = remove_header_permalinks(content)  # remove permalinks from headers345    match_pre = re.search(r"</style>\n\n", content)346    match_start = re.search(r"<!-- sponsors -->", content)347    match_end = re.search(r"<!-- /sponsors -->", content)348    sponsors_data_path = en_docs_path / "data" / "sponsors.yml"349    sponsors = yaml.safe_load(sponsors_data_path.read_text(encoding="utf-8"))350    if not (match_start and match_end):351        raise RuntimeError("Couldn't auto-generate sponsors section")352    if not match_pre:353        raise RuntimeError("Couldn't find pre section (<style>) in index.md")354    frontmatter_end = match_pre.end()355    pre_end = match_start.end()356    post_start = match_end.start()357    template = Template(index_sponsors_template)358    message = template.render(sponsors=sponsors)359    pre_content = content[frontmatter_end:pre_end]360    post_content = content[post_start:]361    new_content = pre_content + message + post_content362    # Remove content between <!-- only-mkdocs --> and <!-- /only-mkdocs -->363    new_content = re.sub(364        r"<!-- only-mkdocs -->.*?<!-- /only-mkdocs -->",365        "",366        new_content,367        flags=re.DOTALL,368    )369    return new_content370371372@app.command()373def generate_readme() -> None:374    """375    Generate README.md content from main index.md376    """377    readme_path = Path("README.md")378    old_content = readme_path.read_text("utf-8")379    new_content = generate_readme_content()380    if new_content != old_content:381        print("README.md outdated from the latest index.md")382        print("Updating README.md")383        readme_path.write_text(new_content, encoding="utf-8")384        raise typer.Exit(1)385    print("README.md is up to date ✅")386387388@app.command()389def build_all() -> None:390    """391    Build the full translated docs site into ./site/.392    """393    update_languages()394    shutil.rmtree(site_path, ignore_errors=True)395    shutil.rmtree(zensical_src_path, ignore_errors=True)396    shutil.copytree(Path("docs_src"), zensical_src_path / "docs_src")397    langs = [398        lang.name399        for lang in get_lang_paths()400        if (lang.is_dir() and lang.name in SUPPORTED_LANGS)401    ]402    process_pool_size = min(4, len(langs), os.cpu_count() or 1)403    typer.echo(f"Using process pool size: {process_pool_size}")404    with Pool(process_pool_size) as p:405        p.map(build_zensical_lang_to_stage, langs)406    if "en" in langs:407        copy_zensical_stage_to_site("en")408    for lang in langs:409        if lang != "en":410            copy_zensical_stage_to_site(lang)411    typer.secho("Successfully built all docs", color=typer.colors.GREEN)412413414@app.command()415def update_languages() -> None:416    """417    Update the docs config Languages section including all the available languages.418    """419    old_config = get_en_config()420    updated_config = get_updated_config_content()421    if old_config != updated_config:422        print("docs/en/mkdocs.yml outdated")423        print("Updating docs/en/mkdocs.yml")424        en_config_path.write_text(425            yaml.dump(updated_config, sort_keys=False, width=200, allow_unicode=True),426            encoding="utf-8",427        )428        raise typer.Exit(1)429    print("docs/en/mkdocs.yml is up to date ✅")430431432@app.command()433def serve() -> None:434    """435    A quick server to preview a built site with translations.436437    For development, prefer the command live.438439    This is here only to preview a site with translations already built.440441    Make sure you run the build-all command first.442    """443    typer.echo("Warning: this is a very simple server.")444    typer.echo("For development, use the command live instead.")445    typer.echo("This is here only to preview a site with translations already built.")446    typer.echo("Make sure you run the build-all command first.")447    os.chdir("site")448    server_address = ("", 8008)449    server = HTTPServer(server_address, SimpleHTTPRequestHandler)450    typer.echo("Serving at: http://127.0.0.1:8008")451    server.serve_forever()452453454@app.command()455def live() -> None:456    """457    Serve the English docs with livereload from the source files.458    """459    subprocess.run(460        [461            "zensical",462            "serve",463            "--config-file",464            mkdocs_name,465            "--dev-addr",466            "127.0.0.1:8008",467        ],468        cwd=en_docs_path,469        check=True,470    )471472473def get_updated_config_content() -> dict[str, Any]:474    config = get_en_config()475    languages = [{"en": "/"}]476    new_alternate: list[dict[str, str]] = []477    # Language names sourced from https://quickref.me/iso-639-1478    # Contributors may wish to update or change these, e.g. to fix capitalization.479    language_names_path = Path(__file__).parent / "../docs/language_names.yml"480    local_language_names: dict[str, str] = yaml.safe_load(481        language_names_path.read_text(encoding="utf-8")482    )483    for lang_path in get_lang_paths():484        if lang_path.name in {"en", "em"} or not lang_path.is_dir():485            continue486        if lang_path.name not in SUPPORTED_LANGS:487            # Skip languages that are not yet ready488            continue489        code = lang_path.name490        languages.append({code: f"/{code}/"})491    for lang_dict in languages:492        code = list(lang_dict.keys())[0]493        url = lang_dict[code]494        if code not in local_language_names:495            print(496                f"Missing language name for: {code}, "497                "update it in docs/language_names.yml"498            )499            raise typer.Abort()500        use_name = f"{code} - {local_language_names[code]}"501        new_alternate.append({"link": url, "name": use_name})502    config["extra"]["alternate"] = new_alternate503    return config504505506@app.command()507def ensure_non_translated() -> None:508    """509    Ensure there are no files in the non translatable pages.510    """511    print("Ensuring no non translated pages")512    lang_paths = get_lang_paths()513    error_paths = []514    for lang in lang_paths:515        if lang.name == "en":516            continue517        for non_translatable in non_translated_sections:518            non_translatable_path = lang / "docs" / non_translatable519            if non_translatable_path.exists():520                error_paths.append(non_translatable_path)521    if error_paths:522        print("Non-translated pages found, removing them:")523        for error_path in error_paths:524            print(error_path)525            if error_path.is_file():526                error_path.unlink()527            else:528                shutil.rmtree(error_path)529        raise typer.Exit(1)530    print("No non-translated pages found ✅")531532533@app.command()534def langs_json():535    langs = []536    for lang_path in get_lang_paths():537        if lang_path.is_dir() and lang_path.name in SUPPORTED_LANGS:538            langs.append(lang_path.name)539    print(json.dumps(langs))540541542@app.command()543def generate_docs_src_versions_for_file(file_path: Path) -> None:544    target_versions = ["py39", "py310"]545    full_path_str = str(file_path)546    for target_version in target_versions:547        if f"_{target_version}" in full_path_str:548            logging.info(549                f"Skipping {file_path}, already a version file for {target_version}"550            )551            return552    base_content = file_path.read_text(encoding="utf-8")553    previous_content = {base_content}554    for target_version in target_versions:555        version_result = subprocess.run(556            [557                find_ruff_bin(),558                "check",559                "--target-version",560                target_version,561                "--fix",562                "--unsafe-fixes",563                "-",564            ],565            input=base_content.encode("utf-8"),566            capture_output=True,567        )568        content_target = version_result.stdout.decode("utf-8")569        format_result = subprocess.run(570            [find_ruff_bin(), "format", "-"],571            input=content_target.encode("utf-8"),572            capture_output=True,573        )574        content_format = format_result.stdout.decode("utf-8")575        if content_format in previous_content:576            continue577        previous_content.add(content_format)578        # Determine where the version label should go: in the parent directory579        # name or in the file name, matching the source structure.580        label_in_parent = False581        for v in target_versions:582            if f"_{v}" in file_path.parent.name:583                label_in_parent = True584                break585        if label_in_parent:586            parent_name = file_path.parent.name587            for v in target_versions:588                parent_name = parent_name.replace(f"_{v}", "")589            new_parent = file_path.parent.parent / f"{parent_name}_{target_version}"590            new_parent.mkdir(parents=True, exist_ok=True)591            version_file = new_parent / file_path.name592        else:593            base_name = file_path.stem594            for v in target_versions:595                if base_name.endswith(f"_{v}"):596                    base_name = base_name[: -len(f"_{v}")]597                    break598            version_file = file_path.with_name(f"{base_name}_{target_version}.py")599        logging.info(f"Writing to {version_file}")600        version_file.write_text(content_format, encoding="utf-8")601602603@app.command()604def generate_docs_src_versions() -> None:605    """606    Generate Python version-specific files for all .py files in docs_src.607    """608    docs_src_path = Path("docs_src")609    for py_file in sorted(docs_src_path.rglob("*.py")):610        generate_docs_src_versions_for_file(py_file)611612613@app.command()614def copy_py39_to_py310() -> None:615    """616    For each docs_src file/directory with a _py39 label that has no _py310617    counterpart, copy it with the _py310 label.618    """619    docs_src_path = Path("docs_src")620    # Handle directory-level labels (e.g. app_b_an_py39/)621    for dir_path in sorted(docs_src_path.rglob("*_py39")):622        if not dir_path.is_dir():623            continue624        py310_dir = dir_path.parent / dir_path.name.replace("_py39", "_py310")625        if py310_dir.exists():626            continue627        logging.info(f"Copying directory {dir_path} -> {py310_dir}")628        shutil.copytree(dir_path, py310_dir)629    # Handle file-level labels (e.g. tutorial001_py39.py)630    for file_path in sorted(docs_src_path.rglob("*_py39.py")):631        if not file_path.is_file():632            continue633        # Skip files inside _py39 directories (already handled above)634        if "_py39" in file_path.parent.name:635            continue636        py310_file = file_path.with_name(637            file_path.name.replace("_py39.py", "_py310.py")638        )639        if py310_file.exists():640            continue641        logging.info(f"Copying file {file_path} -> {py310_file}")642        shutil.copy2(file_path, py310_file)643644645@app.command()646def update_docs_includes_py39_to_py310() -> None:647    """648    Update .md files in docs/en/ to replace _py39 includes with _py310 versions.649650    For each include line referencing a _py39 file or directory in docs_src, replace651    the _py39 label with _py310.652    """653    include_pattern = re.compile(r"\{[^}]*docs_src/[^}]*_py39[^}]*\.py[^}]*\}")654    count = 0655    for md_file in sorted(en_docs_path.rglob("*.md")):656        content = md_file.read_text(encoding="utf-8")657        if "_py39" not in content:658            continue659        new_content = include_pattern.sub(660            lambda m: m.group(0).replace("_py39", "_py310"), content661        )662        if new_content != content:663            md_file.write_text(new_content, encoding="utf-8")664            count += 1665            logging.info(f"Updated includes in {md_file}")666    print(f"Updated {count} file(s) ✅")667668669@app.command()670def remove_unused_docs_src() -> None:671    """672    Delete .py files in docs_src that are not included in any .md file under docs/.673    """674    docs_src_path = Path("docs_src")675    # Collect all docs .md content referencing docs_src676    all_docs_content = ""677    for md_file in docs_path.rglob("*.md"):678        all_docs_content += md_file.read_text(encoding="utf-8")679    # Build a set of directory-based package roots (e.g. docs_src/bigger_applications/app_py39)680    # where at least one file is referenced in docs. All files in these directories681    # should be kept since they may be internally imported by the referenced files.682    used_package_dirs: set[Path] = set()683    for py_file in docs_src_path.rglob("*.py"):684        if py_file.name == "__init__.py":685            continue686        rel_path = str(py_file)687        if rel_path in all_docs_content:688            # Walk up from the file's parent to find the package root689            # (a subdirectory under docs_src/<topic>/)690            parts = py_file.relative_to(docs_src_path).parts691            if len(parts) > 2:692                # File is inside a sub-package like docs_src/topic/app_xxx/...693                package_root = docs_src_path / parts[0] / parts[1]694                used_package_dirs.add(package_root)695    removed = 0696    for py_file in sorted(docs_src_path.rglob("*.py")):697        if py_file.name == "__init__.py":698            continue699        # Build the relative path as it appears in includes (e.g. docs_src/first_steps/tutorial001.py)700        rel_path = str(py_file)701        if rel_path in all_docs_content:702            continue703        # If this file is inside a directory-based package where any sibling is704        # referenced, keep it (it's likely imported internally).705        parts = py_file.relative_to(docs_src_path).parts706        if len(parts) > 2:707            package_root = docs_src_path / parts[0] / parts[1]708            if package_root in used_package_dirs:709                continue710        # Check if the _an counterpart (or non-_an counterpart) is referenced.711        # If either variant is included, keep both.712        # Handle both file-level _an (tutorial001_an.py) and directory-level _an713        # (app_an/main.py)714        counterpart_found = False715        full_path_str = str(py_file)716        if "_an" in py_file.stem:717            # This is an _an file, check if the non-_an version is referenced718            counterpart = full_path_str.replace(719                f"/{py_file.stem}", f"/{py_file.stem.replace('_an', '', 1)}"720            )721            if counterpart in all_docs_content:722                counterpart_found = True723        else:724            # This is a non-_an file, check if there's an _an version referenced725            # Insert _an before any version suffix or at the end of the stem726            stem = py_file.stem727            for suffix in ("_py39", "_py310"):728                if suffix in stem:729                    an_stem = stem.replace(suffix, f"_an{suffix}", 1)730                    break731            else:732                an_stem = f"{stem}_an"733            counterpart = full_path_str.replace(f"/{stem}.", f"/{an_stem}.")734            if counterpart in all_docs_content:735                counterpart_found = True736        # Also check directory-level _an counterparts737        if not counterpart_found:738            parent_name = py_file.parent.name739            if "_an" in parent_name:740                counterpart_parent = parent_name.replace("_an", "", 1)741                counterpart_dir = str(py_file).replace(742                    f"/{parent_name}/", f"/{counterpart_parent}/"743                )744                if counterpart_dir in all_docs_content:745                    counterpart_found = True746            else:747                # Try inserting _an into parent directory name748                for suffix in ("_py39", "_py310"):749                    if suffix in parent_name:750                        an_parent = parent_name.replace(suffix, f"_an{suffix}", 1)751                        break752                else:753                    an_parent = f"{parent_name}_an"754                counterpart_dir = str(py_file).replace(755                    f"/{parent_name}/", f"/{an_parent}/"756                )757                if counterpart_dir in all_docs_content:758                    counterpart_found = True759        if counterpart_found:760            continue761        logging.info(f"Removing unused file: {py_file}")762        py_file.unlink()763        removed += 1764    # Clean up directories that are empty or only contain __init__.py / __pycache__765    for dir_path in sorted(docs_src_path.rglob("*"), reverse=True):766        if not dir_path.is_dir():767            continue768        remaining = [769            f770            for f in dir_path.iterdir()771            if f.name != "__pycache__" and f.name != "__init__.py"772        ]773        if not remaining:774            logging.info(f"Removing empty/init-only directory: {dir_path}")775            shutil.rmtree(dir_path)776    print(f"Removed {removed} unused file(s) ✅")777778779@app.command()780def add_permalinks_page(path: Path, update_existing: bool = False):781    """782    Add or update header permalinks in specific page of En docs.783    """784785    if not path.is_relative_to(en_docs_path / "docs"):786        raise RuntimeError(f"Path must be inside {en_docs_path}")787    rel_path = path.relative_to(en_docs_path / "docs")788789    # Skip excluded sections790    if str(rel_path).startswith(non_translated_sections):791        return792793    visible_text_extractor = VisibleTextExtractor()794    updated_lines = []795    in_code_block3 = False796    in_code_block4 = False797    permalinks = set()798799    with path.open("r", encoding="utf-8") as f:800        lines = f.readlines()801802    for line in lines:803        # Handle codeblocks start and end804        if not (in_code_block3 or in_code_block4):805            if code_block4_pattern.match(line):806                in_code_block4 = True807            elif code_block3_pattern.match(line):808                in_code_block3 = True809        else:810            if in_code_block4 and code_block4_pattern.match(line):811                in_code_block4 = False812            elif in_code_block3 and code_block3_pattern.match(line):813                in_code_block3 = False814815        # Process Headers only outside codeblocks816        if not (in_code_block3 or in_code_block4):817            match = header_pattern.match(line)818            if match:819                hashes, title, _permalink = match.groups()820                if (not _permalink) or update_existing:821                    slug = slugify(822                        visible_text_extractor.extract_visible_text(823                            strip_markdown_links(title)824                        )825                    )826                    if slug in permalinks:827                        # If the slug is already used, append a number to make it unique828                        count = 1829                        original_slug = slug830                        while slug in permalinks:831                            slug = f"{original_slug}_{count}"832                            count += 1833                    permalinks.add(slug)834835                    line = f"{hashes} {title} {{ #{slug} }}\n"836837        updated_lines.append(line)838839    with path.open("w", encoding="utf-8") as f:840        f.writelines(updated_lines)841842843@app.command()844def add_permalinks_pages(pages: list[Path], update_existing: bool = False) -> None:845    """846    Add or update header permalinks in specific pages of En docs.847    """848    for md_file in pages:849        add_permalinks_page(md_file, update_existing=update_existing)850851852@app.command()853def add_permalinks(update_existing: bool = False) -> None:854    """855    Add or update header permalinks in all pages of En docs.856    """857    for md_file in en_docs_path.rglob("*.md"):858        add_permalinks_page(md_file, update_existing=update_existing)859860861if __name__ == "__main__":862    app()