1import re2from typing import TypedDict34CODE_INCLUDE_RE = re.compile(r"^\{\*\s*(\S+)\s*(.*)\*\}$")5CODE_INCLUDE_PLACEHOLDER = "<CODE_INCLUDE>"67HEADER_WITH_PERMALINK_RE = re.compile(r"^(#{1,6}) (.+?)(\s*\{\s*#.*\s*\})?\s*$")8HEADER_LINE_RE = re.compile(r"^(#{1,6}) (.+?)(?:\s*\{\s*(#.*)\s*\})?\s*$")910TIANGOLO_COM = "https://fastapi.tiangolo.com"11ASSETS_URL_PREFIXES = ("/img/", "/css/", "/js/")1213MARKDOWN_LINK_RE = re.compile(14    r"(?<!\\)(?<!\!)"  # not an image ![...] and not escaped \[...]15    r"\[(?P<text>.*?)\]"  # link text (non-greedy)16    r"\("17    r"(?P<url>[^)\s]+)"  # url (no spaces and `)`)18    r'(?:\s+["\'](?P<title>.*?)["\'])?'  # optional title in "" or ''19    r"\)"20    r"(?:\{(?P<attrs>[^}]*)\})?"  # optional attributes in {}21)2223HTML_LINK_RE = re.compile(r"<a\s+[^>]*>.*?</a>")24HTML_LINK_TEXT_RE = re.compile(r"<a\b([^>]*)>(.*?)</a>")25HTML_LINK_OPEN_TAG_RE = re.compile(r"<a\b([^>]*)>")26HTML_ATTR_RE = re.compile(r'(\w+)\s*=\s*([\'"])(.*?)\2')2728CODE_BLOCK_LANG_RE = re.compile(r"^`{3,4}([\w-]*)", re.MULTILINE)2930SLASHES_COMMENT_RE = re.compile(31    r"^(?P<code>.*?)(?P<comment>(?:(?<= )// .*)|(?:^// .*))?$"32)3334HASH_COMMENT_RE = re.compile(r"^(?P<code>.*?)(?P<comment>(?:(?<= )# .*)|(?:^# .*))?$")353637class CodeIncludeInfo(TypedDict):38    line_no: int39    line: str404142class HeaderPermalinkInfo(TypedDict):43    line_no: int44    hashes: str45    title: str46    permalink: str474849class MarkdownLinkInfo(TypedDict):50    line_no: int51    url: str52    text: str53    title: str | None54    attributes: str | None55    full_match: str565758class HTMLLinkAttribute(TypedDict):59    name: str60    quote: str61    value: str626364class HtmlLinkInfo(TypedDict):65    line_no: int66    full_tag: str67    attributes: list[HTMLLinkAttribute]68    text: str697071class MultilineCodeBlockInfo(TypedDict):72    lang: str73    start_line_no: int74    content: list[str]757677# Code includes78# --------------------------------------------------------------------------------------798081def extract_code_includes(lines: list[str]) -> list[CodeIncludeInfo]:82    """83    Extract lines that contain code includes.8485    Return list of CodeIncludeInfo, where each dict contains:86    - `line_no` - line number (1-based)87    - `line` - text of the line88    """8990    includes: list[CodeIncludeInfo] = []91    for line_no, line in enumerate(lines, start=1):92        if CODE_INCLUDE_RE.match(line):93            includes.append(CodeIncludeInfo(line_no=line_no, line=line))94    return includes959697def replace_code_includes_with_placeholders(text: list[str]) -> list[str]:98    """99    Replace code includes with placeholders.100    """101102    modified_text = text.copy()103    includes = extract_code_includes(text)104    for include in includes:105        modified_text[include["line_no"] - 1] = CODE_INCLUDE_PLACEHOLDER106    return modified_text107108109def replace_placeholders_with_code_includes(110    text: list[str], original_includes: list[CodeIncludeInfo]111) -> list[str]:112    """113    Replace code includes placeholders with actual code includes from the original (English) document.114    Fail if the number of placeholders does not match the number of original includes.115    """116117    code_include_lines = [118        line_no119        for line_no, line in enumerate(text)120        if line.strip() == CODE_INCLUDE_PLACEHOLDER121    ]122123    if len(code_include_lines) != len(original_includes):124        raise ValueError(125            "Number of code include placeholders does not match the number of code includes "126            "in the original document "127            f"({len(code_include_lines)} vs {len(original_includes)})"128        )129130    modified_text = text.copy()131    for i, line_no in enumerate(code_include_lines):132        modified_text[line_no] = original_includes[i]["line"]133134    return modified_text135136137# Header permalinks138# --------------------------------------------------------------------------------------139140141def extract_header_permalinks(lines: list[str]) -> list[HeaderPermalinkInfo]:142    """143    Extract list of header permalinks from the given lines.144145    Return list of HeaderPermalinkInfo, where each dict contains:146    - `line_no` - line number (1-based)147    - `hashes` - string of hashes representing header level (e.g., "###")148    - `permalink` - permalink string (e.g., "{#permalink}")149    """150151    headers: list[HeaderPermalinkInfo] = []152    in_code_block3 = False153    in_code_block4 = False154155    for line_no, line in enumerate(lines, start=1):156        if not (in_code_block3 or in_code_block4):157            if line.startswith("```"):158                count = len(line) - len(line.lstrip("`"))159                if count == 3:160                    in_code_block3 = True161                    continue162                elif count >= 4:163                    in_code_block4 = True164                    continue165166            header_match = HEADER_WITH_PERMALINK_RE.match(line)167            if header_match:168                hashes, title, permalink = header_match.groups()169                headers.append(170                    HeaderPermalinkInfo(171                        hashes=hashes, line_no=line_no, permalink=permalink, title=title172                    )173                )174175        elif in_code_block3:176            if line.startswith("```"):177                count = len(line) - len(line.lstrip("`"))178                if count == 3:179                    in_code_block3 = False180                    continue181182        elif in_code_block4:183            if line.startswith("````"):184                count = len(line) - len(line.lstrip("`"))185                if count >= 4:186                    in_code_block4 = False187                    continue188189    return headers190191192def remove_header_permalinks(lines: list[str]) -> list[str]:193    """194    Remove permalinks from headers in the given lines.195    """196197    modified_lines: list[str] = []198    for line in lines:199        header_match = HEADER_WITH_PERMALINK_RE.match(line)200        if header_match:201            hashes, title, _permalink = header_match.groups()202            modified_line = f"{hashes} {title}"203            modified_lines.append(modified_line)204        else:205            modified_lines.append(line)206    return modified_lines207208209def replace_header_permalinks(210    text: list[str],211    header_permalinks: list[HeaderPermalinkInfo],212    original_header_permalinks: list[HeaderPermalinkInfo],213) -> list[str]:214    """215    Replace permalinks in the given text with the permalinks from the original document.216217    Fail if the number or level of headers does not match the original.218    """219220    modified_text: list[str] = text.copy()221222    if len(header_permalinks) != len(original_header_permalinks):223        raise ValueError(224            "Number of headers with permalinks does not match the number in the "225            "original document "226            f"({len(header_permalinks)} vs {len(original_header_permalinks)})"227        )228229    for header_no in range(len(header_permalinks)):230        header_info = header_permalinks[header_no]231        original_header_info = original_header_permalinks[header_no]232233        if header_info["hashes"] != original_header_info["hashes"]:234            raise ValueError(235                "Header levels do not match between document and original document"236                f" (found {header_info['hashes']}, expected {original_header_info['hashes']})"237                f" for header №{header_no + 1} in line {header_info['line_no']}"238            )239        line_no = header_info["line_no"] - 1240        hashes = header_info["hashes"]241        title = header_info["title"]242        permalink = original_header_info["permalink"]243        modified_text[line_no] = f"{hashes} {title}{permalink}"244245    return modified_text246247248# Markdown links249# --------------------------------------------------------------------------------------250251252def extract_markdown_links(lines: list[str]) -> list[MarkdownLinkInfo]:253    """254    Extract all markdown links from the given lines.255256    Return list of MarkdownLinkInfo, where each dict contains:257    - `line_no` - line number (1-based)258    - `url` - link URL259    - `text` - link text260    - `title` - link title (if any)261    """262263    links: list[MarkdownLinkInfo] = []264    for line_no, line in enumerate(lines, start=1):265        for m in MARKDOWN_LINK_RE.finditer(line):266            links.append(267                MarkdownLinkInfo(268                    line_no=line_no,269                    url=m.group("url"),270                    text=m.group("text"),271                    title=m.group("title"),272                    attributes=m.group("attrs"),273                    full_match=m.group(0),274                )275            )276    return links277278279def _add_lang_code_to_url(url: str, lang_code: str) -> str:280    if url.startswith(TIANGOLO_COM):281        rel_url = url[len(TIANGOLO_COM) :]282        if not rel_url.startswith(ASSETS_URL_PREFIXES):283            url = url.replace(TIANGOLO_COM, f"{TIANGOLO_COM}/{lang_code}")284    return url285286287def _construct_markdown_link(288    url: str,289    text: str,290    title: str | None,291    attributes: str | None,292    lang_code: str,293) -> str:294    """295    Construct a markdown link, adjusting the URL for the given language code if needed.296    """297    url = _add_lang_code_to_url(url, lang_code)298299    if title:300        link = f'[{text}]({url} "{title}")'301    else:302        link = f"[{text}]({url})"303304    if attributes:305        link += f"{{{attributes}}}"306307    return link308309310def replace_markdown_links(311    text: list[str],312    links: list[MarkdownLinkInfo],313    original_links: list[MarkdownLinkInfo],314    lang_code: str,315) -> list[str]:316    """317    Replace markdown links in the given text with the original links.318319    Fail if the number of links does not match the original.320    """321322    if len(links) != len(original_links):323        raise ValueError(324            "Number of markdown links does not match the number in the "325            "original document "326            f"({len(links)} vs {len(original_links)})"327        )328329    modified_text = text.copy()330    for i, link_info in enumerate(links):331        link_text = link_info["text"]332        link_title = link_info["title"]333        original_link_info = original_links[i]334335        # Replace336        replacement_link = _construct_markdown_link(337            url=original_link_info["url"],338            text=link_text,339            title=link_title,340            attributes=original_link_info["attributes"],341            lang_code=lang_code,342        )343        line_no = link_info["line_no"] - 1344        modified_line = modified_text[line_no]345        modified_line = modified_line.replace(346            link_info["full_match"], replacement_link, 1347        )348        modified_text[line_no] = modified_line349350    return modified_text351352353# HTML links354# --------------------------------------------------------------------------------------355356357def extract_html_links(lines: list[str]) -> list[HtmlLinkInfo]:358    """359    Extract all HTML links from the given lines.360361    Return list of HtmlLinkInfo, where each dict contains:362    - `line_no` - line number (1-based)363    - `full_tag` - full HTML link tag364    - `attributes` - list of HTMLLinkAttribute (name, quote, value)365    - `text` - link text366    """367368    links = []369    for line_no, line in enumerate(lines, start=1):370        for html_link in HTML_LINK_RE.finditer(line):371            link_str = html_link.group(0)372373            link_text_match = HTML_LINK_TEXT_RE.match(link_str)374            assert link_text_match is not None375            link_text = link_text_match.group(2)376            assert isinstance(link_text, str)377378            link_open_tag_match = HTML_LINK_OPEN_TAG_RE.match(link_str)379            assert link_open_tag_match is not None380            link_open_tag = link_open_tag_match.group(1)381            assert isinstance(link_open_tag, str)382383            attributes: list[HTMLLinkAttribute] = []384            for attr_name, attr_quote, attr_value in re.findall(385                HTML_ATTR_RE, link_open_tag386            ):387                assert isinstance(attr_name, str)388                assert isinstance(attr_quote, str)389                assert isinstance(attr_value, str)390                attributes.append(391                    HTMLLinkAttribute(392                        name=attr_name, quote=attr_quote, value=attr_value393                    )394                )395            links.append(396                HtmlLinkInfo(397                    line_no=line_no,398                    full_tag=link_str,399                    attributes=attributes,400                    text=link_text,401                )402            )403    return links404405406def _construct_html_link(407    link_text: str,408    attributes: list[HTMLLinkAttribute],409    lang_code: str,410) -> str:411    """412    Reconstruct HTML link, adjusting the URL for the given language code if needed.413    """414415    attributes_upd: list[HTMLLinkAttribute] = []416    for attribute in attributes:417        if attribute["name"] == "href":418            original_url = attribute["value"]419            url = _add_lang_code_to_url(original_url, lang_code)420            attributes_upd.append(421                HTMLLinkAttribute(name="href", quote=attribute["quote"], value=url)422            )423        else:424            attributes_upd.append(attribute)425426    attrs_str = " ".join(427        f"{attribute['name']}={attribute['quote']}{attribute['value']}{attribute['quote']}"428        for attribute in attributes_upd429    )430    return f"<a {attrs_str}>{link_text}</a>"431432433def replace_html_links(434    text: list[str],435    links: list[HtmlLinkInfo],436    original_links: list[HtmlLinkInfo],437    lang_code: str,438) -> list[str]:439    """440    Replace HTML links in the given text with the links from the original document.441442    Adjust URLs for the given language code.443    Fail if the number of links does not match the original.444    """445446    if len(links) != len(original_links):447        raise ValueError(448            "Number of HTML links does not match the number in the "449            "original document "450            f"({len(links)} vs {len(original_links)})"451        )452453    modified_text = text.copy()454    for link_index, link in enumerate(links):455        original_link_info = original_links[link_index]456457        # Replace in the document text458        replacement_link = _construct_html_link(459            link_text=link["text"],460            attributes=original_link_info["attributes"],461            lang_code=lang_code,462        )463        line_no = link["line_no"] - 1464        modified_text[line_no] = modified_text[line_no].replace(465            link["full_tag"], replacement_link, 1466        )467468    return modified_text469470471# Multiline code blocks472# --------------------------------------------------------------------------------------473474475def get_code_block_lang(line: str) -> str:476    match = CODE_BLOCK_LANG_RE.match(line)477    if match:478        return match.group(1)479    return ""480481482def extract_multiline_code_blocks(text: list[str]) -> list[MultilineCodeBlockInfo]:483    blocks: list[MultilineCodeBlockInfo] = []484485    in_code_block3 = False486    in_code_block4 = False487    current_block_lang = ""488    current_block_start_line = -1489    current_block_lines = []490491    for line_no, line in enumerate(text, start=1):492        stripped = line.lstrip()493494        # --- Detect opening fence ---495        if not (in_code_block3 or in_code_block4):496            if stripped.startswith("```"):497                current_block_start_line = line_no498                count = len(stripped) - len(stripped.lstrip("`"))499                if count == 3:500                    in_code_block3 = True501                    current_block_lang = get_code_block_lang(stripped)502                    current_block_lines = [line]503                    continue504                elif count >= 4:505                    in_code_block4 = True506                    current_block_lang = get_code_block_lang(stripped)507                    current_block_lines = [line]508                    continue509510        # --- Detect closing fence ---511        elif in_code_block3:512            if stripped.startswith("```"):513                count = len(stripped) - len(stripped.lstrip("`"))514                if count == 3:515                    current_block_lines.append(line)516                    blocks.append(517                        MultilineCodeBlockInfo(518                            lang=current_block_lang,519                            start_line_no=current_block_start_line,520                            content=current_block_lines,521                        )522                    )523                    in_code_block3 = False524                    current_block_lang = ""525                    current_block_start_line = -1526                    current_block_lines = []527                    continue528            current_block_lines.append(line)529530        elif in_code_block4:531            if stripped.startswith("````"):532                count = len(stripped) - len(stripped.lstrip("`"))533                if count >= 4:534                    current_block_lines.append(line)535                    blocks.append(536                        MultilineCodeBlockInfo(537                            lang=current_block_lang,538                            start_line_no=current_block_start_line,539                            content=current_block_lines,540                        )541                    )542                    in_code_block4 = False543                    current_block_lang = ""544                    current_block_start_line = -1545                    current_block_lines = []546                    continue547            current_block_lines.append(line)548549    return blocks550551552def _split_hash_comment(line: str) -> tuple[str, str | None]:553    match = HASH_COMMENT_RE.match(line)554    if match:555        code = match.group("code").rstrip()556        comment = match.group("comment")557        return code, comment558    return line.rstrip(), None559560561def _split_slashes_comment(line: str) -> tuple[str, str | None]:562    match = SLASHES_COMMENT_RE.match(line)563    if match:564        code = match.group("code").rstrip()565        comment = match.group("comment")566        return code, comment567    return line, None568569570def replace_multiline_code_block(571    block_a: MultilineCodeBlockInfo, block_b: MultilineCodeBlockInfo572) -> list[str]:573    """574    Replace multiline code block `a` with block `b` leaving comments intact.575576    Syntax of comments depends on the language of the code block.577    Raises ValueError if the blocks are not compatible (different languages or different number of lines).578    """579580    start_line = block_a["start_line_no"]581    end_line_no = start_line + len(block_a["content"]) - 1582583    if block_a["lang"] != block_b["lang"]:584        raise ValueError(585            f"Code block (lines {start_line}-{end_line_no}) "586            "has different language than the original block "587            f"('{block_a['lang']}' vs '{block_b['lang']}')"588        )589    if len(block_a["content"]) != len(block_b["content"]):590        raise ValueError(591            f"Code block (lines {start_line}-{end_line_no}) "592            "has different number of lines than the original block "593            f"({len(block_a['content'])} vs {len(block_b['content'])})"594        )595596    block_language = block_a["lang"].lower()597    if block_language in {"mermaid"}:598        if block_a != block_b:599            print(600                f"Skipping mermaid code block replacement (lines {start_line}-{end_line_no}). "601                "This should be checked manually."602            )603        return block_a["content"].copy()  # We don't handle mermaid code blocks for now604605    code_block: list[str] = []606    for line_a, line_b in zip(block_a["content"], block_b["content"], strict=False):607        line_a_comment: str | None = None608        line_b_comment: str | None = None609610        # Handle comments based on language611        if block_language in {612            "python",613            "py",614            "sh",615            "bash",616            "dockerfile",617            "requirements",618            "gitignore",619            "toml",620            "yaml",621            "yml",622            "hash-style-comments",623        }:624            _line_a_code, line_a_comment = _split_hash_comment(line_a)625            _line_b_code, line_b_comment = _split_hash_comment(line_b)626            res_line = line_b627            if line_b_comment:628                res_line = res_line.replace(line_b_comment, line_a_comment or "", 1)629            code_block.append(res_line)630        elif block_language in {"console", "json", "slash-style-comments"}:631            _line_a_code, line_a_comment = _split_slashes_comment(line_a)632            _line_b_code, line_b_comment = _split_slashes_comment(line_b)633            res_line = line_b634            if line_b_comment:635                res_line = res_line.replace(line_b_comment, line_a_comment or "", 1)636            code_block.append(res_line)637        else:638            code_block.append(line_b)639640    return code_block641642643def replace_multiline_code_blocks_in_text(644    text: list[str],645    code_blocks: list[MultilineCodeBlockInfo],646    original_code_blocks: list[MultilineCodeBlockInfo],647) -> list[str]:648    """649    Update each code block in `text` with the corresponding code block from650    `original_code_blocks` with comments taken from `code_blocks`.651652    Raises ValueError if the number, language, or shape of code blocks do not match.653    """654655    if len(code_blocks) != len(original_code_blocks):656        raise ValueError(657            "Number of code blocks does not match the number in the original document "658            f"({len(code_blocks)} vs {len(original_code_blocks)})"659        )660661    modified_text = text.copy()662    for block, original_block in zip(code_blocks, original_code_blocks, strict=True):663        updated_content = replace_multiline_code_block(block, original_block)664665        start_line_index = block["start_line_no"] - 1666        for i, updated_line in enumerate(updated_content):667            modified_text[start_line_index + i] = updated_line668669    return modified_text670671672# All checks673# --------------------------------------------------------------------------------------674675676def check_translation(677    doc_lines: list[str],678    en_doc_lines: list[str],679    lang_code: str,680    auto_fix: bool,681    path: str,682) -> list[str]:683    # Fix code includes684    en_code_includes = extract_code_includes(en_doc_lines)685    doc_lines_with_placeholders = replace_code_includes_with_placeholders(doc_lines)686    fixed_doc_lines = replace_placeholders_with_code_includes(687        doc_lines_with_placeholders, en_code_includes688    )689    if auto_fix and (fixed_doc_lines != doc_lines):690        print(f"Fixing code includes in: {path}")691        doc_lines = fixed_doc_lines692693    # Fix permalinks694    en_permalinks = extract_header_permalinks(en_doc_lines)695    doc_permalinks = extract_header_permalinks(doc_lines)696    fixed_doc_lines = replace_header_permalinks(697        doc_lines, doc_permalinks, en_permalinks698    )699    if auto_fix and (fixed_doc_lines != doc_lines):700        print(f"Fixing header permalinks in: {path}")701        doc_lines = fixed_doc_lines702703    # Fix markdown links704    en_markdown_links = extract_markdown_links(en_doc_lines)705    doc_markdown_links = extract_markdown_links(doc_lines)706    fixed_doc_lines = replace_markdown_links(707        doc_lines, doc_markdown_links, en_markdown_links, lang_code708    )709    if auto_fix and (fixed_doc_lines != doc_lines):710        print(f"Fixing markdown links in: {path}")711        doc_lines = fixed_doc_lines712713    # Fix HTML links714    en_html_links = extract_html_links(en_doc_lines)715    doc_html_links = extract_html_links(doc_lines)716    fixed_doc_lines = replace_html_links(717        doc_lines, doc_html_links, en_html_links, lang_code718    )719    if auto_fix and (fixed_doc_lines != doc_lines):720        print(f"Fixing HTML links in: {path}")721        doc_lines = fixed_doc_lines722723    # Fix multiline code blocks724    en_code_blocks = extract_multiline_code_blocks(en_doc_lines)725    doc_code_blocks = extract_multiline_code_blocks(doc_lines)726    fixed_doc_lines = replace_multiline_code_blocks_in_text(727        doc_lines, doc_code_blocks, en_code_blocks728    )729    if auto_fix and (fixed_doc_lines != doc_lines):730        print(f"Fixing multiline code blocks in: {path}")731        doc_lines = fixed_doc_lines732733    return doc_lines