1"""Test text splitting functionality."""23from __future__ import annotations45import json6import random7import re8import string9import textwrap10from typing import TYPE_CHECKING, Any1112import pytest13from langchain_core._api import suppress_langchain_beta_warning14from langchain_core.documents import Document1516from langchain_text_splitters import (17    Language,18    RecursiveCharacterTextSplitter,19    TextSplitter,20    Tokenizer,21)22from langchain_text_splitters.base import split_text_on_tokens23from langchain_text_splitters.character import CharacterTextSplitter24from langchain_text_splitters.html import (25    HTMLHeaderTextSplitter,26    HTMLSectionSplitter,27    HTMLSemanticPreservingSplitter,28)29from langchain_text_splitters.json import RecursiveJsonSplitter30from langchain_text_splitters.jsx import JSFrameworkTextSplitter31from langchain_text_splitters.markdown import (32    ExperimentalMarkdownSyntaxTextSplitter,33    MarkdownHeaderTextSplitter,34)35from langchain_text_splitters.python import PythonCodeTextSplitter3637if TYPE_CHECKING:38    from collections.abc import Callable3940    from bs4 import Tag414243FAKE_PYTHON_TEXT = """44class Foo:4546    def bar():474849def foo():5051def testing_func():5253def bar():54"""555657def test_no_heavy_imports_on_package_load() -> None:58    """Ensure importing the package does not eagerly import heavy dependencies.5960    Runs in a fresh interpreter so the result is unaffected by modules the test61    session already imported. A `sys.meta_path` finder records any *attempt* to62    import a heavy optional dependency, so the guard holds whether or not those63    packages are installed in the current environment (a plain `sys.modules` check64    would pass vacuously when the packages are absent).65    """66    import subprocess  # noqa: PLC041567    import sys  # noqa: PLC04156869    script = textwrap.dedent(70        """71        import sys7273        blocked = {74            "nltk", "spacy", "sentence_transformers", "konlpy", "torch",75            "transformers", "tiktoken",76        }77        attempted = []7879        class _Recorder:80            def find_spec(self, name, path=None, target=None):81                if name.split(".")[0] in blocked:82                    attempted.append(name.split(".")[0])83                return None  # defer to the real finders8485        sys.meta_path.insert(0, _Recorder())86        import langchain_text_splitters  # noqa: F40187        print(",".join(sorted(set(attempted))))88        """89    )90    result = subprocess.run(  # noqa: S603  # list args, no shell; input is static91        [sys.executable, "-c", script],92        capture_output=True,93        text=True,94        check=False,95        timeout=60,96    )97    assert result.returncode == 0, (98        f"Importing langchain_text_splitters failed:\n{result.stderr}"99    )100    attempted = [p for p in result.stdout.strip().split(",") if p]101    assert not attempted, (102        f"Heavy packages imported at langchain_text_splitters load time: {attempted}"103    )104105106@pytest.mark.parametrize(107    ("module_name", "expected_message"),108    [109        ("konlpy", "pip install konlpy"),110        ("nltk", "pip install nltk"),111        ("spacy", "pip install spacy"),112        ("sentence_transformers", "pip install sentence-transformers"),113    ],114)115def test_missing_optional_dependency_raises_importerror(116    module_name: str,117    expected_message: str,118    monkeypatch: pytest.MonkeyPatch,119) -> None:120    """Each splitter raises a helpful ImportError when its optional dep is missing.121122    The missing dependency is simulated by forcing its import to fail, so the test123    is independent of whether the optional package is actually installed.124    """125    import sys  # noqa: PLC0415126127    from langchain_text_splitters.konlpy import KonlpyTextSplitter  # noqa: PLC0415128    from langchain_text_splitters.nltk import NLTKTextSplitter  # noqa: PLC0415129    from langchain_text_splitters.sentence_transformers import (  # noqa: PLC0415130        SentenceTransformersTokenTextSplitter,131    )132    from langchain_text_splitters.spacy import SpacyTextSplitter  # noqa: PLC0415133134    constructors: dict[str, Callable[[], TextSplitter]] = {135        "konlpy": KonlpyTextSplitter,136        "nltk": NLTKTextSplitter,137        "spacy": SpacyTextSplitter,138        "sentence_transformers": SentenceTransformersTokenTextSplitter,139    }140141    # `None` in sys.modules makes both `import x` and `import_module(x)` raise142    # ImportError, exercising the splitter's missing-dependency branch.143    monkeypatch.setitem(sys.modules, module_name, None)144    with pytest.raises(ImportError, match=re.escape(expected_message)):145        constructors[module_name]()146147148@pytest.mark.parametrize(149    "class_name",150    [151        "KonlpyTextSplitter",152        "NLTKTextSplitter",153        "SpacyTextSplitter",154        "SentenceTransformersTokenTextSplitter",155    ],156)157def test_lazy_getattr_resolves(class_name: str) -> None:158    """`__getattr__` resolves lazy splitter classes from the package namespace."""159    import langchain_text_splitters as lts  # noqa: PLC0415160161    try:162        cls = getattr(lts, class_name)163    except ImportError:164        pytest.skip(f"Optional dependency for {class_name} not installed")165    assert isinstance(cls, type), f"{class_name} should be a class, got {type(cls)}"166167168def test_lazy_getattr_raises_for_unknown() -> None:169    """Accessing an unknown attribute raises `AttributeError`."""170    import langchain_text_splitters as lts  # noqa: PLC0415171172    with pytest.raises(AttributeError, match="no_such_thing"):173        _ = lts.no_such_thing  # type: ignore[attr-defined]174175176def test_lightweight_splitters_remain_eagerly_accessible() -> None:177    """Lightweight splitters are still directly importable from the package."""178    import langchain_text_splitters as lts  # noqa: PLC0415179180    assert issubclass(lts.RecursiveCharacterTextSplitter, lts.TextSplitter)181    assert issubclass(lts.CharacterTextSplitter, lts.TextSplitter)182183184def test_character_text_splitter() -> None:185    """Test splitting by character count."""186    text = "foo bar baz 123"187    splitter = CharacterTextSplitter(separator=" ", chunk_size=7, chunk_overlap=3)188    output = splitter.split_text(text)189    expected_output = ["foo bar", "bar baz", "baz 123"]190    assert output == expected_output191192193def test_character_text_splitter_empty_doc() -> None:194    """Test splitting by character count doesn't create empty documents."""195    text = "foo  bar"196    splitter = CharacterTextSplitter(separator=" ", chunk_size=2, chunk_overlap=0)197    output = splitter.split_text(text)198    expected_output = ["foo", "bar"]199    assert output == expected_output200201202def test_character_text_splitter_separtor_empty_doc() -> None:203    """Test edge cases are separators."""204    text = "f b"205    splitter = CharacterTextSplitter(separator=" ", chunk_size=2, chunk_overlap=0)206    output = splitter.split_text(text)207    expected_output = ["f", "b"]208    assert output == expected_output209210211def test_character_text_splitter_long() -> None:212    """Test splitting by character count on long words."""213    text = "foo bar baz a a"214    splitter = CharacterTextSplitter(separator=" ", chunk_size=3, chunk_overlap=1)215    output = splitter.split_text(text)216    expected_output = ["foo", "bar", "baz", "a a"]217    assert output == expected_output218219220def test_character_text_splitter_short_words_first() -> None:221    """Test splitting by character count when shorter words are first."""222    text = "a a foo bar baz"223    splitter = CharacterTextSplitter(separator=" ", chunk_size=3, chunk_overlap=1)224    output = splitter.split_text(text)225    expected_output = ["a a", "foo", "bar", "baz"]226    assert output == expected_output227228229def test_character_text_splitter_longer_words() -> None:230    """Test splitting by characters when splits not found easily."""231    text = "foo bar baz 123"232    splitter = CharacterTextSplitter(separator=" ", chunk_size=1, chunk_overlap=1)233    output = splitter.split_text(text)234    expected_output = ["foo", "bar", "baz", "123"]235    assert output == expected_output236237238# edge cases239def test_character_text_splitter_no_separator_in_text() -> None:240    """Text splitting where there is no separator but a single word."""241    text = "singleword"242    splitter = CharacterTextSplitter(separator=" ", chunk_size=10, chunk_overlap=0)243    output = splitter.split_text(text)244    expected_output = ["singleword"]245    assert output == expected_output246247248def test_character_text_splitter_handle_chunksize_equal_to_chunkoverlap() -> None:249    """Text splitting safe guards when chunk size is equal chunk overlap."""250    text = "hello"251    splitter = CharacterTextSplitter(separator=" ", chunk_size=5, chunk_overlap=5)252    output = splitter.split_text(text)253    expected_output = ["hello"]254    assert output == expected_output255256257def test_character_text_splitter_empty_input() -> None:258    """Test splitting safely where there is no input to process."""259    text = ""260    splitter = CharacterTextSplitter(separator=" ", chunk_size=5, chunk_overlap=0)261    output = splitter.split_text(text)262    expected_output: list[str] = []263    assert output == expected_output264265266def test_character_text_splitter_whitespace_only() -> None:267    """Test splitting safely where there is white space."""268    text = " "269    splitter = CharacterTextSplitter(separator=" ", chunk_size=5, chunk_overlap=0)270    output = splitter.split_text(text)271    expected_output: list[str] = []272    assert output == expected_output273274275@pytest.mark.parametrize(276    ("separator", "is_separator_regex"), [(re.escape("."), True), (".", False)]277)278def test_character_text_splitter_keep_separator_regex(279    *, separator: str, is_separator_regex: bool280) -> None:281    """Test CharacterTextSplitter keep separator regex.282283    Test splitting by characters while keeping the separator284    that is a regex special character.285    """286    text = "foo.bar.baz.123"287    splitter = CharacterTextSplitter(288        separator=separator,289        chunk_size=1,290        chunk_overlap=0,291        keep_separator=True,292        is_separator_regex=is_separator_regex,293    )294    output = splitter.split_text(text)295    expected_output = ["foo", ".bar", ".baz", ".123"]296    assert output == expected_output297298299@pytest.mark.parametrize(300    ("separator", "is_separator_regex"), [(re.escape("."), True), (".", False)]301)302def test_character_text_splitter_keep_separator_regex_start(303    *, separator: str, is_separator_regex: bool304) -> None:305    """Test CharacterTextSplitter keep separator regex and put at start.306307    Test splitting by characters while keeping the separator308    that is a regex special character and placing it at the start of each chunk.309    """310    text = "foo.bar.baz.123"311    splitter = CharacterTextSplitter(312        separator=separator,313        chunk_size=1,314        chunk_overlap=0,315        keep_separator="start",316        is_separator_regex=is_separator_regex,317    )318    output = splitter.split_text(text)319    expected_output = ["foo", ".bar", ".baz", ".123"]320    assert output == expected_output321322323@pytest.mark.parametrize(324    ("separator", "is_separator_regex"), [(re.escape("."), True), (".", False)]325)326def test_character_text_splitter_keep_separator_regex_end(327    *, separator: str, is_separator_regex: bool328) -> None:329    """Test CharacterTextSplitter keep separator regex and put at end.330331    Test splitting by characters while keeping the separator332    that is a regex special character and placing it at the end of each chunk.333    """334    text = "foo.bar.baz.123"335    splitter = CharacterTextSplitter(336        separator=separator,337        chunk_size=1,338        chunk_overlap=0,339        keep_separator="end",340        is_separator_regex=is_separator_regex,341    )342    output = splitter.split_text(text)343    expected_output = ["foo.", "bar.", "baz.", "123"]344    assert output == expected_output345346347@pytest.mark.parametrize(348    ("separator", "is_separator_regex"), [(re.escape("."), True), (".", False)]349)350def test_character_text_splitter_discard_separator_regex(351    *, separator: str, is_separator_regex: bool352) -> None:353    """Test CharacterTextSplitter discard separator regex.354355    Test splitting by characters discarding the separator356    that is a regex special character.357    """358    text = "foo.bar.baz.123"359    splitter = CharacterTextSplitter(360        separator=separator,361        chunk_size=1,362        chunk_overlap=0,363        keep_separator=False,364        is_separator_regex=is_separator_regex,365    )366    output = splitter.split_text(text)367    expected_output = ["foo", "bar", "baz", "123"]368    assert output == expected_output369370371def test_recursive_character_text_splitter_keep_separators() -> None:372    split_tags = [",", "."]373    query = "Apple,banana,orange and tomato."374    # start375    splitter = RecursiveCharacterTextSplitter(376        chunk_size=10,377        chunk_overlap=0,378        separators=split_tags,379        keep_separator="start",380    )381    result = splitter.split_text(query)382    assert result == ["Apple", ",banana", ",orange and tomato", "."]383384    # end385    splitter = RecursiveCharacterTextSplitter(386        chunk_size=10,387        chunk_overlap=0,388        separators=split_tags,389        keep_separator="end",390    )391    result = splitter.split_text(query)392    assert result == ["Apple,", "banana,", "orange and tomato."]393394395def test_character_text_splitting_args() -> None:396    """Test invalid arguments."""397    with pytest.raises(398        ValueError,399        match=re.escape(400            "Got a larger chunk overlap (4) than chunk size (2), should be smaller."401        ),402    ):403        CharacterTextSplitter(chunk_size=2, chunk_overlap=4)404    for invalid_size in (0, -1):405        with pytest.raises(ValueError, match="chunk_size must be > 0, got"):406            CharacterTextSplitter(chunk_size=invalid_size)407    with pytest.raises(ValueError, match="chunk_overlap must be >= 0, got -1"):408        CharacterTextSplitter(chunk_size=2, chunk_overlap=-1)409410411def test_merge_splits() -> None:412    """Test merging splits with a given separator."""413    splitter = CharacterTextSplitter(separator=" ", chunk_size=9, chunk_overlap=2)414    splits = ["foo", "bar", "baz"]415    expected_output = ["foo bar", "baz"]416    output = splitter._merge_splits(splits, separator=" ")417    assert output == expected_output418419420def test_create_documents() -> None:421    """Test create documents method."""422    texts = ["foo bar", "baz"]423    splitter = CharacterTextSplitter(separator=" ", chunk_size=3, chunk_overlap=0)424    docs = splitter.create_documents(texts)425    expected_docs = [426        Document(page_content="foo"),427        Document(page_content="bar"),428        Document(page_content="baz"),429    ]430    assert docs == expected_docs431432433def test_create_documents_with_metadata() -> None:434    """Test create documents with metadata method."""435    texts = ["foo bar", "baz"]436    splitter = CharacterTextSplitter(separator=" ", chunk_size=3, chunk_overlap=0)437    docs = splitter.create_documents(texts, [{"source": "1"}, {"source": "2"}])438    expected_docs = [439        Document(page_content="foo", metadata={"source": "1"}),440        Document(page_content="bar", metadata={"source": "1"}),441        Document(page_content="baz", metadata={"source": "2"}),442    ]443    assert docs == expected_docs444445446@pytest.mark.parametrize(447    ("splitter", "text", "expected_docs"),448    [449        (450            CharacterTextSplitter(451                separator=" ", chunk_size=7, chunk_overlap=3, add_start_index=True452            ),453            "foo bar baz 123",454            [455                Document(page_content="foo bar", metadata={"start_index": 0}),456                Document(page_content="bar baz", metadata={"start_index": 4}),457                Document(page_content="baz 123", metadata={"start_index": 8}),458            ],459        ),460        (461            RecursiveCharacterTextSplitter(462                chunk_size=6,463                chunk_overlap=0,464                separators=["\n\n", "\n", " ", ""],465                add_start_index=True,466            ),467            "w1 w1 w1 w1 w1 w1 w1 w1 w1",468            [469                Document(page_content="w1 w1", metadata={"start_index": 0}),470                Document(page_content="w1 w1", metadata={"start_index": 6}),471                Document(page_content="w1 w1", metadata={"start_index": 12}),472                Document(page_content="w1 w1", metadata={"start_index": 18}),473                Document(page_content="w1", metadata={"start_index": 24}),474            ],475        ),476    ],477)478def test_create_documents_with_start_index(479    splitter: TextSplitter, text: str, expected_docs: list[Document]480) -> None:481    """Test create documents method."""482    docs = splitter.create_documents([text])483    assert docs == expected_docs484    for doc in docs:485        s_i = doc.metadata["start_index"]486        assert text[s_i : s_i + len(doc.page_content)] == doc.page_content487488489def test_metadata_not_shallow() -> None:490    """Test that metadatas are not shallow."""491    texts = ["foo bar"]492    splitter = CharacterTextSplitter(separator=" ", chunk_size=3, chunk_overlap=0)493    docs = splitter.create_documents(texts, [{"source": "1"}])494    expected_docs = [495        Document(page_content="foo", metadata={"source": "1"}),496        Document(page_content="bar", metadata={"source": "1"}),497    ]498    assert docs == expected_docs499    docs[0].metadata["foo"] = 1500    assert docs[0].metadata == {"source": "1", "foo": 1}501    assert docs[1].metadata == {"source": "1"}502503504def test_iterative_text_splitter_keep_separator() -> None:505    chunk_size = 5506    output = __test_iterative_text_splitter(chunk_size=chunk_size, keep_separator=True)507508    assert output == [509        "....5",510        "X..3",511        "Y...4",512        "X....5",513        "Y...",514    ]515516517def test_iterative_text_splitter_discard_separator() -> None:518    chunk_size = 5519    output = __test_iterative_text_splitter(chunk_size=chunk_size, keep_separator=False)520521    assert output == [522        "....5",523        "..3",524        "...4",525        "....5",526        "...",527    ]528529530def __test_iterative_text_splitter(531    *, chunk_size: int, keep_separator: bool532) -> list[str]:533    chunk_size += 1 if keep_separator else 0534535    splitter = RecursiveCharacterTextSplitter(536        chunk_size=chunk_size,537        chunk_overlap=0,538        separators=["X", "Y"],539        keep_separator=keep_separator,540    )541    text = "....5X..3Y...4X....5Y..."542    output = splitter.split_text(text)543    for chunk in output:544        assert len(chunk) <= chunk_size, f"Chunk is larger than {chunk_size}"545    return output546547548def test_iterative_text_splitter() -> None:549    """Test iterative text splitter."""550    text = """Hi.\n\nI'm Harrison.\n\nHow? Are? You?\nOkay then f f f f.551This is a weird text to write, but gotta test the splittingggg some how.552553Bye!\n\n-H."""554    splitter = RecursiveCharacterTextSplitter(chunk_size=10, chunk_overlap=1)555    output = splitter.split_text(text)556    expected_output = [557        "Hi.",558        "I'm",559        "Harrison.",560        "How? Are?",561        "You?",562        "Okay then",563        "f f f f.",564        "This is a",565        "weird",566        "text to",567        "write,",568        "but gotta",569        "test the",570        "splitting",571        "gggg",572        "some how.",573        "Bye!",574        "-H.",575    ]576    assert output == expected_output577578579def test_split_documents() -> None:580    """Test split_documents."""581    splitter = CharacterTextSplitter(separator="", chunk_size=1, chunk_overlap=0)582    docs = [583        Document(page_content="foo", metadata={"source": "1"}),584        Document(page_content="bar", metadata={"source": "2"}),585        Document(page_content="baz", metadata={"source": "1"}),586    ]587    expected_output = [588        Document(page_content="f", metadata={"source": "1"}),589        Document(page_content="o", metadata={"source": "1"}),590        Document(page_content="o", metadata={"source": "1"}),591        Document(page_content="b", metadata={"source": "2"}),592        Document(page_content="a", metadata={"source": "2"}),593        Document(page_content="r", metadata={"source": "2"}),594        Document(page_content="b", metadata={"source": "1"}),595        Document(page_content="a", metadata={"source": "1"}),596        Document(page_content="z", metadata={"source": "1"}),597    ]598    assert splitter.split_documents(docs) == expected_output599600601def test_python_text_splitter() -> None:602    splitter = PythonCodeTextSplitter(chunk_size=30, chunk_overlap=0)603    splits = splitter.split_text(FAKE_PYTHON_TEXT)604    split_0 = """class Foo:\n\n    def bar():"""605    split_1 = """def foo():"""606    split_2 = """def testing_func():"""607    split_3 = """def bar():"""608    expected_splits = [split_0, split_1, split_2, split_3]609    assert splits == expected_splits610611612FAKE_JSX_TEXT = """613import React from 'react';614import OtherComponent from './OtherComponent';615616function MyComponent() {617  const [count, setCount] = React.useState(0);618619  const handleClick = () => {620    setCount(count + 1);621  };622623  return (624    <div>625      <h1>Counter: {count}</h1>626      <button onClick={handleClick}>627        Increment628      </button>629      <OtherComponent />630    </div>631  );632}633634export default MyComponent;635"""636637638def test_jsx_text_splitter() -> None:639    splitter = JSFrameworkTextSplitter(chunk_size=30, chunk_overlap=0)640    splits = splitter.split_text(FAKE_JSX_TEXT)641642    expected_splits = [643        (644            "\nimport React from 'react';\n"645            "import OtherComponent from './OtherComponent';\n"646        ),647        "\nfunction MyComponent() {\n  const [count, setCount] = React.useState(0);",648        "\n\n  const handleClick = () => {\n    setCount(count + 1);\n  };",649        "return (",650        "<div>",651        "<h1>Counter: {count}</h1>\n      ",652        "<button onClick={handleClick}>\n        Increment\n      </button>\n      ",653        "<OtherComponent />\n    </div>\n  );\n}\n",654        "export default MyComponent;",655    ]656    assert [s.strip() for s in splits] == [s.strip() for s in expected_splits]657658659FAKE_VUE_TEXT = """660<template>661  <div>662    <h1>{{ title }}</h1>663    <button @click="increment">664      Count is: {{ count }}665    </button>666  </div>667</template>668669<script>670export default {671  data() {672    return {673      title: 'Counter App',674      count: 0675    }676  },677  methods: {678    increment() {679      this.count++680    }681  }682}683</script>684685<style>686button {687  color: blue;688}689</style>690"""691692693def test_vue_text_splitter() -> None:694    splitter = JSFrameworkTextSplitter(chunk_size=30, chunk_overlap=0)695    splits = splitter.split_text(FAKE_VUE_TEXT)696697    expected_splits = [698        "<template>",699        "<div>",700        "<h1>{{ title }}</h1>",701        (702            '<button @click="increment">\n      Count is: {{ count }}\n'703            "    </button>\n  </div>\n</template>"704        ),705        "<script>",706        "export",707        (708            " default {\n  data() {\n    return {\n      title: 'Counter App',\n      "709            "count: 0\n    }\n  },\n  methods: {\n    increment() {\n      "710            "this.count++\n    }\n  }\n}\n</script>"711        ),712        "<style>\nbutton {\n  color: blue;\n}\n</style>",713    ]714    assert [s.strip() for s in splits] == [s.strip() for s in expected_splits]715716717FAKE_SVELTE_TEXT = """718<script>719  let count = 0720721  function increment() {722    count += 1723  }724</script>725726<main>727  <h1>Counter App</h1>728  <button on:click={increment}>729    Count is: {count}730  </button>731</main>732733<style>734  button {735    color: blue;736  }737</style>738"""739740741def test_svelte_text_splitter() -> None:742    splitter = JSFrameworkTextSplitter(chunk_size=30, chunk_overlap=0)743    splits = splitter.split_text(FAKE_SVELTE_TEXT)744745    expected_splits = [746        "<script>\n  let count = 0",747        "\n\n  function increment() {\n    count += 1\n  }\n</script>",748        "<main>",749        "<h1>Counter App</h1>",750        "<button on:click={increment}>\n    Count is: {count}\n  </button>\n</main>",751        "<style>\n  button {\n    color: blue;\n  }\n</style>",752    ]753    assert [s.strip() for s in splits] == [s.strip() for s in expected_splits]754755756def test_jsx_splitter_separator_not_mutated_across_calls() -> None:757    """Regression test: repeated split_text() calls must not mutate separators.758759    Calling split_text() multiple times on the same JSFrameworkTextSplitter760    instance must not grow the internal separator list between calls.761762    Before the fix, self._separators was overwritten with the full expanded list763    on every invocation, so a second call would start with the already-expanded764    list and append even more separators.765    """766    splitter = JSFrameworkTextSplitter(chunk_size=30, chunk_overlap=0)767768    # Record separator count after constructing (should be 0 - no custom separators)769    initial_sep_count = len(splitter._separators)770771    # Call split_text twice; the results should be identical for identical input772    splits_first = splitter.split_text(FAKE_JSX_TEXT)773    splits_second = splitter.split_text(FAKE_JSX_TEXT)774775    assert splits_first == splits_second, (776        "split_text() must return identical results on repeated calls with the "777        "same input"778    )779    assert len(splitter._separators) == initial_sep_count, (780        "split_text() must not mutate self._separators between calls"781    )782783784CHUNK_SIZE = 16785786787def test_python_code_splitter() -> None:788    splitter = RecursiveCharacterTextSplitter.from_language(789        Language.PYTHON, chunk_size=CHUNK_SIZE, chunk_overlap=0790    )791    code = """792def hello_world():793    print("Hello, World!")794795# Call the function796hello_world()797    """798    chunks = splitter.split_text(code)799    assert chunks == [800        "def",801        "hello_world():",802        'print("Hello,',803        'World!")',804        "# Call the",805        "function",806        "hello_world()",807    ]808809810def test_golang_code_splitter() -> None:811    splitter = RecursiveCharacterTextSplitter.from_language(812        Language.GO, chunk_size=CHUNK_SIZE, chunk_overlap=0813    )814    code = """815package main816817import "fmt"818819func helloWorld() {820    fmt.Println("Hello, World!")821}822823func main() {824    helloWorld()825}826    """827    chunks = splitter.split_text(code)828    assert chunks == [829        "package main",830        'import "fmt"',831        "func",832        "helloWorld() {",833        'fmt.Println("He',834        "llo,",835        'World!")',836        "}",837        "func main() {",838        "helloWorld()",839        "}",840    ]841842843def test_rst_code_splitter() -> None:844    splitter = RecursiveCharacterTextSplitter.from_language(845        Language.RST, chunk_size=CHUNK_SIZE, chunk_overlap=0846    )847    code = """848Sample Document849===============850851Section852-------853854This is the content of the section.855856Lists857-----858859- Item 1860- Item 2861- Item 3862863Comment864*******865Not a comment866867.. This is a comment868    """869    chunks = splitter.split_text(code)870    assert chunks == [871        "Sample Document",872        "===============",873        "Section",874        "-------",875        "This is the",876        "content of the",877        "section.",878        "Lists",879        "-----",880        "- Item 1",881        "- Item 2",882        "- Item 3",883        "Comment",884        "*******",885        "Not a comment",886        ".. This is a",887        "comment",888    ]889    # Special test for special characters890    code = "harry\n***\nbabylon is"891    chunks = splitter.split_text(code)892    assert chunks == ["harry", "***\nbabylon is"]893894895def test_proto_file_splitter() -> None:896    splitter = RecursiveCharacterTextSplitter.from_language(897        Language.PROTO, chunk_size=CHUNK_SIZE, chunk_overlap=0898    )899    code = """900syntax = "proto3";901902package example;903904message Person {905    string name = 1;906    int32 age = 2;907    repeated string hobbies = 3;908}909    """910    chunks = splitter.split_text(code)911    assert chunks == [912        "syntax =",913        '"proto3";',914        "package",915        "example;",916        "message Person",917        "{",918        "string name",919        "= 1;",920        "int32 age =",921        "2;",922        "repeated",923        "string hobbies",924        "= 3;",925        "}",926    ]927928929def test_javascript_code_splitter() -> None:930    splitter = RecursiveCharacterTextSplitter.from_language(931        Language.JS, chunk_size=CHUNK_SIZE, chunk_overlap=0932    )933    code = """934function helloWorld() {935  console.log("Hello, World!");936}937938// Call the function939helloWorld();940    """941    chunks = splitter.split_text(code)942    assert chunks == [943        "function",944        "helloWorld() {",945        'console.log("He',946        "llo,",947        'World!");',948        "}",949        "// Call the",950        "function",951        "helloWorld();",952    ]953954955def test_cobol_code_splitter() -> None:956    splitter = RecursiveCharacterTextSplitter.from_language(957        Language.COBOL, chunk_size=CHUNK_SIZE, chunk_overlap=0958    )959    code = """960IDENTIFICATION DIVISION.961PROGRAM-ID. HelloWorld.962DATA DIVISION.963WORKING-STORAGE SECTION.96401 GREETING           PIC X(12)   VALUE 'Hello, World!'.965PROCEDURE DIVISION.966DISPLAY GREETING.967STOP RUN.968    """969    chunks = splitter.split_text(code)970    assert chunks == [971        "IDENTIFICATION",972        "DIVISION.",973        "PROGRAM-ID.",974        "HelloWorld.",975        "DATA DIVISION.",976        "WORKING-STORAGE",977        "SECTION.",978        "01 GREETING",979        "PIC X(12)",980        "VALUE 'Hello,",981        "World!'.",982        "PROCEDURE",983        "DIVISION.",984        "DISPLAY",985        "GREETING.",986        "STOP RUN.",987    ]988989990def test_typescript_code_splitter() -> None:991    splitter = RecursiveCharacterTextSplitter.from_language(992        Language.TS, chunk_size=CHUNK_SIZE, chunk_overlap=0993    )994    code = """995function helloWorld(): void {996  console.log("Hello, World!");997}998999// Call the function1000helloWorld();1001    """1002    chunks = splitter.split_text(code)1003    assert chunks == [1004        "function",1005        "helloWorld():",1006        "void {",1007        'console.log("He',1008        "llo,",1009        'World!");',1010        "}",1011        "// Call the",1012        "function",1013        "helloWorld();",1014    ]101510161017def test_java_code_splitter() -> None:1018    splitter = RecursiveCharacterTextSplitter.from_language(1019        Language.JAVA, chunk_size=CHUNK_SIZE, chunk_overlap=01020    )1021    code = """1022public class HelloWorld {1023    public static void main(String[] args) {1024        System.out.println("Hello, World!");1025    }1026}1027    """1028    chunks = splitter.split_text(code)1029    assert chunks == [1030        "public class",1031        "HelloWorld {",1032        "public",1033        "static void",1034        "main(String[]",1035        "args) {",1036        "System.out.prin",1037        'tln("Hello,',1038        'World!");',1039        "}\n}",1040    ]104110421043def test_kotlin_code_splitter() -> None:1044    splitter = RecursiveCharacterTextSplitter.from_language(1045        Language.KOTLIN, chunk_size=CHUNK_SIZE, chunk_overlap=01046    )1047    code = """1048class HelloWorld {1049    companion object {1050        @JvmStatic1051        fun main(args: Array<String>) {1052            println("Hello, World!")1053        }1054    }1055}1056    """1057    chunks = splitter.split_text(code)1058    assert chunks == [1059        "class",1060        "HelloWorld {",1061        "companion",1062        "object {",1063        "@JvmStatic",1064        "fun",1065        "main(args:",1066        "Array<String>)",1067        "{",1068        'println("Hello,',1069        'World!")',1070        "}\n    }",1071        "}",1072    ]107310741075def test_csharp_code_splitter() -> None:1076    splitter = RecursiveCharacterTextSplitter.from_language(1077        Language.CSHARP, chunk_size=CHUNK_SIZE, chunk_overlap=01078    )1079    code = """1080using System;1081class Program1082{1083    static void Main()1084    {1085        int age = 30; // Change the age value as needed10861087        // Categorize the age without any console output1088        if (age < 18)1089        {1090            // Age is under 181091        }1092        else if (age >= 18 && age < 65)1093        {1094            // Age is an adult1095        }1096        else1097        {1098            // Age is a senior citizen1099        }1100    }1101}1102    """11031104    chunks = splitter.split_text(code)1105    assert chunks == [1106        "using System;",1107        "class Program\n{",1108        "static void",1109        "Main()",1110        "{",1111        "int age",1112        "= 30; // Change",1113        "the age value",1114        "as needed",1115        "//",1116        "Categorize the",1117        "age without any",1118        "console output",1119        "if (age",1120        "< 18)",1121        "{",1122        "//",1123        "Age is under 18",1124        "}",1125        "else if",1126        "(age >= 18 &&",1127        "age < 65)",1128        "{",1129        "//",1130        "Age is an adult",1131        "}",1132        "else",1133        "{",1134        "//",1135        "Age is a senior",1136        "citizen",1137        "}\n    }",1138        "}",1139    ]114011411142def test_csharp_separators_no_java_keywords() -> None:1143    """C# separators should not contain Java-only keywords."""1144    splitter = RecursiveCharacterTextSplitter.from_language(1145        Language.CSHARP, chunk_size=CHUNK_SIZE, chunk_overlap=01146    )1147    # "implements" is a Java keyword; C# uses ":" for interface implementation1148    assert "\nimplements " not in splitter._separators114911501151def test_elixir_separators_no_while() -> None:1152    """Elixir has no while loop; the separator should not be present."""1153    splitter = RecursiveCharacterTextSplitter.from_language(1154        Language.ELIXIR, chunk_size=CHUNK_SIZE, chunk_overlap=01155    )1156    assert "\nwhile " not in splitter._separators115711581159def test_cpp_code_splitter() -> None:1160    splitter = RecursiveCharacterTextSplitter.from_language(1161        Language.CPP, chunk_size=CHUNK_SIZE, chunk_overlap=01162    )1163    code = """1164#include <iostream>11651166int main() {1167    std::cout << "Hello, World!" << std::endl;1168    return 0;1169}1170    """1171    chunks = splitter.split_text(code)1172    assert chunks == [1173        "#include",1174        "<iostream>",1175        "int main() {",1176        "std::cout",1177        '<< "Hello,',1178        'World!" <<',1179        "std::endl;",1180        "return 0;\n}",1181    ]118211831184def test_scala_code_splitter() -> None:1185    splitter = RecursiveCharacterTextSplitter.from_language(1186        Language.SCALA, chunk_size=CHUNK_SIZE, chunk_overlap=01187    )1188    code = """1189object HelloWorld {1190  def main(args: Array[String]): Unit = {1191    println("Hello, World!")1192  }1193}1194    """1195    chunks = splitter.split_text(code)1196    assert chunks == [1197        "object",1198        "HelloWorld {",1199        "def",1200        "main(args:",1201        "Array[String]):",1202        "Unit = {",1203        'println("Hello,',1204        'World!")',1205        "}\n}",1206    ]120712081209def test_ruby_code_splitter() -> None:1210    splitter = RecursiveCharacterTextSplitter.from_language(1211        Language.RUBY, chunk_size=CHUNK_SIZE, chunk_overlap=01212    )1213    code = """1214def hello_world1215  puts "Hello, World!"1216end12171218hello_world1219    """1220    chunks = splitter.split_text(code)1221    assert chunks == [1222        "def hello_world",1223        'puts "Hello,',1224        'World!"',1225        "end",1226        "hello_world",1227    ]122812291230def test_php_code_splitter() -> None:1231    splitter = RecursiveCharacterTextSplitter.from_language(1232        Language.PHP, chunk_size=CHUNK_SIZE, chunk_overlap=01233    )1234    code = """1235<?php1236function hello_world() {1237    echo "Hello, World!";1238}12391240hello_world();1241?>1242    """1243    chunks = splitter.split_text(code)1244    assert chunks == [1245        "<?php",1246        "function",1247        "hello_world() {",1248        "echo",1249        '"Hello,',1250        'World!";',1251        "}",1252        "hello_world();",1253        "?>",1254    ]125512561257def test_swift_code_splitter() -> None:1258    splitter = RecursiveCharacterTextSplitter.from_language(1259        Language.SWIFT, chunk_size=CHUNK_SIZE, chunk_overlap=01260    )1261    code = """1262func helloWorld() {1263    print("Hello, World!")1264}12651266helloWorld()1267    """1268    chunks = splitter.split_text(code)1269    assert chunks == [1270        "func",1271        "helloWorld() {",1272        'print("Hello,',1273        'World!")',1274        "}",1275        "helloWorld()",1276    ]127712781279def test_rust_code_splitter() -> None:1280    splitter = RecursiveCharacterTextSplitter.from_language(1281        Language.RUST, chunk_size=CHUNK_SIZE, chunk_overlap=01282    )1283    code = """1284fn main() {1285    println!("Hello, World!");1286}1287    """1288    chunks = splitter.split_text(code)1289    assert chunks == ["fn main() {", 'println!("Hello', ",", 'World!");', "}"]129012911292def test_r_code_splitter() -> None:1293    splitter = RecursiveCharacterTextSplitter.from_language(1294        Language.R, chunk_size=CHUNK_SIZE, chunk_overlap=01295    )1296    code = """1297library(dplyr)12981299my_func <- function(x) {1300    return(x + 1)1301}13021303if (TRUE) {1304    print("Hello")1305}1306    """1307    chunks = splitter.split_text(code)1308    assert chunks == [1309        "library(dplyr)",1310        "my_func <-",1311        "function(x) {",1312        "return(x +",1313        "1)",1314        "}",1315        "if (TRUE) {",1316        'print("Hello")',1317        "}",1318    ]131913201321def test_markdown_code_splitter() -> None:1322    splitter = RecursiveCharacterTextSplitter.from_language(1323        Language.MARKDOWN, chunk_size=CHUNK_SIZE, chunk_overlap=01324    )1325    code = """1326# Sample Document13271328## Section13291330This is the content of the section.13311332## Lists13331334- Item 11335- Item 21336- Item 313371338### Horizontal lines13391340***********1341____________1342-------------------13431344#### Code blocks1345```1346This is a code block13471348# sample code1349a = 11350b = 21351```1352    """1353    chunks = splitter.split_text(code)1354    assert chunks == [1355        "# Sample",1356        "Document",1357        "## Section",1358        "This is the",1359        "content of the",1360        "section.",1361        "## Lists",1362        "- Item 1",1363        "- Item 2",1364        "- Item 3",1365        "### Horizontal",1366        "lines",1367        "***********",1368        "____________",1369        "---------------",1370        "----",1371        "#### Code",1372        "blocks",1373        "```",1374        "This is a code",1375        "block",1376        "# sample code",1377        "a = 1\nb = 2",1378        "```",1379    ]1380    # Special test for special characters1381    code = "harry\n***\nbabylon is"1382    chunks = splitter.split_text(code)1383    assert chunks == ["harry", "***\nbabylon is"]138413851386def test_latex_code_splitter() -> None:1387    splitter = RecursiveCharacterTextSplitter.from_language(1388        Language.LATEX, chunk_size=CHUNK_SIZE, chunk_overlap=01389    )1390    code = """1391Hi Harrison!1392\\chapter{1}1393"""1394    chunks = splitter.split_text(code)1395    assert chunks == ["Hi Harrison!", "\\chapter{1}"]139613971398def test_html_code_splitter() -> None:1399    splitter = RecursiveCharacterTextSplitter.from_language(1400        Language.HTML, chunk_size=60, chunk_overlap=01401    )1402    code = """1403<h1>Sample Document</h1>1404    <h2>Section</h2>1405        <p id="1234">Reference content.</p>14061407    <h2>Lists</h2>1408        <ul>1409            <li>Item 1</li>1410            <li>Item 2</li>1411            <li>Item 3</li>1412        </ul>14131414        <h3>A block</h3>1415            <div class="amazing">1416                <p>Some text</p>1417                <p>Some more text</p>1418            </div>1419    """1420    chunks = splitter.split_text(code)1421    assert chunks == [1422        "<h1>Sample Document</h1>\n    <h2>Section</h2>",1423        '<p id="1234">Reference content.</p>',1424        "<h2>Lists</h2>\n        <ul>",1425        "<li>Item 1</li>\n            <li>Item 2</li>",1426        "<li>Item 3</li>\n        </ul>",1427        "<h3>A block</h3>",1428        '<div class="amazing">',1429        "<p>Some text</p>",1430        "<p>Some more text</p>\n            </div>",1431    ]143214331434def test_md_header_text_splitter_1() -> None:1435    """Test markdown splitter by header: Case 1."""1436    markdown_document = (1437        "# Foo\n\n"1438        "    ## Bar\n\n"1439        "Hi this is Jim\n\n"1440        "Hi this is Joe\n\n"1441        " ## Baz\n\n"1442        " Hi this is Molly"1443    )1444    headers_to_split_on = [1445        ("#", "Header 1"),1446        ("##", "Header 2"),1447    ]1448    markdown_splitter = MarkdownHeaderTextSplitter(1449        headers_to_split_on=headers_to_split_on,1450    )1451    output = markdown_splitter.split_text(markdown_document)1452    expected_output = [1453        Document(1454            page_content="Hi this is Jim  \nHi this is Joe",1455            metadata={"Header 1": "Foo", "Header 2": "Bar"},1456        ),1457        Document(1458            page_content="Hi this is Molly",1459            metadata={"Header 1": "Foo", "Header 2": "Baz"},1460        ),1461    ]1462    assert output == expected_output146314641465def test_md_header_text_splitter_2() -> None:1466    """Test markdown splitter by header: Case 2."""1467    markdown_document = (1468        "# Foo\n\n"1469        "    ## Bar\n\n"1470        "Hi this is Jim\n\n"1471        "Hi this is Joe\n\n"1472        " ### Boo \n\n"1473        " Hi this is Lance \n\n"1474        " ## Baz\n\n"1475        " Hi this is Molly"1476    )14771478    headers_to_split_on = [1479        ("#", "Header 1"),1480        ("##", "Header 2"),1481        ("###", "Header 3"),1482    ]1483    markdown_splitter = MarkdownHeaderTextSplitter(1484        headers_to_split_on=headers_to_split_on,1485    )1486    output = markdown_splitter.split_text(markdown_document)1487    expected_output = [1488        Document(1489            page_content="Hi this is Jim  \nHi this is Joe",1490            metadata={"Header 1": "Foo", "Header 2": "Bar"},1491        ),1492        Document(1493            page_content="Hi this is Lance",1494            metadata={"Header 1": "Foo", "Header 2": "Bar", "Header 3": "Boo"},1495        ),1496        Document(1497            page_content="Hi this is Molly",1498            metadata={"Header 1": "Foo", "Header 2": "Baz"},1499        ),1500    ]1501    assert output == expected_output150215031504def test_md_header_text_splitter_3() -> None:1505    """Test markdown splitter by header: Case 3."""1506    markdown_document = (1507        "# Foo\n\n"1508        "    ## Bar\n\n"1509        "Hi this is Jim\n\n"1510        "Hi this is Joe\n\n"1511        " ### Boo \n\n"1512        " Hi this is Lance \n\n"1513        " #### Bim \n\n"1514        " Hi this is John \n\n"1515        " ## Baz\n\n"1516        " Hi this is Molly"1517    )15181519    headers_to_split_on = [1520        ("#", "Header 1"),1521        ("##", "Header 2"),1522        ("###", "Header 3"),1523        ("####", "Header 4"),1524    ]15251526    markdown_splitter = MarkdownHeaderTextSplitter(1527        headers_to_split_on=headers_to_split_on,1528    )1529    output = markdown_splitter.split_text(markdown_document)15301531    expected_output = [1532        Document(1533            page_content="Hi this is Jim  \nHi this is Joe",1534            metadata={"Header 1": "Foo", "Header 2": "Bar"},1535        ),1536        Document(1537            page_content="Hi this is Lance",1538            metadata={"Header 1": "Foo", "Header 2": "Bar", "Header 3": "Boo"},1539        ),1540        Document(1541            page_content="Hi this is John",1542            metadata={1543                "Header 1": "Foo",1544                "Header 2": "Bar",1545                "Header 3": "Boo",1546                "Header 4": "Bim",1547            },1548        ),1549        Document(1550            page_content="Hi this is Molly",1551            metadata={"Header 1": "Foo", "Header 2": "Baz"},1552        ),1553    ]15541555    assert output == expected_output155615571558def test_md_header_text_splitter_preserve_headers_1() -> None:1559    """Test markdown splitter by header: Preserve Headers."""1560    markdown_document = (1561        "# Foo\n\n"1562        "    ## Bat\n\n"1563        "Hi this is Jim\n\n"1564        "Hi Joe\n\n"1565        "## Baz\n\n"1566        "# Bar\n\n"1567        "This is Alice\n\n"1568        "This is Bob"1569    )1570    headers_to_split_on = [1571        ("#", "Header 1"),1572    ]1573    markdown_splitter = MarkdownHeaderTextSplitter(1574        headers_to_split_on=headers_to_split_on,1575        strip_headers=False,1576    )1577    output = markdown_splitter.split_text(markdown_document)1578    expected_output = [1579        Document(1580            page_content="# Foo  \n## Bat  \nHi this is Jim  \nHi Joe  \n## Baz",1581            metadata={"Header 1": "Foo"},1582        ),1583        Document(1584            page_content="# Bar  \nThis is Alice  \nThis is Bob",1585            metadata={"Header 1": "Bar"},1586        ),1587    ]1588    assert output == expected_output158915901591def test_md_header_text_splitter_preserve_headers_2() -> None:1592    """Test markdown splitter by header: Preserve Headers."""1593    markdown_document = (1594        "# Foo\n\n"1595        "    ## Bar\n\n"1596        "Hi this is Jim\n\n"1597        "Hi this is Joe\n\n"1598        "### Boo \n\n"1599        "Hi this is Lance\n\n"1600        "## Baz\n\n"1601        "Hi this is Molly\n"1602        "    ## Buz\n"1603        "# Bop"1604    )1605    headers_to_split_on = [1606        ("#", "Header 1"),1607        ("##", "Header 2"),1608        ("###", "Header 3"),1609    ]1610    markdown_splitter = MarkdownHeaderTextSplitter(1611        headers_to_split_on=headers_to_split_on,1612        strip_headers=False,1613    )1614    output = markdown_splitter.split_text(markdown_document)1615    expected_output = [1616        Document(1617            page_content="# Foo  \n## Bar  \nHi this is Jim  \nHi this is Joe",1618            metadata={"Header 1": "Foo", "Header 2": "Bar"},1619        ),1620        Document(1621            page_content="### Boo  \nHi this is Lance",1622            metadata={"Header 1": "Foo", "Header 2": "Bar", "Header 3": "Boo"},1623        ),1624        Document(1625            page_content="## Baz  \nHi this is Molly",1626            metadata={"Header 1": "Foo", "Header 2": "Baz"},1627        ),1628        Document(1629            page_content="## Buz",1630            metadata={"Header 1": "Foo", "Header 2": "Buz"},1631        ),1632        Document(page_content="# Bop", metadata={"Header 1": "Bop"}),1633    ]1634    assert output == expected_output163516361637@pytest.mark.parametrize("fence", [("```"), ("~~~")])1638def test_md_header_text_splitter_fenced_code_block(fence: str) -> None:1639    """Test markdown splitter by header: Fenced code block."""1640    markdown_document = (1641        f"# This is a Header\n\n{fence}\nfoo()\n# Not a header\nbar()\n{fence}"1642    )16431644    headers_to_split_on = [1645        ("#", "Header 1"),1646        ("##", "Header 2"),1647    ]16481649    markdown_splitter = MarkdownHeaderTextSplitter(1650        headers_to_split_on=headers_to_split_on,1651    )1652    output = markdown_splitter.split_text(markdown_document)16531654    expected_output = [1655        Document(1656            page_content=f"{fence}\nfoo()\n# Not a header\nbar()\n{fence}",1657            metadata={"Header 1": "This is a Header"},1658        ),1659    ]16601661    assert output == expected_output166216631664@pytest.mark.parametrize(("fence", "other_fence"), [("```", "~~~"), ("~~~", "```")])1665def test_md_header_text_splitter_fenced_code_block_interleaved(1666    fence: str, other_fence: str1667) -> None:1668    """Test markdown splitter by header: Interleaved fenced code block."""1669    markdown_document = (1670        "# This is a Header\n\n"1671        f"{fence}\n"1672        "foo\n"1673        "# Not a header\n"1674        f"{other_fence}\n"1675        "# Not a header\n"1676        f"{fence}"1677    )16781679    headers_to_split_on = [1680        ("#", "Header 1"),1681        ("##", "Header 2"),1682    ]16831684    markdown_splitter = MarkdownHeaderTextSplitter(1685        headers_to_split_on=headers_to_split_on,1686    )1687    output = markdown_splitter.split_text(markdown_document)16881689    expected_output = [1690        Document(1691            page_content=(1692                f"{fence}\nfoo\n# Not a header\n{other_fence}\n# Not a header\n{fence}"1693            ),1694            metadata={"Header 1": "This is a Header"},1695        ),1696    ]16971698    assert output == expected_output169917001701@pytest.mark.parametrize("characters", ["\ufeff"])1702def test_md_header_text_splitter_with_invisible_characters(characters: str) -> None:1703    """Test markdown splitter by header: Fenced code block."""1704    markdown_document = f"{characters}# Foo\n\nfoo()\n{characters}## Bar\n\nbar()"17051706    headers_to_split_on = [1707        ("#", "Header 1"),1708        ("##", "Header 2"),1709    ]17101711    markdown_splitter = MarkdownHeaderTextSplitter(1712        headers_to_split_on=headers_to_split_on,1713    )1714    output = markdown_splitter.split_text(markdown_document)17151716    expected_output = [1717        Document(1718            page_content="foo()",1719            metadata={"Header 1": "Foo"},1720        ),1721        Document(1722            page_content="bar()",1723            metadata={"Header 1": "Foo", "Header 2": "Bar"},1724        ),1725    ]17261727    assert output == expected_output172817291730def test_md_header_text_splitter_with_custom_headers() -> None:1731    """Test markdown splitter with custom header patterns like **Header**."""1732    markdown_document = """**Chapter 1**17331734This is the content for chapter 1.17351736***Section 1.1***17371738This is the content for section 1.1.17391740**Chapter 2**17411742This is the content for chapter 2.17431744***Section 2.1***17451746This is the content for section 2.1.1747"""17481749    headers_to_split_on = [1750        ("**", "Bold Header"),1751        ("***", "Bold Italic Header"),1752    ]17531754    custom_header_patterns = {1755        "**": 1,  # Level 1 headers1756        "***": 2,  # Level 2 headers1757    }1758    markdown_splitter = MarkdownHeaderTextSplitter(1759        headers_to_split_on=headers_to_split_on,1760        custom_header_patterns=custom_header_patterns,1761    )1762    output = markdown_splitter.split_text(markdown_document)17631764    expected_output = [1765        Document(1766            page_content="This is the content for chapter 1.",1767            metadata={"Bold Header": "Chapter 1"},1768        ),1769        Document(1770            page_content="This is the content for section 1.1.",1771            metadata={"Bold Header": "Chapter 1", "Bold Italic Header": "Section 1.1"},1772        ),1773        Document(1774            page_content="This is the content for chapter 2.",1775            metadata={"Bold Header": "Chapter 2"},1776        ),1777        Document(1778            page_content="This is the content for section 2.1.",1779            metadata={"Bold Header": "Chapter 2", "Bold Italic Header": "Section 2.1"},1780        ),1781    ]17821783    assert output == expected_output178417851786def test_md_header_text_splitter_mixed_headers() -> None:1787    """Test markdown splitter with both standard and custom headers."""1788    markdown_document = """# Standard Header 117891790Content under standard header.17911792**Custom Header 1**17931794Content under custom header.17951796## Standard Header 217971798Content under standard header 2.17991800***Custom Header 2***18011802Content under custom header 2.1803"""18041805    headers_to_split_on = [1806        ("#", "Header 1"),1807        ("##", "Header 2"),1808        ("**", "Bold Header"),1809        ("***", "Bold Italic Header"),1810    ]18111812    custom_header_patterns = {1813        "**": 1,  # Same level as #1814        "***": 2,  # Same level as ##1815    }18161817    markdown_splitter = MarkdownHeaderTextSplitter(1818        headers_to_split_on=headers_to_split_on,1819        custom_header_patterns=custom_header_patterns,1820    )1821    output = markdown_splitter.split_text(markdown_document)18221823    expected_output = [1824        Document(1825            page_content="Content under standard header.",1826            metadata={"Header 1": "Standard Header 1"},1827        ),1828        Document(1829            page_content="Content under custom header.",1830            metadata={"Bold Header": "Custom Header 1"},1831        ),1832        Document(1833            page_content="Content under standard header 2.",1834            metadata={1835                "Bold Header": "Custom Header 1",1836                "Header 2": "Standard Header 2",1837            },1838        ),1839        Document(1840            page_content="Content under custom header 2.",1841            metadata={1842                "Bold Header": "Custom Header 1",1843                "Bold Italic Header": "Custom Header 2",1844            },1845        ),1846    ]18471848    assert output == expected_output184918501851EXPERIMENTAL_MARKDOWN_DOCUMENT = (1852    "# My Header 1\n"1853    "Content for header 1\n"1854    "## Header 2\n"1855    "Content for header 2\n"1856    "### Header 3\n"1857    "Content for header 3\n"1858    "## Header 2 Again\n"1859    "This should be tagged with Header 1 and Header 2 Again\n"1860    "```python\n"1861    "def func_definition():\n"1862    "   print('Keep the whitespace consistent')\n"1863    "```\n"1864    "# Header 1 again\n"1865    "We should also split on the horizontal line\n"1866    "----\n"1867    "This will be a new doc but with the same header metadata\n\n"1868    "And it includes a new paragraph"1869)187018711872def test_experimental_markdown_syntax_text_splitter() -> None:1873    """Test experimental markdown syntax splitter."""1874    markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter()1875    output = markdown_splitter.split_text(EXPERIMENTAL_MARKDOWN_DOCUMENT)18761877    expected_output = [1878        Document(1879            page_content="Content for header 1\n",1880            metadata={"Header 1": "My Header 1"},1881        ),1882        Document(1883            page_content="Content for header 2\n",1884            metadata={"Header 1": "My Header 1", "Header 2": "Header 2"},1885        ),1886        Document(1887            page_content="Content for header 3\n",1888            metadata={1889                "Header 1": "My Header 1",1890                "Header 2": "Header 2",1891                "Header 3": "Header 3",1892            },1893        ),1894        Document(1895            page_content="This should be tagged with Header 1 and Header 2 Again\n",1896            metadata={"Header 1": "My Header 1", "Header 2": "Header 2 Again"},1897        ),1898        Document(1899            page_content=(1900                "```python\ndef func_definition():\n   "1901                "print('Keep the whitespace consistent')\n```\n"1902            ),1903            metadata={1904                "Code": "python",1905                "Header 1": "My Header 1",1906                "Header 2": "Header 2 Again",1907            },1908        ),1909        Document(1910            page_content="We should also split on the horizontal line\n",1911            metadata={"Header 1": "Header 1 again"},1912        ),1913        Document(1914            page_content=(1915                "This will be a new doc but with the same header metadata\n\n"1916                "And it includes a new paragraph"1917            ),1918            metadata={"Header 1": "Header 1 again"},1919        ),1920    ]19211922    assert output == expected_output192319241925def test_experimental_markdown_syntax_text_splitter_header_configuration() -> None:1926    """Test experimental markdown syntax splitter."""1927    headers_to_split_on = [("#", "Encabezamiento 1")]19281929    markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter(1930        headers_to_split_on=headers_to_split_on1931    )1932    output = markdown_splitter.split_text(EXPERIMENTAL_MARKDOWN_DOCUMENT)19331934    expected_output = [1935        Document(1936            page_content=(1937                "Content for header 1\n"1938                "## Header 2\n"1939                "Content for header 2\n"1940                "### Header 3\n"1941                "Content for header 3\n"1942                "## Header 2 Again\n"1943                "This should be tagged with Header 1 and Header 2 Again\n"1944            ),1945            metadata={"Encabezamiento 1": "My Header 1"},1946        ),1947        Document(1948            page_content=(1949                "```python\ndef func_definition():\n   "1950                "print('Keep the whitespace consistent')\n```\n"1951            ),1952            metadata={"Code": "python", "Encabezamiento 1": "My Header 1"},1953        ),1954        Document(1955            page_content="We should also split on the horizontal line\n",1956            metadata={"Encabezamiento 1": "Header 1 again"},1957        ),1958        Document(1959            page_content=(1960                "This will be a new doc but with the same header metadata\n\n"1961                "And it includes a new paragraph"1962            ),1963            metadata={"Encabezamiento 1": "Header 1 again"},1964        ),1965    ]19661967    assert output == expected_output196819691970def test_experimental_markdown_syntax_text_splitter_with_headers() -> None:1971    """Test experimental markdown syntax splitter."""1972    markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter(strip_headers=False)1973    output = markdown_splitter.split_text(EXPERIMENTAL_MARKDOWN_DOCUMENT)19741975    expected_output = [1976        Document(1977            page_content="# My Header 1\nContent for header 1\n",1978            metadata={"Header 1": "My Header 1"},1979        ),1980        Document(1981            page_content="## Header 2\nContent for header 2\n",1982            metadata={"Header 1": "My Header 1", "Header 2": "Header 2"},1983        ),1984        Document(1985            page_content="### Header 3\nContent for header 3\n",1986            metadata={1987                "Header 1": "My Header 1",1988                "Header 2": "Header 2",1989                "Header 3": "Header 3",1990            },1991        ),1992        Document(1993            page_content=(1994                "## Header 2 Again\n"1995                "This should be tagged with Header 1 and Header 2 Again\n"1996            ),1997            metadata={"Header 1": "My Header 1", "Header 2": "Header 2 Again"},1998        ),1999        Document(2000            page_content=(