Avoid unnecessary list conversions; use generators where possible
splits = list(text)
1"""Character text splitters."""23from __future__ import annotations45import re6from typing import Any, Literal78from typing_extensions import override910from langchain_text_splitters.base import Language, TextSplitter111213class CharacterTextSplitter(TextSplitter):14 """Splitting text that looks at characters."""1516 def __init__(17 self,18 separator: str = "\n\n",19 is_separator_regex: bool = False, # noqa: FBT001,FBT00220 **kwargs: Any,21 ) -> None:22 """Create a new TextSplitter."""23 super().__init__(**kwargs)24 self._separator = separator25 self._is_separator_regex = is_separator_regex2627 @override28 def split_text(self, text: str) -> list[str]:29 """Split into chunks without re-inserting lookaround separators.3031 Args:32 text: The text to split.3334 Returns:35 A list of text chunks.36 """37 # 1. Determine split pattern: raw regex or escaped literal38 sep_pattern = (39 self._separator if self._is_separator_regex else re.escape(self._separator)40 )4142 # 2. Initial split (keep separator if requested)43 splits = _split_text_with_regex(44 text, sep_pattern, keep_separator=self._keep_separator45 )4647 # 3. Detect zero-width lookaround so we never re-insert it48 lookaround_prefixes = ("(?=", "(?<!", "(?<=", "(?!")49 is_lookaround = self._is_separator_regex and any(50 self._separator.startswith(p) for p in lookaround_prefixes51 )5253 # 4. Decide merge separator:54 # - if keep_separator or lookaround -> don't re-insert55 # - else -> re-insert literal separator56 merge_sep = ""57 if not (self._keep_separator or is_lookaround):58 merge_sep = self._separator5960 # 5. Merge adjacent splits and return61 return self._merge_splits(splits, merge_sep)626364def _split_text_with_regex(65 text: str, separator: str, *, keep_separator: bool | Literal["start", "end"]66) -> list[str]:67 # Now that we have the separator, split the text68 if separator:69 if keep_separator:70 # The parentheses in the pattern keep the delimiters in the result.71 splits_ = re.split(f"({separator})", text)72 splits = (73 ([splits_[i] + splits_[i + 1] for i in range(0, len(splits_) - 1, 2)])74 if keep_separator == "end"75 else ([splits_[i] + splits_[i + 1] for i in range(1, len(splits_), 2)])76 )77 if len(splits_) % 2 == 0:78 splits += splits_[-1:]79 splits = (80 ([*splits, splits_[-1]])81 if keep_separator == "end"82 else ([splits_[0], *splits])83 )84 else:85 splits = re.split(separator, text)86 else:87 splits = list(text)88 return [s for s in splits if s]899091class RecursiveCharacterTextSplitter(TextSplitter):92 """Splitting text by recursively look at characters.9394 Recursively tries to split by different characters to find one95 that works.96 """9798 def __init__(99 self,100 separators: list[str] | None = None,101 keep_separator: bool | Literal["start", "end"] = True, # noqa: FBT001,FBT002102 is_separator_regex: bool = False, # noqa: FBT001,FBT002103 **kwargs: Any,104 ) -> None:105 """Create a new TextSplitter."""106 super().__init__(keep_separator=keep_separator, **kwargs)107 self._separators = separators or ["\n\n", "\n", " ", ""]108 self._is_separator_regex = is_separator_regex109110 def _split_text(self, text: str, separators: list[str]) -> list[str]:111 """Split incoming text and return chunks."""112 final_chunks = []113 # Get appropriate separator to use114 separator = separators[-1]115 new_separators = []116 for i, s_ in enumerate(separators):117 separator_ = s_ if self._is_separator_regex else re.escape(s_)118 if not s_:119 separator = s_120 break121 if re.search(separator_, text):122 separator = s_123 new_separators = separators[i + 1 :]124 break125126 separator_ = separator if self._is_separator_regex else re.escape(separator)127 splits = _split_text_with_regex(128 text, separator_, keep_separator=self._keep_separator129 )130131 # Now go merging things, recursively splitting longer texts.132 good_splits = []133 separator_ = "" if self._keep_separator else separator134 for s in splits:135 if self._length_function(s) < self._chunk_size:136 good_splits.append(s)137 else:138 if good_splits:139 merged_text = self._merge_splits(good_splits, separator_)140 final_chunks.extend(merged_text)141 good_splits = []142 if not new_separators:143 final_chunks.append(s)144 else:145 other_info = self._split_text(s, new_separators)146 final_chunks.extend(other_info)147 if good_splits:148 merged_text = self._merge_splits(good_splits, separator_)149 final_chunks.extend(merged_text)150 return final_chunks151152 @override153 def split_text(self, text: str) -> list[str]:154 """Split the input text into smaller chunks based on predefined separators.155156 Args:157 text: The input text to be split.158159 Returns:160 A list of text chunks obtained after splitting.161 """162 return self._split_text(text, self._separators)163164 @classmethod165 def from_language(166 cls, language: Language, **kwargs: Any167 ) -> RecursiveCharacterTextSplitter:168 """Return an instance of this class based on a specific language.169170 This method initializes the text splitter with language-specific separators.171172 Args:173 language: The language to configure the text splitter for.174 **kwargs: Additional keyword arguments to customize the splitter.175176 Returns:177 An instance of the text splitter configured for the specified language.178 """179 separators = cls.get_separators_for_language(language)180 return cls(separators=separators, is_separator_regex=True, **kwargs)181182 @staticmethod183 def get_separators_for_language(language: Language) -> list[str]:184 """Retrieve a list of separators specific to the given language.185186 Args:187 language: The language for which to get the separators.188189 Returns:190 A list of separators appropriate for the specified language.191192 Raises:193 ValueError: If the language is not implemented or supported.194 """195 if language in {Language.C, Language.CPP}:196 return [197 # Split along class definitions198 "\nclass ",199 # Split along function definitions200 "\nvoid ",201 "\nint ",202 "\nfloat ",203 "\ndouble ",204 # Split along control flow statements205 "\nif ",206 "\nfor ",207 "\nwhile ",208 "\nswitch ",209 "\ncase ",210 # Split by the normal type of lines211 "\n\n",212 "\n",213 " ",214 "",215 ]216 if language == Language.GO:217 return [218 # Split along function definitions219 "\nfunc ",220 "\nvar ",221 "\nconst ",222 "\ntype ",223 # Split along control flow statements224 "\nif ",225 "\nfor ",226 "\nswitch ",227 "\ncase ",228 # Split by the normal type of lines229 "\n\n",230 "\n",231 " ",232 "",233 ]234 if language == Language.JAVA:235 return [236 # Split along class definitions237 "\nclass ",238 # Split along method definitions239 "\npublic ",240 "\nprotected ",241 "\nprivate ",242 "\nstatic ",243 # Split along control flow statements244 "\nif ",245 "\nfor ",246 "\nwhile ",247 "\nswitch ",248 "\ncase ",249 # Split by the normal type of lines250 "\n\n",251 "\n",252 " ",253 "",254 ]255 if language == Language.KOTLIN:256 return [257 # Split along class definitions258 "\nclass ",259 # Split along method definitions260 "\npublic ",261 "\nprotected ",262 "\nprivate ",263 "\ninternal ",264 "\ncompanion ",265 "\nfun ",266 "\nval ",267 "\nvar ",268 # Split along control flow statements269 "\nif ",270 "\nfor ",271 "\nwhile ",272 "\nwhen ",273 "\nelse ",274 # Split by the normal type of lines275 "\n\n",276 "\n",277 " ",278 "",279 ]280 if language == Language.JS:281 return [282 # Split along function definitions283 "\nfunction ",284 "\nconst ",285 "\nlet ",286 "\nvar ",287 "\nclass ",288 # Split along control flow statements289 "\nif ",290 "\nfor ",291 "\nwhile ",292 "\nswitch ",293 "\ncase ",294 "\ndefault ",295 # Split by the normal type of lines296 "\n\n",297 "\n",298 " ",299 "",300 ]301 if language == Language.TS:302 return [303 "\nenum ",304 "\ninterface ",305 "\nnamespace ",306 "\ntype ",307 # Split along class definitions308 "\nclass ",309 # Split along function definitions310 "\nfunction ",311 "\nconst ",312 "\nlet ",313 "\nvar ",314 # Split along control flow statements315 "\nif ",316 "\nfor ",317 "\nwhile ",318 "\nswitch ",319 "\ncase ",320 "\ndefault ",321 # Split by the normal type of lines322 "\n\n",323 "\n",324 " ",325 "",326 ]327 if language == Language.PHP:328 return [329 # Split along function definitions330 "\nfunction ",331 # Split along class definitions332 "\nclass ",333 # Split along control flow statements334 "\nif ",335 "\nforeach ",336 "\nwhile ",337 "\ndo ",338 "\nswitch ",339 "\ncase ",340 # Split by the normal type of lines341 "\n\n",342 "\n",343 " ",344 "",345 ]346 if language == Language.PROTO:347 return [348 # Split along message definitions349 "\nmessage ",350 # Split along service definitions351 "\nservice ",352 # Split along enum definitions353 "\nenum ",354 # Split along option definitions355 "\noption ",356 # Split along import statements357 "\nimport ",358 # Split along syntax declarations359 "\nsyntax ",360 # Split by the normal type of lines361 "\n\n",362 "\n",363 " ",364 "",365 ]366 if language == Language.PYTHON:367 return [368 # First, try to split along class definitions369 "\nclass ",370 "\ndef ",371 "\n\tdef ",372 # Now split by the normal type of lines373 "\n\n",374 "\n",375 " ",376 "",377 ]378 if language == Language.R:379 return [380 # Split along function definitions381 "\nfunction ",382 # Split along S4 class and method definitions383 "\nsetClass\\(",384 "\nsetMethod\\(",385 "\nsetGeneric\\(",386 # Split along control flow statements387 "\nif ",388 "\nelse ",389 "\nfor ",390 "\nwhile ",391 "\nrepeat ",392 # Split along package loading393 "\nlibrary\\(",394 "\nrequire\\(",395 # Split by the normal type of lines396 "\n\n",397 "\n",398 " ",399 "",400 ]401 if language == Language.RST:402 return [403 # Split along section titles404 "\n=+\n",405 "\n-+\n",406 "\n\\*+\n",407 # Split along directive markers408 "\n\n.. *\n\n",409 # Split by the normal type of lines410 "\n\n",411 "\n",412 " ",413 "",414 ]415 if language == Language.RUBY:416 return [417 # Split along method definitions418 "\ndef ",419 "\nclass ",420 # Split along control flow statements421 "\nif ",422 "\nunless ",423 "\nwhile ",424 "\nfor ",425 "\ndo ",426 "\nbegin ",427 "\nrescue ",428 # Split by the normal type of lines429 "\n\n",430 "\n",431 " ",432 "",433 ]434 if language == Language.ELIXIR:435 return [436 # Split along method function and module definition437 "\ndef ",438 "\ndefp ",439 "\ndefmodule ",440 "\ndefprotocol ",441 "\ndefmacro ",442 "\ndefmacrop ",443 # Split along control flow statements444 "\nif ",445 "\nunless ",446 "\ncase ",447 "\ncond ",448 "\nwith ",449 "\nfor ",450 "\ndo ",451 # Split by the normal type of lines452 "\n\n",453 "\n",454 " ",455 "",456 ]457 if language == Language.RUST:458 return [459 # Split along function definitions460 "\nfn ",461 "\nconst ",462 "\nlet ",463 # Split along control flow statements464 "\nif ",465 "\nwhile ",466 "\nfor ",467 "\nloop ",468 "\nmatch ",469 # Split by the normal type of lines470 "\n\n",471 "\n",472 " ",473 "",474 ]475 if language == Language.SCALA:476 return [477 # Split along class definitions478 "\nclass ",479 "\nobject ",480 # Split along method definitions481 "\ndef ",482 "\nval ",483 "\nvar ",484 # Split along control flow statements485 "\nif ",486 "\nfor ",487 "\nwhile ",488 "\nmatch ",489 "\ncase ",490 # Split by the normal type of lines491 "\n\n",492 "\n",493 " ",494 "",495 ]496 if language == Language.SWIFT:497 return [498 # Split along function definitions499 "\nfunc ",500 # Split along class definitions501 "\nclass ",502 "\nstruct ",503 "\nenum ",504 # Split along control flow statements505 "\nif ",506 "\nfor ",507 "\nwhile ",508 "\ndo ",509 "\nswitch ",510 "\ncase ",511 # Split by the normal type of lines512 "\n\n",513 "\n",514 " ",515 "",516 ]517 if language == Language.MARKDOWN:518 return [519 # First, try to split along Markdown headings (starting with level 2)520 "\n#{1,6} ",521 # Note the alternative syntax for headings (below) is not handled here522 # Heading level 2523 # ---------------524 # End of code block525 "```\n",526 # Horizontal lines527 "\n\\*\\*\\*+\n",528 "\n---+\n",529 "\n___+\n",530 # Note that this splitter doesn't handle horizontal lines defined531 # by *three or more* of ***, ---, or ___, but this is not handled532 "\n\n",533 "\n",534 " ",535 "",536 ]537 if language == Language.LATEX:538 return [539 # First, try to split along Latex sections540 "\n\\\\chapter{",541 "\n\\\\section{",542 "\n\\\\subsection{",543 "\n\\\\subsubsection{",544 # Now split by environments545 "\n\\\\begin{enumerate}",546 "\n\\\\begin{itemize}",547 "\n\\\\begin{description}",548 "\n\\\\begin{list}",549 "\n\\\\begin{quote}",550 "\n\\\\begin{quotation}",551 "\n\\\\begin{verse}",552 "\n\\\\begin{verbatim}",553 # Now split by math environments554 "\n\\\\begin{align}",555 "$$",556 "$",557 # Now split by the normal type of lines558 " ",559 "",560 ]561 if language == Language.HTML:562 return [563 # First, try to split along HTML tags564 "<body",565 "<div",566 "<p",567 "<br",568 "<li",569 "<h1",570 "<h2",571 "<h3",572 "<h4",573 "<h5",574 "<h6",575 "<span",576 "<table",577 "<tr",578 "<td",579 "<th",580 "<ul",581 "<ol",582 "<header",583 "<footer",584 "<nav",585 # Head586 "<head",587 "<style",588 "<script",589 "<meta",590 "<title",591 "",592 ]593 if language == Language.CSHARP:594 return [595 "\ninterface ",596 "\nenum ",597 "\ndelegate ",598 "\nevent ",599 # Split along class definitions600 "\nclass ",601 "\nabstract ",602 # Split along method definitions603 "\npublic ",604 "\nprotected ",605 "\nprivate ",606 "\nstatic ",607 "\nreturn ",608 # Split along control flow statements609 "\nif ",610 "\ncontinue ",611 "\nfor ",612 "\nforeach ",613 "\nwhile ",614 "\nswitch ",615 "\nbreak ",616 "\ncase ",617 "\nelse ",618 # Split by exceptions619 "\ntry ",620 "\nthrow ",621 "\nfinally ",622 "\ncatch ",623 # Split by the normal type of lines624 "\n\n",625 "\n",626 " ",627 "",628 ]629 if language == Language.SOL:630 return [631 # Split along compiler information definitions632 "\npragma ",633 "\nusing ",634 # Split along contract definitions635 "\ncontract ",636 "\ninterface ",637 "\nlibrary ",638 # Split along method definitions639 "\nconstructor ",640 "\ntype ",641 "\nfunction ",642 "\nevent ",643 "\nmodifier ",644 "\nerror ",645 "\nstruct ",646 "\nenum ",647 # Split along control flow statements648 "\nif ",649 "\nfor ",650 "\nwhile ",651 "\ndo while ",652 "\nassembly ",653 # Split by the normal type of lines654 "\n\n",655 "\n",656 " ",657 "",658 ]659 if language == Language.COBOL:660 return [661 # Split along divisions662 "\nIDENTIFICATION DIVISION.",663 "\nENVIRONMENT DIVISION.",664 "\nDATA DIVISION.",665 "\nPROCEDURE DIVISION.",666 # Split along sections within DATA DIVISION667 "\nWORKING-STORAGE SECTION.",668 "\nLINKAGE SECTION.",669 "\nFILE SECTION.",670 # Split along sections within PROCEDURE DIVISION671 "\nINPUT-OUTPUT SECTION.",672 # Split along paragraphs and common statements673 "\nOPEN ",674 "\nCLOSE ",675 "\nREAD ",676 "\nWRITE ",677 "\nIF ",678 "\nELSE ",679 "\nMOVE ",680 "\nPERFORM ",681 "\nUNTIL ",682 "\nVARYING ",683 "\nACCEPT ",684 "\nDISPLAY ",685 "\nSTOP RUN.",686 # Split by the normal type of lines687 "\n",688 " ",689 "",690 ]691 if language == Language.LUA:692 return [693 # Split along variable and table definitions694 "\nlocal ",695 # Split along function definitions696 "\nfunction ",697 # Split along control flow statements698 "\nif ",699 "\nfor ",700 "\nwhile ",701 "\nrepeat ",702 # Split by the normal type of lines703 "\n\n",704 "\n",705 " ",706 "",707 ]708 if language == Language.HASKELL:709 return [710 # Split along function definitions711 "\nmain :: ",712 "\nmain = ",713 "\nlet ",714 "\nin ",715 "\ndo ",716 "\nwhere ",717 "\n:: ",718 "\n= ",719 # Split along type declarations720 "\ndata ",721 "\nnewtype ",722 "\ntype ",723 # Split along module declarations724 "\nmodule ",725 # Split along import statements726 "\nimport ",727 "\nqualified ",728 "\nimport qualified ",729 # Split along typeclass declarations730 "\nclass ",731 "\ninstance ",732 # Split along case expressions733 "\ncase ",734 # Split along guards in function definitions735 "\n| ",736 # Split along record field declarations737 "\n= {",738 "\n, ",739 # Split by the normal type of lines740 "\n\n",741 "\n",742 " ",743 "",744 ]745 if language == Language.POWERSHELL:746 return [747 # Split along function definitions748 "\nfunction ",749 # Split along parameter declarations (escape parentheses)750 "\nparam ",751 # Split along control flow statements752 "\nif ",753 "\nforeach ",754 "\nfor ",755 "\nwhile ",756 "\nswitch ",757 # Split along class definitions (for PowerShell 5.0 and above)758 "\nclass ",759 # Split along try-catch-finally blocks760 "\ntry ",761 "\ncatch ",762 "\nfinally ",763 # Split by normal lines and empty spaces764 "\n\n",765 "\n",766 " ",767 "",768 ]769 if language == Language.VISUALBASIC6:770 vis = r"(?:Public|Private|Friend|Global|Static)\s+"771 return [772 # Split along definitions773 rf"\n(?!End\s){vis}?Sub\s+",774 rf"\n(?!End\s){vis}?Function\s+",775 rf"\n(?!End\s){vis}?Property\s+(?:Get|Let|Set)\s+",776 rf"\n(?!End\s){vis}?Type\s+",777 rf"\n(?!End\s){vis}?Enum\s+",778 # Split along control flow statements779 r"\n(?!End\s)If\s+",780 r"\nElseIf\s+",781 r"\nElse\s+",782 r"\nSelect\s+Case\s+",783 r"\nCase\s+",784 r"\nFor\s+",785 r"\nDo\s+",786 r"\nWhile\s+",787 r"\nWith\s+",788 # Split by the normal type of lines789 r"\n\n",790 r"\n",791 " ",792 "",793 ]794795 if language in Language._value2member_map_:796 msg = f"Language {language} is not implemented yet!"797 raise ValueError(msg)798 msg = (799 f"Language {language} is not supported! Please choose from {list(Language)}"800 )801 raise ValueError(msg)
Code quality findings 7
Ensure functions have docstrings for documentation
def from_language(
Ensure functions have docstrings for documentation
"\ndef ",
Ensure functions have docstrings for documentation
"\n\tdef ",
Ensure functions have docstrings for documentation
"\ndef ",
Ensure functions have docstrings for documentation
"\ndef ",
Ensure functions have docstrings for documentation
"\ndef ",