Ensure functions have docstrings for documentation
def lazy_load(
1from collections.abc import AsyncIterator, Iterable, Iterator, Sequence2from datetime import UTC, datetime3from typing import (4 Any,5)6from unittest.mock import AsyncMock, MagicMock, patch78import pytest9import pytest_asyncio10from pytest_mock import MockerFixture1112from langchain_core.document_loaders.base import BaseLoader13from langchain_core.documents import Document14from langchain_core.embeddings import DeterministicFakeEmbedding15from langchain_core.indexing import InMemoryRecordManager, aindex, index16from langchain_core.indexing.api import (17 IndexingException,18 _abatch,19 _batch,20 _get_document_with_hash,21)22from langchain_core.indexing.in_memory import InMemoryDocumentIndex23from langchain_core.vectorstores import InMemoryVectorStore, VectorStore242526class ToyLoader(BaseLoader):27 """Toy loader that always returns the same documents."""2829 def __init__(self, documents: Sequence[Document]) -> None:30 """Initialize with the documents to return."""31 self.documents = documents3233 def lazy_load(34 self,35 ) -> Iterator[Document]:36 yield from self.documents3738 async def alazy_load(39 self,40 ) -> AsyncIterator[Document]:41 for document in self.documents:42 yield document434445@pytest.fixture46def record_manager() -> InMemoryRecordManager:47 """Timestamped set fixture."""48 record_manager = InMemoryRecordManager(namespace="hello")49 record_manager.create_schema()50 return record_manager515253@pytest_asyncio.fixture54async def arecord_manager() -> InMemoryRecordManager:55 """Timestamped set fixture."""56 record_manager = InMemoryRecordManager(namespace="hello")57 await record_manager.acreate_schema()58 return record_manager596061@pytest.fixture62def vector_store() -> InMemoryVectorStore:63 """Vector store fixture."""64 embeddings = DeterministicFakeEmbedding(size=5)65 return InMemoryVectorStore(embeddings)666768@pytest.fixture69def upserting_vector_store() -> InMemoryVectorStore:70 """Vector store fixture."""71 embeddings = DeterministicFakeEmbedding(size=5)72 return InMemoryVectorStore(embeddings)737475def test_indexing_same_content(76 record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore77) -> None:78 """Indexing some content to confirm it gets added only once."""79 loader = ToyLoader(80 documents=[81 Document(82 page_content="This is a test document.",83 ),84 Document(85 page_content="This is another document.",86 ),87 ]88 )8990 assert index(loader, record_manager, vector_store, key_encoder="sha256") == {91 "num_added": 2,92 "num_deleted": 0,93 "num_skipped": 0,94 "num_updated": 0,95 }9697 assert len(list(vector_store.store)) == 29899 for _ in range(2):100 # Run the indexing again101 assert index(loader, record_manager, vector_store, key_encoder="sha256") == {102 "num_added": 0,103 "num_deleted": 0,104 "num_skipped": 2,105 "num_updated": 0,106 }107108109async def test_aindexing_same_content(110 arecord_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore111) -> None:112 """Indexing some content to confirm it gets added only once."""113 loader = ToyLoader(114 documents=[115 Document(116 page_content="This is a test document.",117 ),118 Document(119 page_content="This is another document.",120 ),121 ]122 )123124 assert await aindex(125 loader,126 arecord_manager,127 vector_store,128 key_encoder="sha256",129 ) == {130 "num_added": 2,131 "num_deleted": 0,132 "num_skipped": 0,133 "num_updated": 0,134 }135136 assert len(list(vector_store.store)) == 2137138 for _ in range(2):139 # Run the indexing again140 assert await aindex(141 loader,142 arecord_manager,143 vector_store,144 key_encoder="sha256",145 ) == {146 "num_added": 0,147 "num_deleted": 0,148 "num_skipped": 2,149 "num_updated": 0,150 }151152153def test_index_simple_delete_full(154 record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore155) -> None:156 """Indexing some content to confirm it gets added only once."""157 loader = ToyLoader(158 documents=[159 Document(160 page_content="This is a test document.",161 ),162 Document(163 page_content="This is another document.",164 ),165 ]166 )167168 with patch.object(169 record_manager,170 "get_time",171 return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),172 ):173 assert index(174 loader,175 record_manager,176 vector_store,177 cleanup="full",178 key_encoder="sha256",179 ) == {180 "num_added": 2,181 "num_deleted": 0,182 "num_skipped": 0,183 "num_updated": 0,184 }185186 with patch.object(187 record_manager,188 "get_time",189 return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),190 ):191 assert index(192 loader,193 record_manager,194 vector_store,195 cleanup="full",196 key_encoder="sha256",197 ) == {198 "num_added": 0,199 "num_deleted": 0,200 "num_skipped": 2,201 "num_updated": 0,202 }203204 loader = ToyLoader(205 documents=[206 Document(207 page_content="mutated document 1",208 ),209 Document(210 page_content="This is another document.", # <-- Same as original211 ),212 ]213 )214215 with patch.object(216 record_manager,217 "get_time",218 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),219 ):220 indexing_result = index(221 loader,222 record_manager,223 vector_store,224 cleanup="full",225 key_encoder="sha256",226 )227228 doc_texts = {229 # Ignoring type since doc should be in the store and not a None230 vector_store.get_by_ids([uid])[0].page_content231 for uid in vector_store.store232 }233 assert doc_texts == {"mutated document 1", "This is another document."}234235 assert indexing_result == {236 "num_added": 1,237 "num_deleted": 1,238 "num_skipped": 1,239 "num_updated": 0,240 }241242 # Attempt to index again verify that nothing changes243 with patch.object(244 record_manager,245 "get_time",246 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),247 ):248 assert index(249 loader,250 record_manager,251 vector_store,252 cleanup="full",253 key_encoder="sha256",254 ) == {255 "num_added": 0,256 "num_deleted": 0,257 "num_skipped": 2,258 "num_updated": 0,259 }260261262async def test_aindex_simple_delete_full(263 arecord_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore264) -> None:265 """Indexing some content to confirm it gets added only once."""266 loader = ToyLoader(267 documents=[268 Document(269 page_content="This is a test document.",270 ),271 Document(272 page_content="This is another document.",273 ),274 ]275 )276277 with patch.object(278 arecord_manager,279 "get_time",280 return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),281 ):282 assert await aindex(283 loader,284 arecord_manager,285 vector_store,286 cleanup="full",287 key_encoder="sha256",288 ) == {289 "num_added": 2,290 "num_deleted": 0,291 "num_skipped": 0,292 "num_updated": 0,293 }294295 with patch.object(296 arecord_manager,297 "get_time",298 return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),299 ):300 assert await aindex(301 loader,302 arecord_manager,303 vector_store,304 cleanup="full",305 key_encoder="sha256",306 ) == {307 "num_added": 0,308 "num_deleted": 0,309 "num_skipped": 2,310 "num_updated": 0,311 }312313 loader = ToyLoader(314 documents=[315 Document(316 page_content="mutated document 1",317 ),318 Document(319 page_content="This is another document.", # <-- Same as original320 ),321 ]322 )323324 with patch.object(325 arecord_manager,326 "get_time",327 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),328 ):329 assert await aindex(330 loader,331 arecord_manager,332 vector_store,333 cleanup="full",334 key_encoder="sha256",335 ) == {336 "num_added": 1,337 "num_deleted": 1,338 "num_skipped": 1,339 "num_updated": 0,340 }341342 doc_texts = {343 # Ignoring type since doc should be in the store and not a None344 vector_store.get_by_ids([uid])[0].page_content345 for uid in vector_store.store346 }347 assert doc_texts == {"mutated document 1", "This is another document."}348349 # Attempt to index again verify that nothing changes350 with patch.object(351 arecord_manager,352 "get_time",353 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),354 ):355 assert await aindex(356 loader,357 arecord_manager,358 vector_store,359 cleanup="full",360 key_encoder="sha256",361 ) == {362 "num_added": 0,363 "num_deleted": 0,364 "num_skipped": 2,365 "num_updated": 0,366 }367368369def test_index_delete_full_recovery_after_deletion_failure(370 record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore371) -> None:372 """Indexing some content to confirm it gets added only once."""373 loader = ToyLoader(374 documents=[375 Document(376 page_content="This is a test document.",377 ),378 Document(379 page_content="This is another document.",380 ),381 ]382 )383384 with patch.object(385 record_manager,386 "get_time",387 return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),388 ):389 assert index(390 loader,391 record_manager,392 vector_store,393 cleanup="full",394 key_encoder="sha256",395 ) == {396 "num_added": 2,397 "num_deleted": 0,398 "num_skipped": 0,399 "num_updated": 0,400 }401402 loader = ToyLoader(403 documents=[404 Document(405 page_content="mutated document 1",406 ),407 Document(408 page_content="This is another document.", # <-- Same as original409 ),410 ]411 )412413 with (414 patch.object(415 record_manager,416 "get_time",417 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),418 ),419 patch.object(vector_store, "delete", return_value=False),420 pytest.raises(IndexingException),421 ):422 indexing_result = index(423 loader,424 record_manager,425 vector_store,426 cleanup="full",427 key_encoder="sha256",428 )429430 # At this point, there should be 3 records in both the record manager431 # and the vector store432 doc_texts = {433 # Ignoring type since doc should be in the store and not a None434 vector_store.get_by_ids([uid])[0].page_content435 for uid in vector_store.store436 }437 assert doc_texts == {438 "This is a test document.",439 "mutated document 1",440 "This is another document.",441 }442443 with patch.object(444 record_manager,445 "get_time",446 return_value=datetime(2021, 1, 3, tzinfo=UTC).timestamp(),447 ):448 indexing_result = index(449 loader,450 record_manager,451 vector_store,452 cleanup="full",453 key_encoder="sha256",454 )455 doc_texts = {456 # Ignoring type since doc should be in the store and not a None457 vector_store.get_by_ids([uid])[0].page_content458 for uid in vector_store.store459 }460 assert doc_texts == {"mutated document 1", "This is another document."}461462 assert indexing_result == {463 "num_added": 0,464 "num_deleted": 1,465 "num_skipped": 2,466 "num_updated": 0,467 }468469470async def test_aindex_delete_full_recovery_after_deletion_failure(471 arecord_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore472) -> None:473 """Indexing some content to confirm it gets added only once."""474 loader = ToyLoader(475 documents=[476 Document(477 page_content="This is a test document.",478 ),479 Document(480 page_content="This is another document.",481 ),482 ]483 )484485 with patch.object(486 arecord_manager,487 "get_time",488 return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),489 ):490 assert await aindex(491 loader,492 arecord_manager,493 vector_store,494 cleanup="full",495 key_encoder="sha256",496 ) == {497 "num_added": 2,498 "num_deleted": 0,499 "num_skipped": 0,500 "num_updated": 0,501 }502503 loader = ToyLoader(504 documents=[505 Document(506 page_content="mutated document 1",507 ),508 Document(509 page_content="This is another document.", # <-- Same as original510 ),511 ]512 )513514 with (515 patch.object(516 arecord_manager,517 "get_time",518 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),519 ),520 patch.object(vector_store, "adelete", return_value=False),521 pytest.raises(IndexingException),522 ):523 indexing_result = await aindex(524 loader,525 arecord_manager,526 vector_store,527 cleanup="full",528 key_encoder="sha256",529 )530531 # At this point, there should be 3 records in both the record manager532 # and the vector store533 doc_texts = {534 # Ignoring type since doc should be in the store and not a None535 vector_store.get_by_ids([uid])[0].page_content536 for uid in vector_store.store537 }538 assert doc_texts == {539 "This is a test document.",540 "mutated document 1",541 "This is another document.",542 }543544 with patch.object(545 arecord_manager,546 "get_time",547 return_value=datetime(2021, 1, 3, tzinfo=UTC).timestamp(),548 ):549 indexing_result = await aindex(550 loader,551 arecord_manager,552 vector_store,553 cleanup="full",554 key_encoder="sha256",555 )556 doc_texts = {557 # Ignoring type since doc should be in the store and not a None558 vector_store.get_by_ids([uid])[0].page_content559 for uid in vector_store.store560 }561 assert doc_texts == {"mutated document 1", "This is another document."}562563 assert indexing_result == {564 "num_added": 0,565 "num_deleted": 1,566 "num_skipped": 2,567 "num_updated": 0,568 }569570571def test_incremental_fails_with_bad_source_ids(572 record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore573) -> None:574 """Test indexing with incremental deletion strategy."""575 loader = ToyLoader(576 documents=[577 Document(578 page_content="This is a test document.",579 metadata={"source": "1"},580 ),581 Document(582 page_content="This is another document.",583 metadata={"source": "2"},584 ),585 Document(586 page_content="This is yet another document.",587 metadata={"source": None},588 ),589 ]590 )591592 with pytest.raises(593 ValueError,594 match="Source id key is required when cleanup mode is "595 "incremental or scoped_full",596 ):597 # Should raise an error because no source id function was specified598 index(599 loader,600 record_manager,601 vector_store,602 cleanup="incremental",603 key_encoder="sha256",604 )605606 with pytest.raises(607 ValueError,608 match="Source IDs are required when cleanup mode is incremental or scoped_full",609 ):610 # Should raise an error because no source id function was specified611 index(612 loader,613 record_manager,614 vector_store,615 cleanup="incremental",616 source_id_key="source",617 key_encoder="sha256",618 )619620621async def test_aincremental_fails_with_bad_source_ids(622 arecord_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore623) -> None:624 """Test indexing with incremental deletion strategy."""625 loader = ToyLoader(626 documents=[627 Document(628 page_content="This is a test document.",629 metadata={"source": "1"},630 ),631 Document(632 page_content="This is another document.",633 metadata={"source": "2"},634 ),635 Document(636 page_content="This is yet another document.",637 metadata={"source": None},638 ),639 ]640 )641642 with pytest.raises(643 ValueError,644 match="Source id key is required when cleanup mode "645 "is incremental or scoped_full",646 ):647 # Should raise an error because no source id function was specified648 await aindex(649 loader,650 arecord_manager,651 vector_store,652 cleanup="incremental",653 key_encoder="sha256",654 )655656 with pytest.raises(657 ValueError,658 match="Source IDs are required when cleanup mode is incremental or scoped_full",659 ):660 # Should raise an error because no source id function was specified661 await aindex(662 loader,663 arecord_manager,664 vector_store,665 cleanup="incremental",666 source_id_key="source",667 key_encoder="sha256",668 )669670671def test_index_simple_delete_scoped_full(672 record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore673) -> None:674 """Test Indexing with scoped_full strategy."""675 loader = ToyLoader(676 documents=[677 Document(678 page_content="This is a test document.",679 metadata={"source": "1"},680 ),681 Document(682 page_content="This is another document.",683 metadata={"source": "1"},684 ),685 Document(686 page_content="This is yet another document.",687 metadata={"source": "1"},688 ),689 Document(690 page_content="This is a test document from another source.",691 metadata={"source": "2"},692 ),693 ]694 )695696 with patch.object(697 record_manager,698 "get_time",699 return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),700 ):701 assert index(702 loader,703 record_manager,704 vector_store,705 cleanup="scoped_full",706 source_id_key="source",707 key_encoder="sha256",708 ) == {709 "num_added": 4,710 "num_deleted": 0,711 "num_skipped": 0,712 "num_updated": 0,713 }714715 with patch.object(716 record_manager,717 "get_time",718 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),719 ):720 assert index(721 loader,722 record_manager,723 vector_store,724 cleanup="scoped_full",725 source_id_key="source",726 key_encoder="sha256",727 ) == {728 "num_added": 0,729 "num_deleted": 0,730 "num_skipped": 4,731 "num_updated": 0,732 }733734 loader = ToyLoader(735 documents=[736 Document(737 page_content="mutated document 1",738 metadata={"source": "1"},739 ),740 Document(741 page_content="This is another document.", # <-- Same as original742 metadata={"source": "1"},743 ),744 ]745 )746747 with patch.object(748 record_manager,749 "get_time",750 return_value=datetime(2021, 1, 3, tzinfo=UTC).timestamp(),751 ):752 assert index(753 loader,754 record_manager,755 vector_store,756 cleanup="scoped_full",757 source_id_key="source",758 key_encoder="sha256",759 ) == {760 "num_added": 1,761 "num_deleted": 2,762 "num_skipped": 1,763 "num_updated": 0,764 }765 doc_texts = {766 # Ignoring type since doc should be in the store and not a None767 vector_store.get_by_ids([uid])[0].page_content768 for uid in vector_store.store769 }770 assert doc_texts == {771 "mutated document 1",772 "This is another document.",773 "This is a test document from another source.",774 }775776 # Attempt to index again verify that nothing changes777 with patch.object(778 record_manager,779 "get_time",780 return_value=datetime(2021, 1, 4, tzinfo=UTC).timestamp(),781 ):782 assert index(783 loader,784 record_manager,785 vector_store,786 cleanup="scoped_full",787 source_id_key="source",788 key_encoder="sha256",789 ) == {790 "num_added": 0,791 "num_deleted": 0,792 "num_skipped": 2,793 "num_updated": 0,794 }795796797async def test_aindex_simple_delete_scoped_full(798 arecord_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore799) -> None:800 """Test Indexing with scoped_full strategy."""801 loader = ToyLoader(802 documents=[803 Document(804 page_content="This is a test document.",805 metadata={"source": "1"},806 ),807 Document(808 page_content="This is another document.",809 metadata={"source": "1"},810 ),811 Document(812 page_content="This is yet another document.",813 metadata={"source": "1"},814 ),815 Document(816 page_content="This is a test document from another source.",817 metadata={"source": "2"},818 ),819 ]820 )821822 with patch.object(823 arecord_manager,824 "get_time",825 return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),826 ):827 assert await aindex(828 loader,829 arecord_manager,830 vector_store,831 cleanup="scoped_full",832 source_id_key="source",833 key_encoder="sha256",834 ) == {835 "num_added": 4,836 "num_deleted": 0,837 "num_skipped": 0,838 "num_updated": 0,839 }840841 with patch.object(842 arecord_manager,843 "get_time",844 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),845 ):846 assert await aindex(847 loader,848 arecord_manager,849 vector_store,850 cleanup="scoped_full",851 source_id_key="source",852 key_encoder="sha256",853 ) == {854 "num_added": 0,855 "num_deleted": 0,856 "num_skipped": 4,857 "num_updated": 0,858 }859860 loader = ToyLoader(861 documents=[862 Document(863 page_content="mutated document 1",864 metadata={"source": "1"},865 ),866 Document(867 page_content="This is another document.", # <-- Same as original868 metadata={"source": "1"},869 ),870 ]871 )872873 with patch.object(874 arecord_manager,875 "get_time",876 return_value=datetime(2021, 1, 3, tzinfo=UTC).timestamp(),877 ):878 assert await aindex(879 loader,880 arecord_manager,881 vector_store,882 cleanup="scoped_full",883 source_id_key="source",884 key_encoder="sha256",885 ) == {886 "num_added": 1,887 "num_deleted": 2,888 "num_skipped": 1,889 "num_updated": 0,890 }891 doc_texts = {892 # Ignoring type since doc should be in the store and not a None893 vector_store.get_by_ids([uid])[0].page_content894 for uid in vector_store.store895 }896 assert doc_texts == {897 "mutated document 1",898 "This is another document.",899 "This is a test document from another source.",900 }901902 # Attempt to index again verify that nothing changes903 with patch.object(904 arecord_manager,905 "get_time",906 return_value=datetime(2021, 1, 4, tzinfo=UTC).timestamp(),907 ):908 assert await aindex(909 loader,910 arecord_manager,911 vector_store,912 cleanup="scoped_full",913 source_id_key="source",914 key_encoder="sha256",915 ) == {916 "num_added": 0,917 "num_deleted": 0,918 "num_skipped": 2,919 "num_updated": 0,920 }921922923def test_scoped_full_fails_with_bad_source_ids(924 record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore925) -> None:926 """Test Indexing with scoped_full strategy."""927 loader = ToyLoader(928 documents=[929 Document(930 page_content="This is a test document.",931 metadata={"source": "1"},932 ),933 Document(934 page_content="This is another document.",935 metadata={"source": "2"},936 ),937 Document(938 page_content="This is yet another document.",939 metadata={"source": None},940 ),941 ]942 )943944 with pytest.raises(945 ValueError,946 match="Source id key is required when cleanup mode "947 "is incremental or scoped_full",948 ):949 # Should raise an error because no source id function was specified950 index(951 loader,952 record_manager,953 vector_store,954 cleanup="scoped_full",955 key_encoder="sha256",956 )957958 with pytest.raises(959 ValueError,960 match="Source IDs are required when cleanup mode is incremental or scoped_full",961 ):962 # Should raise an error because no source id function was specified963 index(964 loader,965 record_manager,966 vector_store,967 cleanup="scoped_full",968 source_id_key="source",969 key_encoder="sha256",970 )971972973async def test_ascoped_full_fails_with_bad_source_ids(974 arecord_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore975) -> None:976 """Test Indexing with scoped_full strategy."""977 loader = ToyLoader(978 documents=[979 Document(980 page_content="This is a test document.",981 metadata={"source": "1"},982 ),983 Document(984 page_content="This is another document.",985 metadata={"source": "2"},986 ),987 Document(988 page_content="This is yet another document.",989 metadata={"source": None},990 ),991 ]992 )993994 with pytest.raises(995 ValueError,996 match="Source id key is required when cleanup mode "997 "is incremental or scoped_full",998 ):999 # Should raise an error because no source id function was specified1000 await aindex(1001 loader,1002 arecord_manager,1003 vector_store,1004 cleanup="scoped_full",1005 key_encoder="sha256",1006 )10071008 with pytest.raises(1009 ValueError,1010 match="Source IDs are required when cleanup mode is incremental or scoped_full",1011 ):1012 # Should raise an error because no source id function was specified1013 await aindex(1014 loader,1015 arecord_manager,1016 vector_store,1017 cleanup="scoped_full",1018 source_id_key="source",1019 key_encoder="sha256",1020 )102110221023def test_index_empty_doc_scoped_full(1024 record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore1025) -> None:1026 """Test Indexing with scoped_full strategy."""1027 loader = ToyLoader(1028 documents=[1029 Document(1030 page_content="This is a test document.",1031 metadata={"source": "1"},1032 ),1033 Document(1034 page_content="This is another document.",1035 metadata={"source": "1"},1036 ),1037 Document(1038 page_content="This is yet another document.",1039 metadata={"source": "1"},1040 ),1041 Document(1042 page_content="This is a test document from another source.",1043 metadata={"source": "2"},1044 ),1045 ]1046 )10471048 with patch.object(1049 record_manager,1050 "get_time",1051 return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),1052 ):1053 assert index(1054 loader,1055 record_manager,1056 vector_store,1057 cleanup="scoped_full",1058 source_id_key="source",1059 key_encoder="sha256",1060 ) == {1061 "num_added": 4,1062 "num_deleted": 0,1063 "num_skipped": 0,1064 "num_updated": 0,1065 }10661067 with patch.object(1068 record_manager,1069 "get_time",1070 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1071 ):1072 assert index(1073 loader,1074 record_manager,1075 vector_store,1076 cleanup="scoped_full",1077 source_id_key="source",1078 key_encoder="sha256",1079 ) == {1080 "num_added": 0,1081 "num_deleted": 0,1082 "num_skipped": 4,1083 "num_updated": 0,1084 }10851086 loader = ToyLoader(documents=[])10871088 with patch.object(1089 record_manager,1090 "get_time",1091 return_value=datetime(2021, 1, 3, tzinfo=UTC).timestamp(),1092 ):1093 assert index(1094 loader,1095 record_manager,1096 vector_store,1097 cleanup="scoped_full",1098 source_id_key="source",1099 key_encoder="sha256",1100 ) == {1101 "num_added": 0,1102 "num_deleted": 0,1103 "num_skipped": 0,1104 "num_updated": 0,1105 }110611071108async def test_aindex_empty_doc_scoped_full(1109 arecord_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore1110) -> None:1111 """Test Indexing with scoped_full strategy."""1112 loader = ToyLoader(1113 documents=[1114 Document(1115 page_content="This is a test document.",1116 metadata={"source": "1"},1117 ),1118 Document(1119 page_content="This is another document.",1120 metadata={"source": "1"},1121 ),1122 Document(1123 page_content="This is yet another document.",1124 metadata={"source": "1"},1125 ),1126 Document(1127 page_content="This is a test document from another source.",1128 metadata={"source": "2"},1129 ),1130 ]1131 )11321133 with patch.object(1134 arecord_manager,1135 "get_time",1136 return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),1137 ):1138 assert await aindex(1139 loader,1140 arecord_manager,1141 vector_store,1142 cleanup="scoped_full",1143 source_id_key="source",1144 key_encoder="sha256",1145 ) == {1146 "num_added": 4,1147 "num_deleted": 0,1148 "num_skipped": 0,1149 "num_updated": 0,1150 }11511152 with patch.object(1153 arecord_manager,1154 "get_time",1155 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1156 ):1157 assert await aindex(1158 loader,1159 arecord_manager,1160 vector_store,1161 cleanup="scoped_full",1162 source_id_key="source",1163 key_encoder="sha256",1164 ) == {1165 "num_added": 0,1166 "num_deleted": 0,1167 "num_skipped": 4,1168 "num_updated": 0,1169 }11701171 loader = ToyLoader(documents=[])11721173 with patch.object(1174 arecord_manager,1175 "get_time",1176 return_value=datetime(2021, 1, 3, tzinfo=UTC).timestamp(),1177 ):1178 assert await aindex(1179 loader,1180 arecord_manager,1181 vector_store,1182 cleanup="scoped_full",1183 source_id_key="source",1184 key_encoder="sha256",1185 ) == {1186 "num_added": 0,1187 "num_deleted": 0,1188 "num_skipped": 0,1189 "num_updated": 0,1190 }119111921193def test_no_delete(1194 record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore1195) -> None:1196 """Test indexing without a deletion strategy."""1197 loader = ToyLoader(1198 documents=[1199 Document(1200 page_content="This is a test document.",1201 metadata={"source": "1"},1202 ),1203 Document(1204 page_content="This is another document.",1205 metadata={"source": "2"},1206 ),1207 ]1208 )12091210 with patch.object(1211 record_manager,1212 "get_time",1213 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1214 ):1215 assert index(1216 loader,1217 record_manager,1218 vector_store,1219 cleanup=None,1220 source_id_key="source",1221 key_encoder="sha256",1222 ) == {1223 "num_added": 2,1224 "num_deleted": 0,1225 "num_skipped": 0,1226 "num_updated": 0,1227 }12281229 # If we add the same content twice it should be skipped1230 with patch.object(1231 record_manager,1232 "get_time",1233 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1234 ):1235 assert index(1236 loader,1237 record_manager,1238 vector_store,1239 cleanup=None,1240 source_id_key="source",1241 key_encoder="sha256",1242 ) == {1243 "num_added": 0,1244 "num_deleted": 0,1245 "num_skipped": 2,1246 "num_updated": 0,1247 }12481249 loader = ToyLoader(1250 documents=[1251 Document(1252 page_content="mutated content",1253 metadata={"source": "1"},1254 ),1255 Document(1256 page_content="This is another document.",1257 metadata={"source": "2"},1258 ),1259 ]1260 )12611262 # Should result in no updates or deletions!1263 with patch.object(1264 record_manager,1265 "get_time",1266 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1267 ):1268 assert index(1269 loader,1270 record_manager,1271 vector_store,1272 cleanup=None,1273 source_id_key="source",1274 key_encoder="sha256",1275 ) == {1276 "num_added": 1,1277 "num_deleted": 0,1278 "num_skipped": 1,1279 "num_updated": 0,1280 }128112821283async def test_ano_delete(1284 arecord_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore1285) -> None:1286 """Test indexing without a deletion strategy."""1287 loader = ToyLoader(1288 documents=[1289 Document(1290 page_content="This is a test document.",1291 metadata={"source": "1"},1292 ),1293 Document(1294 page_content="This is another document.",1295 metadata={"source": "2"},1296 ),1297 ]1298 )12991300 with patch.object(1301 arecord_manager,1302 "get_time",1303 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1304 ):1305 assert await aindex(1306 loader,1307 arecord_manager,1308 vector_store,1309 cleanup=None,1310 source_id_key="source",1311 key_encoder="sha256",1312 ) == {1313 "num_added": 2,1314 "num_deleted": 0,1315 "num_skipped": 0,1316 "num_updated": 0,1317 }13181319 # If we add the same content twice it should be skipped1320 with patch.object(1321 arecord_manager,1322 "get_time",1323 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1324 ):1325 assert await aindex(1326 loader,1327 arecord_manager,1328 vector_store,1329 cleanup=None,1330 source_id_key="source",1331 key_encoder="sha256",1332 ) == {1333 "num_added": 0,1334 "num_deleted": 0,1335 "num_skipped": 2,1336 "num_updated": 0,1337 }13381339 loader = ToyLoader(1340 documents=[1341 Document(1342 page_content="mutated content",1343 metadata={"source": "1"},1344 ),1345 Document(1346 page_content="This is another document.",1347 metadata={"source": "2"},1348 ),1349 ]1350 )13511352 # Should result in no updates or deletions!1353 with patch.object(1354 arecord_manager,1355 "get_time",1356 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1357 ):1358 assert await aindex(1359 loader,1360 arecord_manager,1361 vector_store,1362 cleanup=None,1363 source_id_key="source",1364 key_encoder="sha256",1365 ) == {1366 "num_added": 1,1367 "num_deleted": 0,1368 "num_skipped": 1,1369 "num_updated": 0,1370 }137113721373def test_incremental_delete(1374 record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore1375) -> None:1376 """Test indexing with incremental deletion strategy."""1377 loader = ToyLoader(1378 documents=[1379 Document(1380 page_content="This is a test document.",1381 metadata={"source": "1"},1382 ),1383 Document(1384 page_content="This is another document.",1385 metadata={"source": "2"},1386 ),1387 ]1388 )13891390 with patch.object(1391 record_manager,1392 "get_time",1393 return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),1394 ):1395 assert index(1396 loader,1397 record_manager,1398 vector_store,1399 cleanup="incremental",1400 source_id_key="source",1401 key_encoder="sha256",1402 ) == {1403 "num_added": 2,1404 "num_deleted": 0,1405 "num_skipped": 0,1406 "num_updated": 0,1407 }14081409 doc_texts = {1410 # Ignoring type since doc should be in the store and not a None1411 vector_store.get_by_ids([uid])[0].page_content1412 for uid in vector_store.store1413 }1414 assert doc_texts == {"This is another document.", "This is a test document."}14151416 # Attempt to index again verify that nothing changes1417 with patch.object(1418 record_manager,1419 "get_time",1420 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1421 ):1422 assert index(1423 loader,1424 record_manager,1425 vector_store,1426 cleanup="incremental",1427 source_id_key="source",1428 key_encoder="sha256",1429 ) == {1430 "num_added": 0,1431 "num_deleted": 0,1432 "num_skipped": 2,1433 "num_updated": 0,1434 }14351436 # Create 2 documents from the same source all with mutated content1437 loader = ToyLoader(1438 documents=[1439 Document(1440 page_content="mutated document 1",1441 metadata={"source": "1"},1442 ),1443 Document(1444 page_content="mutated document 2",1445 metadata={"source": "1"},1446 ),1447 Document(1448 page_content="This is another document.", # <-- Same as original1449 metadata={"source": "2"},1450 ),1451 ]1452 )14531454 # Attempt to index again verify that nothing changes1455 with patch.object(1456 record_manager,1457 "get_time",1458 return_value=datetime(2021, 1, 3, tzinfo=UTC).timestamp(),1459 ):1460 assert index(1461 loader,1462 record_manager,1463 vector_store,1464 cleanup="incremental",1465 source_id_key="source",1466 key_encoder="sha256",1467 ) == {1468 "num_added": 2,1469 "num_deleted": 1,1470 "num_skipped": 1,1471 "num_updated": 0,1472 }14731474 doc_texts = {1475 # Ignoring type since doc should be in the store and not a None1476 vector_store.get_by_ids([uid])[0].page_content1477 for uid in vector_store.store1478 }1479 assert doc_texts == {1480 "mutated document 1",1481 "mutated document 2",1482 "This is another document.",1483 }148414851486def test_incremental_delete_with_same_source(1487 record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore1488) -> None:1489 """Test indexing with incremental deletion strategy."""1490 loader = ToyLoader(1491 documents=[1492 Document(1493 page_content="This is a test document.",1494 metadata={"source": "1"},1495 ),1496 Document(1497 page_content="This is another document.",1498 metadata={"source": "1"},1499 ),1500 ]1501 )15021503 with patch.object(1504 record_manager,1505 "get_time",1506 return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),1507 ):1508 assert index(1509 loader,1510 record_manager,1511 vector_store,1512 cleanup="incremental",1513 source_id_key="source",1514 key_encoder="sha256",1515 ) == {1516 "num_added": 2,1517 "num_deleted": 0,1518 "num_skipped": 0,1519 "num_updated": 0,1520 }15211522 doc_texts = {1523 # Ignoring type since doc should be in the store and not a None1524 vector_store.get_by_ids([uid])[0].page_content1525 for uid in vector_store.store1526 }1527 assert doc_texts == {"This is another document.", "This is a test document."}15281529 # Delete 1 document and unchange 1 document1530 loader = ToyLoader(1531 documents=[1532 Document(1533 page_content="This is another document.", # <-- Same as original1534 metadata={"source": "1"},1535 ),1536 ]1537 )15381539 with patch.object(1540 record_manager,1541 "get_time",1542 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1543 ):1544 assert index(1545 loader,1546 record_manager,1547 vector_store,1548 cleanup="incremental",1549 source_id_key="source",1550 key_encoder="sha256",1551 ) == {1552 "num_added": 0,1553 "num_deleted": 1,1554 "num_skipped": 1,1555 "num_updated": 0,1556 }15571558 doc_texts = {1559 # Ignoring type since doc should be in the store and not a None1560 vector_store.get_by_ids([uid])[0].page_content1561 for uid in vector_store.store1562 }1563 assert doc_texts == {1564 "This is another document.",1565 }156615671568def test_incremental_indexing_with_batch_size(1569 record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore1570) -> None:1571 """Test indexing with incremental indexing."""1572 loader = ToyLoader(1573 documents=[1574 Document(1575 page_content="1",1576 metadata={"source": "1"},1577 ),1578 Document(1579 page_content="2",1580 metadata={"source": "1"},1581 ),1582 Document(1583 page_content="3",1584 metadata={"source": "1"},1585 ),1586 Document(1587 page_content="4",1588 metadata={"source": "1"},1589 ),1590 ]1591 )15921593 with patch.object(1594 record_manager,1595 "get_time",1596 return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),1597 ):1598 assert index(1599 loader,1600 record_manager,1601 vector_store,1602 cleanup="incremental",1603 source_id_key="source",1604 batch_size=2,1605 key_encoder="sha256",1606 ) == {1607 "num_added": 4,1608 "num_deleted": 0,1609 "num_skipped": 0,1610 "num_updated": 0,1611 }16121613 doc_texts = {1614 # Ignoring type since doc should be in the store and not a None1615 vector_store.get_by_ids([uid])[0].page_content1616 for uid in vector_store.store1617 }1618 assert doc_texts == {"1", "2", "3", "4"}16191620 with patch.object(1621 record_manager,1622 "get_time",1623 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1624 ):1625 assert index(1626 loader,1627 record_manager,1628 vector_store,1629 cleanup="incremental",1630 source_id_key="source",1631 batch_size=2,1632 key_encoder="sha256",1633 ) == {1634 "num_added": 2,1635 "num_deleted": 2,1636 "num_skipped": 2,1637 "num_updated": 0,1638 }16391640 doc_texts = {1641 # Ignoring type since doc should be in the store and not a None1642 vector_store.get_by_ids([uid])[0].page_content1643 for uid in vector_store.store1644 }1645 assert doc_texts == {"1", "2", "3", "4"}164616471648def test_incremental_delete_with_batch_size(1649 record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore1650) -> None:1651 """Test indexing with incremental deletion strategy and batch size."""1652 loader = ToyLoader(1653 documents=[1654 Document(1655 page_content="1",1656 metadata={"source": "1"},1657 ),1658 Document(1659 page_content="2",1660 metadata={"source": "2"},1661 ),1662 Document(1663 page_content="3",1664 metadata={"source": "3"},1665 ),1666 Document(1667 page_content="4",1668 metadata={"source": "4"},1669 ),1670 ]1671 )16721673 with patch.object(1674 record_manager,1675 "get_time",1676 return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),1677 ):1678 assert index(1679 loader,1680 record_manager,1681 vector_store,1682 cleanup="incremental",1683 source_id_key="source",1684 batch_size=3,1685 key_encoder="sha256",1686 ) == {1687 "num_added": 4,1688 "num_deleted": 0,1689 "num_skipped": 0,1690 "num_updated": 0,1691 }16921693 doc_texts = {1694 # Ignoring type since doc should be in the store and not a None1695 vector_store.get_by_ids([uid])[0].page_content1696 for uid in vector_store.store1697 }1698 assert doc_texts == {"1", "2", "3", "4"}16991700 # Attempt to index again verify that nothing changes1701 with patch.object(1702 record_manager,1703 "get_time",1704 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1705 ):1706 assert index(1707 loader,1708 record_manager,1709 vector_store,1710 cleanup="incremental",1711 source_id_key="source",1712 batch_size=3,1713 key_encoder="sha256",1714 ) == {1715 "num_added": 0,1716 "num_deleted": 0,1717 "num_skipped": 4,1718 "num_updated": 0,1719 }17201721 doc_texts = {1722 # Ignoring type since doc should be in the store and not a None1723 vector_store.get_by_ids([uid])[0].page_content1724 for uid in vector_store.store1725 }1726 assert doc_texts == {"1", "2", "3", "4"}17271728 # Attempt to index again verify that nothing changes1729 with patch.object(1730 record_manager,1731 "get_time",1732 return_value=datetime(2022, 1, 3, tzinfo=UTC).timestamp(),1733 ):1734 # Docs with same content1735 docs = [1736 Document(1737 page_content="1",1738 metadata={"source": "1"},1739 ),1740 Document(1741 page_content="2",1742 metadata={"source": "2"},1743 ),1744 ]1745 assert index(1746 docs,1747 record_manager,1748 vector_store,1749 cleanup="incremental",1750 source_id_key="source",1751 batch_size=1,1752 key_encoder="sha256",1753 ) == {1754 "num_added": 0,1755 "num_deleted": 0,1756 "num_skipped": 2,1757 "num_updated": 0,1758 }17591760 doc_texts = {1761 # Ignoring type since doc should be in the store and not a None1762 vector_store.get_by_ids([uid])[0].page_content1763 for uid in vector_store.store1764 }1765 assert doc_texts == {"1", "2", "3", "4"}17661767 # Attempt to index again verify that nothing changes1768 with patch.object(1769 record_manager,1770 "get_time",1771 return_value=datetime(2023, 1, 4, tzinfo=UTC).timestamp(),1772 ):1773 # Docs with same content1774 docs = [1775 Document(1776 page_content="1",1777 metadata={"source": "1"},1778 ),1779 Document(1780 page_content="2",1781 metadata={"source": "2"},1782 ),1783 ]1784 assert index(1785 docs,1786 record_manager,1787 vector_store,1788 cleanup="incremental",1789 source_id_key="source",1790 batch_size=1,1791 key_encoder="sha256",1792 ) == {1793 "num_added": 0,1794 "num_deleted": 0,1795 "num_skipped": 2,1796 "num_updated": 0,1797 }17981799 doc_texts = {1800 # Ignoring type since doc should be in the store and not a None1801 vector_store.get_by_ids([uid])[0].page_content1802 for uid in vector_store.store1803 }1804 assert doc_texts == {"1", "2", "3", "4"}18051806 # Try to index with changed docs now1807 with patch.object(1808 record_manager,1809 "get_time",1810 return_value=datetime(2024, 1, 5, tzinfo=UTC).timestamp(),1811 ):1812 # Docs with same content1813 docs = [1814 Document(1815 page_content="changed 1",1816 metadata={"source": "1"},1817 ),1818 Document(1819 page_content="changed 2",1820 metadata={"source": "2"},1821 ),1822 ]1823 assert index(1824 docs,1825 record_manager,1826 vector_store,1827 cleanup="incremental",1828 source_id_key="source",1829 key_encoder="sha256",1830 ) == {1831 "num_added": 2,1832 "num_deleted": 2,1833 "num_skipped": 0,1834 "num_updated": 0,1835 }18361837 doc_texts = {1838 # Ignoring type since doc should be in the store and not a None1839 vector_store.get_by_ids([uid])[0].page_content1840 for uid in vector_store.store1841 }1842 assert doc_texts == {"changed 1", "changed 2", "3", "4"}184318441845async def test_aincremental_delete(1846 arecord_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore1847) -> None:1848 """Test indexing with incremental deletion strategy."""1849 loader = ToyLoader(1850 documents=[1851 Document(1852 page_content="This is a test document.",1853 metadata={"source": "1"},1854 ),1855 Document(1856 page_content="This is another document.",1857 metadata={"source": "2"},1858 ),1859 ]1860 )18611862 with patch.object(1863 arecord_manager,1864 "get_time",1865 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1866 ):1867 assert await aindex(1868 loader.lazy_load(),1869 arecord_manager,1870 vector_store,1871 cleanup="incremental",1872 source_id_key="source",1873 key_encoder="sha256",1874 ) == {1875 "num_added": 2,1876 "num_deleted": 0,1877 "num_skipped": 0,1878 "num_updated": 0,1879 }18801881 doc_texts = {1882 # Ignoring type since doc should be in the store and not a None1883 vector_store.get_by_ids([uid])[0].page_content1884 for uid in vector_store.store1885 }1886 assert doc_texts == {"This is another document.", "This is a test document."}18871888 # Attempt to index again verify that nothing changes1889 with patch.object(1890 arecord_manager,1891 "get_time",1892 return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1893 ):1894 assert await aindex(1895 loader.lazy_load(),1896 arecord_manager,1897 vector_store,1898 cleanup="incremental",1899 source_id_key="source",1900 key_encoder="sha256",1901 ) == {1902 "num_added": 0,1903 "num_deleted": 0,1904 "num_skipped": 2,1905 "num_updated": 0,1906 }19071908 # Create 2 documents from the same source all with mutated content1909 loader = ToyLoader(1910 documents=[1911 Document(1912 page_content="mutated document 1",1913 metadata={"source": "1"},1914 ),1915 Document(1916 page_content="mutated document 2",1917 metadata={"source": "1"},1918 ),1919 Document(1920 page_content="This is another document.", # <-- Same as original1921 metadata={"source": "2"},1922 ),1923 ]1924 )19251926 # Attempt to index again verify that nothing changes1927 with patch.object(1928 arecord_manager,1929 "get_time",1930 return_value=datetime(2021, 1, 3, tzinfo=UTC).timestamp(),1931 ):1932 assert await aindex(1933 loader.lazy_load(),1934 arecord_manager,1935 vector_store,1936 cleanup="incremental",1937 source_id_key="source",1938 key_encoder="sha256",1939 ) == {1940 "num_added": 2,1941 "num_deleted": 1,1942 "num_skipped": 1,1943 "num_updated": 0,1944 }19451946 doc_texts = {1947 # Ignoring type since doc should be in the store and not a None1948 vector_store.get_by_ids([uid])[0].page_content1949 for uid in vector_store.store1950 }1951 assert doc_texts == {1952 "mutated document 1",1953 "mutated document 2",1954 "This is another document.",1955 }195619571958def test_indexing_with_no_docs(1959 record_manager: InMemoryRecordManager, vector_store: VectorStore1960) -> None:1961 """Check edge case when loader returns no new docs."""1962 loader = ToyLoader(documents=[])19631964 assert index(1965 loader,1966 record_manager,1967 vector_store,1968 cleanup="full",1969 key_encoder="sha256",1970 ) == {1971 "num_added": 0,1972 "num_deleted": 0,1973 "num_skipped": 0,1974 "num_updated": 0,1975 }197619771978async def test_aindexing_with_no_docs(1979 arecord_manager: InMemoryRecordManager, vector_store: VectorStore1980) -> None:1981 """Check edge case when loader returns no new docs."""1982 loader = ToyLoader(documents=[])19831984 assert await aindex(1985 loader,1986 arecord_manager,1987 vector_store,1988 cleanup="full",1989 key_encoder="sha256",1990 ) == {1991 "num_added": 0,1992 "num_deleted": 0,1993 "num_skipped": 0,1994 "num_updated": 0,1995 }199619971998def test_deduplication(1999 record_manager: InMemoryRecordManager, vector_store: VectorStore2000) -> None:
Same data, no extra tab — call code_get_file + code_get_findings over MCP from Claude/Cursor/Copilot.
Generated by searchcode.com · powered by code_get_file + code_get_findings