libs/core/tests/unit_tests/indexing/test_indexing.py PYTHON 2,964 lines View on github.com → Search inside
File is large — showing lines 1–2,000 of 2,964.
1from collections.abc import AsyncIterator, Iterable, Iterator, Sequence2from datetime import UTC, datetime3from typing import (4    Any,5)6from unittest.mock import AsyncMock, MagicMock, patch78import pytest9import pytest_asyncio10from pytest_mock import MockerFixture1112from langchain_core.document_loaders.base import BaseLoader13from langchain_core.documents import Document14from langchain_core.embeddings import DeterministicFakeEmbedding15from langchain_core.indexing import InMemoryRecordManager, aindex, index16from langchain_core.indexing.api import (17    IndexingException,18    _abatch,19    _batch,20    _get_document_with_hash,21)22from langchain_core.indexing.in_memory import InMemoryDocumentIndex23from langchain_core.vectorstores import InMemoryVectorStore, VectorStore242526class ToyLoader(BaseLoader):27    """Toy loader that always returns the same documents."""2829    def __init__(self, documents: Sequence[Document]) -> None:30        """Initialize with the documents to return."""31        self.documents = documents3233    def lazy_load(34        self,35    ) -> Iterator[Document]:36        yield from self.documents3738    async def alazy_load(39        self,40    ) -> AsyncIterator[Document]:41        for document in self.documents:42            yield document434445@pytest.fixture46def record_manager() -> InMemoryRecordManager:47    """Timestamped set fixture."""48    record_manager = InMemoryRecordManager(namespace="hello")49    record_manager.create_schema()50    return record_manager515253@pytest_asyncio.fixture54async def arecord_manager() -> InMemoryRecordManager:55    """Timestamped set fixture."""56    record_manager = InMemoryRecordManager(namespace="hello")57    await record_manager.acreate_schema()58    return record_manager596061@pytest.fixture62def vector_store() -> InMemoryVectorStore:63    """Vector store fixture."""64    embeddings = DeterministicFakeEmbedding(size=5)65    return InMemoryVectorStore(embeddings)666768@pytest.fixture69def upserting_vector_store() -> InMemoryVectorStore:70    """Vector store fixture."""71    embeddings = DeterministicFakeEmbedding(size=5)72    return InMemoryVectorStore(embeddings)737475def test_indexing_same_content(76    record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore77) -> None:78    """Indexing some content to confirm it gets added only once."""79    loader = ToyLoader(80        documents=[81            Document(82                page_content="This is a test document.",83            ),84            Document(85                page_content="This is another document.",86            ),87        ]88    )8990    assert index(loader, record_manager, vector_store, key_encoder="sha256") == {91        "num_added": 2,92        "num_deleted": 0,93        "num_skipped": 0,94        "num_updated": 0,95    }9697    assert len(list(vector_store.store)) == 29899    for _ in range(2):100        # Run the indexing again101        assert index(loader, record_manager, vector_store, key_encoder="sha256") == {102            "num_added": 0,103            "num_deleted": 0,104            "num_skipped": 2,105            "num_updated": 0,106        }107108109async def test_aindexing_same_content(110    arecord_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore111) -> None:112    """Indexing some content to confirm it gets added only once."""113    loader = ToyLoader(114        documents=[115            Document(116                page_content="This is a test document.",117            ),118            Document(119                page_content="This is another document.",120            ),121        ]122    )123124    assert await aindex(125        loader,126        arecord_manager,127        vector_store,128        key_encoder="sha256",129    ) == {130        "num_added": 2,131        "num_deleted": 0,132        "num_skipped": 0,133        "num_updated": 0,134    }135136    assert len(list(vector_store.store)) == 2137138    for _ in range(2):139        # Run the indexing again140        assert await aindex(141            loader,142            arecord_manager,143            vector_store,144            key_encoder="sha256",145        ) == {146            "num_added": 0,147            "num_deleted": 0,148            "num_skipped": 2,149            "num_updated": 0,150        }151152153def test_index_simple_delete_full(154    record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore155) -> None:156    """Indexing some content to confirm it gets added only once."""157    loader = ToyLoader(158        documents=[159            Document(160                page_content="This is a test document.",161            ),162            Document(163                page_content="This is another document.",164            ),165        ]166    )167168    with patch.object(169        record_manager,170        "get_time",171        return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),172    ):173        assert index(174            loader,175            record_manager,176            vector_store,177            cleanup="full",178            key_encoder="sha256",179        ) == {180            "num_added": 2,181            "num_deleted": 0,182            "num_skipped": 0,183            "num_updated": 0,184        }185186    with patch.object(187        record_manager,188        "get_time",189        return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),190    ):191        assert index(192            loader,193            record_manager,194            vector_store,195            cleanup="full",196            key_encoder="sha256",197        ) == {198            "num_added": 0,199            "num_deleted": 0,200            "num_skipped": 2,201            "num_updated": 0,202        }203204    loader = ToyLoader(205        documents=[206            Document(207                page_content="mutated document 1",208            ),209            Document(210                page_content="This is another document.",  # <-- Same as original211            ),212        ]213    )214215    with patch.object(216        record_manager,217        "get_time",218        return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),219    ):220        indexing_result = index(221            loader,222            record_manager,223            vector_store,224            cleanup="full",225            key_encoder="sha256",226        )227228        doc_texts = {229            # Ignoring type since doc should be in the store and not a None230            vector_store.get_by_ids([uid])[0].page_content231            for uid in vector_store.store232        }233        assert doc_texts == {"mutated document 1", "This is another document."}234235        assert indexing_result == {236            "num_added": 1,237            "num_deleted": 1,238            "num_skipped": 1,239            "num_updated": 0,240        }241242    # Attempt to index again verify that nothing changes243    with patch.object(244        record_manager,245        "get_time",246        return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),247    ):248        assert index(249            loader,250            record_manager,251            vector_store,252            cleanup="full",253            key_encoder="sha256",254        ) == {255            "num_added": 0,256            "num_deleted": 0,257            "num_skipped": 2,258            "num_updated": 0,259        }260261262async def test_aindex_simple_delete_full(263    arecord_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore264) -> None:265    """Indexing some content to confirm it gets added only once."""266    loader = ToyLoader(267        documents=[268            Document(269                page_content="This is a test document.",270            ),271            Document(272                page_content="This is another document.",273            ),274        ]275    )276277    with patch.object(278        arecord_manager,279        "get_time",280        return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),281    ):282        assert await aindex(283            loader,284            arecord_manager,285            vector_store,286            cleanup="full",287            key_encoder="sha256",288        ) == {289            "num_added": 2,290            "num_deleted": 0,291            "num_skipped": 0,292            "num_updated": 0,293        }294295    with patch.object(296        arecord_manager,297        "get_time",298        return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),299    ):300        assert await aindex(301            loader,302            arecord_manager,303            vector_store,304            cleanup="full",305            key_encoder="sha256",306        ) == {307            "num_added": 0,308            "num_deleted": 0,309            "num_skipped": 2,310            "num_updated": 0,311        }312313    loader = ToyLoader(314        documents=[315            Document(316                page_content="mutated document 1",317            ),318            Document(319                page_content="This is another document.",  # <-- Same as original320            ),321        ]322    )323324    with patch.object(325        arecord_manager,326        "get_time",327        return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),328    ):329        assert await aindex(330            loader,331            arecord_manager,332            vector_store,333            cleanup="full",334            key_encoder="sha256",335        ) == {336            "num_added": 1,337            "num_deleted": 1,338            "num_skipped": 1,339            "num_updated": 0,340        }341342    doc_texts = {343        # Ignoring type since doc should be in the store and not a None344        vector_store.get_by_ids([uid])[0].page_content345        for uid in vector_store.store346    }347    assert doc_texts == {"mutated document 1", "This is another document."}348349    # Attempt to index again verify that nothing changes350    with patch.object(351        arecord_manager,352        "get_time",353        return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),354    ):355        assert await aindex(356            loader,357            arecord_manager,358            vector_store,359            cleanup="full",360            key_encoder="sha256",361        ) == {362            "num_added": 0,363            "num_deleted": 0,364            "num_skipped": 2,365            "num_updated": 0,366        }367368369def test_index_delete_full_recovery_after_deletion_failure(370    record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore371) -> None:372    """Indexing some content to confirm it gets added only once."""373    loader = ToyLoader(374        documents=[375            Document(376                page_content="This is a test document.",377            ),378            Document(379                page_content="This is another document.",380            ),381        ]382    )383384    with patch.object(385        record_manager,386        "get_time",387        return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),388    ):389        assert index(390            loader,391            record_manager,392            vector_store,393            cleanup="full",394            key_encoder="sha256",395        ) == {396            "num_added": 2,397            "num_deleted": 0,398            "num_skipped": 0,399            "num_updated": 0,400        }401402    loader = ToyLoader(403        documents=[404            Document(405                page_content="mutated document 1",406            ),407            Document(408                page_content="This is another document.",  # <-- Same as original409            ),410        ]411    )412413    with (414        patch.object(415            record_manager,416            "get_time",417            return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),418        ),419        patch.object(vector_store, "delete", return_value=False),420        pytest.raises(IndexingException),421    ):422        indexing_result = index(423            loader,424            record_manager,425            vector_store,426            cleanup="full",427            key_encoder="sha256",428        )429430    # At this point, there should be 3 records in both the record manager431    # and the vector store432    doc_texts = {433        # Ignoring type since doc should be in the store and not a None434        vector_store.get_by_ids([uid])[0].page_content435        for uid in vector_store.store436    }437    assert doc_texts == {438        "This is a test document.",439        "mutated document 1",440        "This is another document.",441    }442443    with patch.object(444        record_manager,445        "get_time",446        return_value=datetime(2021, 1, 3, tzinfo=UTC).timestamp(),447    ):448        indexing_result = index(449            loader,450            record_manager,451            vector_store,452            cleanup="full",453            key_encoder="sha256",454        )455        doc_texts = {456            # Ignoring type since doc should be in the store and not a None457            vector_store.get_by_ids([uid])[0].page_content458            for uid in vector_store.store459        }460        assert doc_texts == {"mutated document 1", "This is another document."}461462    assert indexing_result == {463        "num_added": 0,464        "num_deleted": 1,465        "num_skipped": 2,466        "num_updated": 0,467    }468469470async def test_aindex_delete_full_recovery_after_deletion_failure(471    arecord_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore472) -> None:473    """Indexing some content to confirm it gets added only once."""474    loader = ToyLoader(475        documents=[476            Document(477                page_content="This is a test document.",478            ),479            Document(480                page_content="This is another document.",481            ),482        ]483    )484485    with patch.object(486        arecord_manager,487        "get_time",488        return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),489    ):490        assert await aindex(491            loader,492            arecord_manager,493            vector_store,494            cleanup="full",495            key_encoder="sha256",496        ) == {497            "num_added": 2,498            "num_deleted": 0,499            "num_skipped": 0,500            "num_updated": 0,501        }502503    loader = ToyLoader(504        documents=[505            Document(506                page_content="mutated document 1",507            ),508            Document(509                page_content="This is another document.",  # <-- Same as original510            ),511        ]512    )513514    with (515        patch.object(516            arecord_manager,517            "get_time",518            return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),519        ),520        patch.object(vector_store, "adelete", return_value=False),521        pytest.raises(IndexingException),522    ):523        indexing_result = await aindex(524            loader,525            arecord_manager,526            vector_store,527            cleanup="full",528            key_encoder="sha256",529        )530531    # At this point, there should be 3 records in both the record manager532    # and the vector store533    doc_texts = {534        # Ignoring type since doc should be in the store and not a None535        vector_store.get_by_ids([uid])[0].page_content536        for uid in vector_store.store537    }538    assert doc_texts == {539        "This is a test document.",540        "mutated document 1",541        "This is another document.",542    }543544    with patch.object(545        arecord_manager,546        "get_time",547        return_value=datetime(2021, 1, 3, tzinfo=UTC).timestamp(),548    ):549        indexing_result = await aindex(550            loader,551            arecord_manager,552            vector_store,553            cleanup="full",554            key_encoder="sha256",555        )556        doc_texts = {557            # Ignoring type since doc should be in the store and not a None558            vector_store.get_by_ids([uid])[0].page_content559            for uid in vector_store.store560        }561        assert doc_texts == {"mutated document 1", "This is another document."}562563    assert indexing_result == {564        "num_added": 0,565        "num_deleted": 1,566        "num_skipped": 2,567        "num_updated": 0,568    }569570571def test_incremental_fails_with_bad_source_ids(572    record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore573) -> None:574    """Test indexing with incremental deletion strategy."""575    loader = ToyLoader(576        documents=[577            Document(578                page_content="This is a test document.",579                metadata={"source": "1"},580            ),581            Document(582                page_content="This is another document.",583                metadata={"source": "2"},584            ),585            Document(586                page_content="This is yet another document.",587                metadata={"source": None},588            ),589        ]590    )591592    with pytest.raises(593        ValueError,594        match="Source id key is required when cleanup mode is "595        "incremental or scoped_full",596    ):597        # Should raise an error because no source id function was specified598        index(599            loader,600            record_manager,601            vector_store,602            cleanup="incremental",603            key_encoder="sha256",604        )605606    with pytest.raises(607        ValueError,608        match="Source IDs are required when cleanup mode is incremental or scoped_full",609    ):610        # Should raise an error because no source id function was specified611        index(612            loader,613            record_manager,614            vector_store,615            cleanup="incremental",616            source_id_key="source",617            key_encoder="sha256",618        )619620621async def test_aincremental_fails_with_bad_source_ids(622    arecord_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore623) -> None:624    """Test indexing with incremental deletion strategy."""625    loader = ToyLoader(626        documents=[627            Document(628                page_content="This is a test document.",629                metadata={"source": "1"},630            ),631            Document(632                page_content="This is another document.",633                metadata={"source": "2"},634            ),635            Document(636                page_content="This is yet another document.",637                metadata={"source": None},638            ),639        ]640    )641642    with pytest.raises(643        ValueError,644        match="Source id key is required when cleanup mode "645        "is incremental or scoped_full",646    ):647        # Should raise an error because no source id function was specified648        await aindex(649            loader,650            arecord_manager,651            vector_store,652            cleanup="incremental",653            key_encoder="sha256",654        )655656    with pytest.raises(657        ValueError,658        match="Source IDs are required when cleanup mode is incremental or scoped_full",659    ):660        # Should raise an error because no source id function was specified661        await aindex(662            loader,663            arecord_manager,664            vector_store,665            cleanup="incremental",666            source_id_key="source",667            key_encoder="sha256",668        )669670671def test_index_simple_delete_scoped_full(672    record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore673) -> None:674    """Test Indexing with scoped_full strategy."""675    loader = ToyLoader(676        documents=[677            Document(678                page_content="This is a test document.",679                metadata={"source": "1"},680            ),681            Document(682                page_content="This is another document.",683                metadata={"source": "1"},684            ),685            Document(686                page_content="This is yet another document.",687                metadata={"source": "1"},688            ),689            Document(690                page_content="This is a test document from another source.",691                metadata={"source": "2"},692            ),693        ]694    )695696    with patch.object(697        record_manager,698        "get_time",699        return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),700    ):701        assert index(702            loader,703            record_manager,704            vector_store,705            cleanup="scoped_full",706            source_id_key="source",707            key_encoder="sha256",708        ) == {709            "num_added": 4,710            "num_deleted": 0,711            "num_skipped": 0,712            "num_updated": 0,713        }714715    with patch.object(716        record_manager,717        "get_time",718        return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),719    ):720        assert index(721            loader,722            record_manager,723            vector_store,724            cleanup="scoped_full",725            source_id_key="source",726            key_encoder="sha256",727        ) == {728            "num_added": 0,729            "num_deleted": 0,730            "num_skipped": 4,731            "num_updated": 0,732        }733734    loader = ToyLoader(735        documents=[736            Document(737                page_content="mutated document 1",738                metadata={"source": "1"},739            ),740            Document(741                page_content="This is another document.",  # <-- Same as original742                metadata={"source": "1"},743            ),744        ]745    )746747    with patch.object(748        record_manager,749        "get_time",750        return_value=datetime(2021, 1, 3, tzinfo=UTC).timestamp(),751    ):752        assert index(753            loader,754            record_manager,755            vector_store,756            cleanup="scoped_full",757            source_id_key="source",758            key_encoder="sha256",759        ) == {760            "num_added": 1,761            "num_deleted": 2,762            "num_skipped": 1,763            "num_updated": 0,764        }765        doc_texts = {766            # Ignoring type since doc should be in the store and not a None767            vector_store.get_by_ids([uid])[0].page_content768            for uid in vector_store.store769        }770        assert doc_texts == {771            "mutated document 1",772            "This is another document.",773            "This is a test document from another source.",774        }775776    # Attempt to index again verify that nothing changes777    with patch.object(778        record_manager,779        "get_time",780        return_value=datetime(2021, 1, 4, tzinfo=UTC).timestamp(),781    ):782        assert index(783            loader,784            record_manager,785            vector_store,786            cleanup="scoped_full",787            source_id_key="source",788            key_encoder="sha256",789        ) == {790            "num_added": 0,791            "num_deleted": 0,792            "num_skipped": 2,793            "num_updated": 0,794        }795796797async def test_aindex_simple_delete_scoped_full(798    arecord_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore799) -> None:800    """Test Indexing with scoped_full strategy."""801    loader = ToyLoader(802        documents=[803            Document(804                page_content="This is a test document.",805                metadata={"source": "1"},806            ),807            Document(808                page_content="This is another document.",809                metadata={"source": "1"},810            ),811            Document(812                page_content="This is yet another document.",813                metadata={"source": "1"},814            ),815            Document(816                page_content="This is a test document from another source.",817                metadata={"source": "2"},818            ),819        ]820    )821822    with patch.object(823        arecord_manager,824        "get_time",825        return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),826    ):827        assert await aindex(828            loader,829            arecord_manager,830            vector_store,831            cleanup="scoped_full",832            source_id_key="source",833            key_encoder="sha256",834        ) == {835            "num_added": 4,836            "num_deleted": 0,837            "num_skipped": 0,838            "num_updated": 0,839        }840841    with patch.object(842        arecord_manager,843        "get_time",844        return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),845    ):846        assert await aindex(847            loader,848            arecord_manager,849            vector_store,850            cleanup="scoped_full",851            source_id_key="source",852            key_encoder="sha256",853        ) == {854            "num_added": 0,855            "num_deleted": 0,856            "num_skipped": 4,857            "num_updated": 0,858        }859860    loader = ToyLoader(861        documents=[862            Document(863                page_content="mutated document 1",864                metadata={"source": "1"},865            ),866            Document(867                page_content="This is another document.",  # <-- Same as original868                metadata={"source": "1"},869            ),870        ]871    )872873    with patch.object(874        arecord_manager,875        "get_time",876        return_value=datetime(2021, 1, 3, tzinfo=UTC).timestamp(),877    ):878        assert await aindex(879            loader,880            arecord_manager,881            vector_store,882            cleanup="scoped_full",883            source_id_key="source",884            key_encoder="sha256",885        ) == {886            "num_added": 1,887            "num_deleted": 2,888            "num_skipped": 1,889            "num_updated": 0,890        }891        doc_texts = {892            # Ignoring type since doc should be in the store and not a None893            vector_store.get_by_ids([uid])[0].page_content894            for uid in vector_store.store895        }896        assert doc_texts == {897            "mutated document 1",898            "This is another document.",899            "This is a test document from another source.",900        }901902    # Attempt to index again verify that nothing changes903    with patch.object(904        arecord_manager,905        "get_time",906        return_value=datetime(2021, 1, 4, tzinfo=UTC).timestamp(),907    ):908        assert await aindex(909            loader,910            arecord_manager,911            vector_store,912            cleanup="scoped_full",913            source_id_key="source",914            key_encoder="sha256",915        ) == {916            "num_added": 0,917            "num_deleted": 0,918            "num_skipped": 2,919            "num_updated": 0,920        }921922923def test_scoped_full_fails_with_bad_source_ids(924    record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore925) -> None:926    """Test Indexing with scoped_full strategy."""927    loader = ToyLoader(928        documents=[929            Document(930                page_content="This is a test document.",931                metadata={"source": "1"},932            ),933            Document(934                page_content="This is another document.",935                metadata={"source": "2"},936            ),937            Document(938                page_content="This is yet another document.",939                metadata={"source": None},940            ),941        ]942    )943944    with pytest.raises(945        ValueError,946        match="Source id key is required when cleanup mode "947        "is incremental or scoped_full",948    ):949        # Should raise an error because no source id function was specified950        index(951            loader,952            record_manager,953            vector_store,954            cleanup="scoped_full",955            key_encoder="sha256",956        )957958    with pytest.raises(959        ValueError,960        match="Source IDs are required when cleanup mode is incremental or scoped_full",961    ):962        # Should raise an error because no source id function was specified963        index(964            loader,965            record_manager,966            vector_store,967            cleanup="scoped_full",968            source_id_key="source",969            key_encoder="sha256",970        )971972973async def test_ascoped_full_fails_with_bad_source_ids(974    arecord_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore975) -> None:976    """Test Indexing with scoped_full strategy."""977    loader = ToyLoader(978        documents=[979            Document(980                page_content="This is a test document.",981                metadata={"source": "1"},982            ),983            Document(984                page_content="This is another document.",985                metadata={"source": "2"},986            ),987            Document(988                page_content="This is yet another document.",989                metadata={"source": None},990            ),991        ]992    )993994    with pytest.raises(995        ValueError,996        match="Source id key is required when cleanup mode "997        "is incremental or scoped_full",998    ):999        # Should raise an error because no source id function was specified1000        await aindex(1001            loader,1002            arecord_manager,1003            vector_store,1004            cleanup="scoped_full",1005            key_encoder="sha256",1006        )10071008    with pytest.raises(1009        ValueError,1010        match="Source IDs are required when cleanup mode is incremental or scoped_full",1011    ):1012        # Should raise an error because no source id function was specified1013        await aindex(1014            loader,1015            arecord_manager,1016            vector_store,1017            cleanup="scoped_full",1018            source_id_key="source",1019            key_encoder="sha256",1020        )102110221023def test_index_empty_doc_scoped_full(1024    record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore1025) -> None:1026    """Test Indexing with scoped_full strategy."""1027    loader = ToyLoader(1028        documents=[1029            Document(1030                page_content="This is a test document.",1031                metadata={"source": "1"},1032            ),1033            Document(1034                page_content="This is another document.",1035                metadata={"source": "1"},1036            ),1037            Document(1038                page_content="This is yet another document.",1039                metadata={"source": "1"},1040            ),1041            Document(1042                page_content="This is a test document from another source.",1043                metadata={"source": "2"},1044            ),1045        ]1046    )10471048    with patch.object(1049        record_manager,1050        "get_time",1051        return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),1052    ):1053        assert index(1054            loader,1055            record_manager,1056            vector_store,1057            cleanup="scoped_full",1058            source_id_key="source",1059            key_encoder="sha256",1060        ) == {1061            "num_added": 4,1062            "num_deleted": 0,1063            "num_skipped": 0,1064            "num_updated": 0,1065        }10661067    with patch.object(1068        record_manager,1069        "get_time",1070        return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1071    ):1072        assert index(1073            loader,1074            record_manager,1075            vector_store,1076            cleanup="scoped_full",1077            source_id_key="source",1078            key_encoder="sha256",1079        ) == {1080            "num_added": 0,1081            "num_deleted": 0,1082            "num_skipped": 4,1083            "num_updated": 0,1084        }10851086    loader = ToyLoader(documents=[])10871088    with patch.object(1089        record_manager,1090        "get_time",1091        return_value=datetime(2021, 1, 3, tzinfo=UTC).timestamp(),1092    ):1093        assert index(1094            loader,1095            record_manager,1096            vector_store,1097            cleanup="scoped_full",1098            source_id_key="source",1099            key_encoder="sha256",1100        ) == {1101            "num_added": 0,1102            "num_deleted": 0,1103            "num_skipped": 0,1104            "num_updated": 0,1105        }110611071108async def test_aindex_empty_doc_scoped_full(1109    arecord_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore1110) -> None:1111    """Test Indexing with scoped_full strategy."""1112    loader = ToyLoader(1113        documents=[1114            Document(1115                page_content="This is a test document.",1116                metadata={"source": "1"},1117            ),1118            Document(1119                page_content="This is another document.",1120                metadata={"source": "1"},1121            ),1122            Document(1123                page_content="This is yet another document.",1124                metadata={"source": "1"},1125            ),1126            Document(1127                page_content="This is a test document from another source.",1128                metadata={"source": "2"},1129            ),1130        ]1131    )11321133    with patch.object(1134        arecord_manager,1135        "get_time",1136        return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),1137    ):1138        assert await aindex(1139            loader,1140            arecord_manager,1141            vector_store,1142            cleanup="scoped_full",1143            source_id_key="source",1144            key_encoder="sha256",1145        ) == {1146            "num_added": 4,1147            "num_deleted": 0,1148            "num_skipped": 0,1149            "num_updated": 0,1150        }11511152    with patch.object(1153        arecord_manager,1154        "get_time",1155        return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1156    ):1157        assert await aindex(1158            loader,1159            arecord_manager,1160            vector_store,1161            cleanup="scoped_full",1162            source_id_key="source",1163            key_encoder="sha256",1164        ) == {1165            "num_added": 0,1166            "num_deleted": 0,1167            "num_skipped": 4,1168            "num_updated": 0,1169        }11701171    loader = ToyLoader(documents=[])11721173    with patch.object(1174        arecord_manager,1175        "get_time",1176        return_value=datetime(2021, 1, 3, tzinfo=UTC).timestamp(),1177    ):1178        assert await aindex(1179            loader,1180            arecord_manager,1181            vector_store,1182            cleanup="scoped_full",1183            source_id_key="source",1184            key_encoder="sha256",1185        ) == {1186            "num_added": 0,1187            "num_deleted": 0,1188            "num_skipped": 0,1189            "num_updated": 0,1190        }119111921193def test_no_delete(1194    record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore1195) -> None:1196    """Test indexing without a deletion strategy."""1197    loader = ToyLoader(1198        documents=[1199            Document(1200                page_content="This is a test document.",1201                metadata={"source": "1"},1202            ),1203            Document(1204                page_content="This is another document.",1205                metadata={"source": "2"},1206            ),1207        ]1208    )12091210    with patch.object(1211        record_manager,1212        "get_time",1213        return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1214    ):1215        assert index(1216            loader,1217            record_manager,1218            vector_store,1219            cleanup=None,1220            source_id_key="source",1221            key_encoder="sha256",1222        ) == {1223            "num_added": 2,1224            "num_deleted": 0,1225            "num_skipped": 0,1226            "num_updated": 0,1227        }12281229    # If we add the same content twice it should be skipped1230    with patch.object(1231        record_manager,1232        "get_time",1233        return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1234    ):1235        assert index(1236            loader,1237            record_manager,1238            vector_store,1239            cleanup=None,1240            source_id_key="source",1241            key_encoder="sha256",1242        ) == {1243            "num_added": 0,1244            "num_deleted": 0,1245            "num_skipped": 2,1246            "num_updated": 0,1247        }12481249    loader = ToyLoader(1250        documents=[1251            Document(1252                page_content="mutated content",1253                metadata={"source": "1"},1254            ),1255            Document(1256                page_content="This is another document.",1257                metadata={"source": "2"},1258            ),1259        ]1260    )12611262    # Should result in no updates or deletions!1263    with patch.object(1264        record_manager,1265        "get_time",1266        return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1267    ):1268        assert index(1269            loader,1270            record_manager,1271            vector_store,1272            cleanup=None,1273            source_id_key="source",1274            key_encoder="sha256",1275        ) == {1276            "num_added": 1,1277            "num_deleted": 0,1278            "num_skipped": 1,1279            "num_updated": 0,1280        }128112821283async def test_ano_delete(1284    arecord_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore1285) -> None:1286    """Test indexing without a deletion strategy."""1287    loader = ToyLoader(1288        documents=[1289            Document(1290                page_content="This is a test document.",1291                metadata={"source": "1"},1292            ),1293            Document(1294                page_content="This is another document.",1295                metadata={"source": "2"},1296            ),1297        ]1298    )12991300    with patch.object(1301        arecord_manager,1302        "get_time",1303        return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1304    ):1305        assert await aindex(1306            loader,1307            arecord_manager,1308            vector_store,1309            cleanup=None,1310            source_id_key="source",1311            key_encoder="sha256",1312        ) == {1313            "num_added": 2,1314            "num_deleted": 0,1315            "num_skipped": 0,1316            "num_updated": 0,1317        }13181319    # If we add the same content twice it should be skipped1320    with patch.object(1321        arecord_manager,1322        "get_time",1323        return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1324    ):1325        assert await aindex(1326            loader,1327            arecord_manager,1328            vector_store,1329            cleanup=None,1330            source_id_key="source",1331            key_encoder="sha256",1332        ) == {1333            "num_added": 0,1334            "num_deleted": 0,1335            "num_skipped": 2,1336            "num_updated": 0,1337        }13381339    loader = ToyLoader(1340        documents=[1341            Document(1342                page_content="mutated content",1343                metadata={"source": "1"},1344            ),1345            Document(1346                page_content="This is another document.",1347                metadata={"source": "2"},1348            ),1349        ]1350    )13511352    # Should result in no updates or deletions!1353    with patch.object(1354        arecord_manager,1355        "get_time",1356        return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1357    ):1358        assert await aindex(1359            loader,1360            arecord_manager,1361            vector_store,1362            cleanup=None,1363            source_id_key="source",1364            key_encoder="sha256",1365        ) == {1366            "num_added": 1,1367            "num_deleted": 0,1368            "num_skipped": 1,1369            "num_updated": 0,1370        }137113721373def test_incremental_delete(1374    record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore1375) -> None:1376    """Test indexing with incremental deletion strategy."""1377    loader = ToyLoader(1378        documents=[1379            Document(1380                page_content="This is a test document.",1381                metadata={"source": "1"},1382            ),1383            Document(1384                page_content="This is another document.",1385                metadata={"source": "2"},1386            ),1387        ]1388    )13891390    with patch.object(1391        record_manager,1392        "get_time",1393        return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),1394    ):1395        assert index(1396            loader,1397            record_manager,1398            vector_store,1399            cleanup="incremental",1400            source_id_key="source",1401            key_encoder="sha256",1402        ) == {1403            "num_added": 2,1404            "num_deleted": 0,1405            "num_skipped": 0,1406            "num_updated": 0,1407        }14081409    doc_texts = {1410        # Ignoring type since doc should be in the store and not a None1411        vector_store.get_by_ids([uid])[0].page_content1412        for uid in vector_store.store1413    }1414    assert doc_texts == {"This is another document.", "This is a test document."}14151416    # Attempt to index again verify that nothing changes1417    with patch.object(1418        record_manager,1419        "get_time",1420        return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1421    ):1422        assert index(1423            loader,1424            record_manager,1425            vector_store,1426            cleanup="incremental",1427            source_id_key="source",1428            key_encoder="sha256",1429        ) == {1430            "num_added": 0,1431            "num_deleted": 0,1432            "num_skipped": 2,1433            "num_updated": 0,1434        }14351436    # Create 2 documents from the same source all with mutated content1437    loader = ToyLoader(1438        documents=[1439            Document(1440                page_content="mutated document 1",1441                metadata={"source": "1"},1442            ),1443            Document(1444                page_content="mutated document 2",1445                metadata={"source": "1"},1446            ),1447            Document(1448                page_content="This is another document.",  # <-- Same as original1449                metadata={"source": "2"},1450            ),1451        ]1452    )14531454    # Attempt to index again verify that nothing changes1455    with patch.object(1456        record_manager,1457        "get_time",1458        return_value=datetime(2021, 1, 3, tzinfo=UTC).timestamp(),1459    ):1460        assert index(1461            loader,1462            record_manager,1463            vector_store,1464            cleanup="incremental",1465            source_id_key="source",1466            key_encoder="sha256",1467        ) == {1468            "num_added": 2,1469            "num_deleted": 1,1470            "num_skipped": 1,1471            "num_updated": 0,1472        }14731474    doc_texts = {1475        # Ignoring type since doc should be in the store and not a None1476        vector_store.get_by_ids([uid])[0].page_content1477        for uid in vector_store.store1478    }1479    assert doc_texts == {1480        "mutated document 1",1481        "mutated document 2",1482        "This is another document.",1483    }148414851486def test_incremental_delete_with_same_source(1487    record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore1488) -> None:1489    """Test indexing with incremental deletion strategy."""1490    loader = ToyLoader(1491        documents=[1492            Document(1493                page_content="This is a test document.",1494                metadata={"source": "1"},1495            ),1496            Document(1497                page_content="This is another document.",1498                metadata={"source": "1"},1499            ),1500        ]1501    )15021503    with patch.object(1504        record_manager,1505        "get_time",1506        return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),1507    ):1508        assert index(1509            loader,1510            record_manager,1511            vector_store,1512            cleanup="incremental",1513            source_id_key="source",1514            key_encoder="sha256",1515        ) == {1516            "num_added": 2,1517            "num_deleted": 0,1518            "num_skipped": 0,1519            "num_updated": 0,1520        }15211522    doc_texts = {1523        # Ignoring type since doc should be in the store and not a None1524        vector_store.get_by_ids([uid])[0].page_content1525        for uid in vector_store.store1526    }1527    assert doc_texts == {"This is another document.", "This is a test document."}15281529    # Delete 1 document and unchange 1 document1530    loader = ToyLoader(1531        documents=[1532            Document(1533                page_content="This is another document.",  # <-- Same as original1534                metadata={"source": "1"},1535            ),1536        ]1537    )15381539    with patch.object(1540        record_manager,1541        "get_time",1542        return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1543    ):1544        assert index(1545            loader,1546            record_manager,1547            vector_store,1548            cleanup="incremental",1549            source_id_key="source",1550            key_encoder="sha256",1551        ) == {1552            "num_added": 0,1553            "num_deleted": 1,1554            "num_skipped": 1,1555            "num_updated": 0,1556        }15571558    doc_texts = {1559        # Ignoring type since doc should be in the store and not a None1560        vector_store.get_by_ids([uid])[0].page_content1561        for uid in vector_store.store1562    }1563    assert doc_texts == {1564        "This is another document.",1565    }156615671568def test_incremental_indexing_with_batch_size(1569    record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore1570) -> None:1571    """Test indexing with incremental indexing."""1572    loader = ToyLoader(1573        documents=[1574            Document(1575                page_content="1",1576                metadata={"source": "1"},1577            ),1578            Document(1579                page_content="2",1580                metadata={"source": "1"},1581            ),1582            Document(1583                page_content="3",1584                metadata={"source": "1"},1585            ),1586            Document(1587                page_content="4",1588                metadata={"source": "1"},1589            ),1590        ]1591    )15921593    with patch.object(1594        record_manager,1595        "get_time",1596        return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),1597    ):1598        assert index(1599            loader,1600            record_manager,1601            vector_store,1602            cleanup="incremental",1603            source_id_key="source",1604            batch_size=2,1605            key_encoder="sha256",1606        ) == {1607            "num_added": 4,1608            "num_deleted": 0,1609            "num_skipped": 0,1610            "num_updated": 0,1611        }16121613    doc_texts = {1614        # Ignoring type since doc should be in the store and not a None1615        vector_store.get_by_ids([uid])[0].page_content1616        for uid in vector_store.store1617    }1618    assert doc_texts == {"1", "2", "3", "4"}16191620    with patch.object(1621        record_manager,1622        "get_time",1623        return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1624    ):1625        assert index(1626            loader,1627            record_manager,1628            vector_store,1629            cleanup="incremental",1630            source_id_key="source",1631            batch_size=2,1632            key_encoder="sha256",1633        ) == {1634            "num_added": 2,1635            "num_deleted": 2,1636            "num_skipped": 2,1637            "num_updated": 0,1638        }16391640    doc_texts = {1641        # Ignoring type since doc should be in the store and not a None1642        vector_store.get_by_ids([uid])[0].page_content1643        for uid in vector_store.store1644    }1645    assert doc_texts == {"1", "2", "3", "4"}164616471648def test_incremental_delete_with_batch_size(1649    record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore1650) -> None:1651    """Test indexing with incremental deletion strategy and batch size."""1652    loader = ToyLoader(1653        documents=[1654            Document(1655                page_content="1",1656                metadata={"source": "1"},1657            ),1658            Document(1659                page_content="2",1660                metadata={"source": "2"},1661            ),1662            Document(1663                page_content="3",1664                metadata={"source": "3"},1665            ),1666            Document(1667                page_content="4",1668                metadata={"source": "4"},1669            ),1670        ]1671    )16721673    with patch.object(1674        record_manager,1675        "get_time",1676        return_value=datetime(2021, 1, 1, tzinfo=UTC).timestamp(),1677    ):1678        assert index(1679            loader,1680            record_manager,1681            vector_store,1682            cleanup="incremental",1683            source_id_key="source",1684            batch_size=3,1685            key_encoder="sha256",1686        ) == {1687            "num_added": 4,1688            "num_deleted": 0,1689            "num_skipped": 0,1690            "num_updated": 0,1691        }16921693    doc_texts = {1694        # Ignoring type since doc should be in the store and not a None1695        vector_store.get_by_ids([uid])[0].page_content1696        for uid in vector_store.store1697    }1698    assert doc_texts == {"1", "2", "3", "4"}16991700    # Attempt to index again verify that nothing changes1701    with patch.object(1702        record_manager,1703        "get_time",1704        return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1705    ):1706        assert index(1707            loader,1708            record_manager,1709            vector_store,1710            cleanup="incremental",1711            source_id_key="source",1712            batch_size=3,1713            key_encoder="sha256",1714        ) == {1715            "num_added": 0,1716            "num_deleted": 0,1717            "num_skipped": 4,1718            "num_updated": 0,1719        }17201721    doc_texts = {1722        # Ignoring type since doc should be in the store and not a None1723        vector_store.get_by_ids([uid])[0].page_content1724        for uid in vector_store.store1725    }1726    assert doc_texts == {"1", "2", "3", "4"}17271728    # Attempt to index again verify that nothing changes1729    with patch.object(1730        record_manager,1731        "get_time",1732        return_value=datetime(2022, 1, 3, tzinfo=UTC).timestamp(),1733    ):1734        # Docs with same content1735        docs = [1736            Document(1737                page_content="1",1738                metadata={"source": "1"},1739            ),1740            Document(1741                page_content="2",1742                metadata={"source": "2"},1743            ),1744        ]1745        assert index(1746            docs,1747            record_manager,1748            vector_store,1749            cleanup="incremental",1750            source_id_key="source",1751            batch_size=1,1752            key_encoder="sha256",1753        ) == {1754            "num_added": 0,1755            "num_deleted": 0,1756            "num_skipped": 2,1757            "num_updated": 0,1758        }17591760    doc_texts = {1761        # Ignoring type since doc should be in the store and not a None1762        vector_store.get_by_ids([uid])[0].page_content1763        for uid in vector_store.store1764    }1765    assert doc_texts == {"1", "2", "3", "4"}17661767    # Attempt to index again verify that nothing changes1768    with patch.object(1769        record_manager,1770        "get_time",1771        return_value=datetime(2023, 1, 4, tzinfo=UTC).timestamp(),1772    ):1773        # Docs with same content1774        docs = [1775            Document(1776                page_content="1",1777                metadata={"source": "1"},1778            ),1779            Document(1780                page_content="2",1781                metadata={"source": "2"},1782            ),1783        ]1784        assert index(1785            docs,1786            record_manager,1787            vector_store,1788            cleanup="incremental",1789            source_id_key="source",1790            batch_size=1,1791            key_encoder="sha256",1792        ) == {1793            "num_added": 0,1794            "num_deleted": 0,1795            "num_skipped": 2,1796            "num_updated": 0,1797        }17981799    doc_texts = {1800        # Ignoring type since doc should be in the store and not a None1801        vector_store.get_by_ids([uid])[0].page_content1802        for uid in vector_store.store1803    }1804    assert doc_texts == {"1", "2", "3", "4"}18051806    # Try to index with changed docs now1807    with patch.object(1808        record_manager,1809        "get_time",1810        return_value=datetime(2024, 1, 5, tzinfo=UTC).timestamp(),1811    ):1812        # Docs with same content1813        docs = [1814            Document(1815                page_content="changed 1",1816                metadata={"source": "1"},1817            ),1818            Document(1819                page_content="changed 2",1820                metadata={"source": "2"},1821            ),1822        ]1823        assert index(1824            docs,1825            record_manager,1826            vector_store,1827            cleanup="incremental",1828            source_id_key="source",1829            key_encoder="sha256",1830        ) == {1831            "num_added": 2,1832            "num_deleted": 2,1833            "num_skipped": 0,1834            "num_updated": 0,1835        }18361837    doc_texts = {1838        # Ignoring type since doc should be in the store and not a None1839        vector_store.get_by_ids([uid])[0].page_content1840        for uid in vector_store.store1841    }1842    assert doc_texts == {"changed 1", "changed 2", "3", "4"}184318441845async def test_aincremental_delete(1846    arecord_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore1847) -> None:1848    """Test indexing with incremental deletion strategy."""1849    loader = ToyLoader(1850        documents=[1851            Document(1852                page_content="This is a test document.",1853                metadata={"source": "1"},1854            ),1855            Document(1856                page_content="This is another document.",1857                metadata={"source": "2"},1858            ),1859        ]1860    )18611862    with patch.object(1863        arecord_manager,1864        "get_time",1865        return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1866    ):1867        assert await aindex(1868            loader.lazy_load(),1869            arecord_manager,1870            vector_store,1871            cleanup="incremental",1872            source_id_key="source",1873            key_encoder="sha256",1874        ) == {1875            "num_added": 2,1876            "num_deleted": 0,1877            "num_skipped": 0,1878            "num_updated": 0,1879        }18801881    doc_texts = {1882        # Ignoring type since doc should be in the store and not a None1883        vector_store.get_by_ids([uid])[0].page_content1884        for uid in vector_store.store1885    }1886    assert doc_texts == {"This is another document.", "This is a test document."}18871888    # Attempt to index again verify that nothing changes1889    with patch.object(1890        arecord_manager,1891        "get_time",1892        return_value=datetime(2021, 1, 2, tzinfo=UTC).timestamp(),1893    ):1894        assert await aindex(1895            loader.lazy_load(),1896            arecord_manager,1897            vector_store,1898            cleanup="incremental",1899            source_id_key="source",1900            key_encoder="sha256",1901        ) == {1902            "num_added": 0,1903            "num_deleted": 0,1904            "num_skipped": 2,1905            "num_updated": 0,1906        }19071908    # Create 2 documents from the same source all with mutated content1909    loader = ToyLoader(1910        documents=[1911            Document(1912                page_content="mutated document 1",1913                metadata={"source": "1"},1914            ),1915            Document(1916                page_content="mutated document 2",1917                metadata={"source": "1"},1918            ),1919            Document(1920                page_content="This is another document.",  # <-- Same as original1921                metadata={"source": "2"},1922            ),1923        ]1924    )19251926    # Attempt to index again verify that nothing changes1927    with patch.object(1928        arecord_manager,1929        "get_time",1930        return_value=datetime(2021, 1, 3, tzinfo=UTC).timestamp(),1931    ):1932        assert await aindex(1933            loader.lazy_load(),1934            arecord_manager,1935            vector_store,1936            cleanup="incremental",1937            source_id_key="source",1938            key_encoder="sha256",1939        ) == {1940            "num_added": 2,1941            "num_deleted": 1,1942            "num_skipped": 1,1943            "num_updated": 0,1944        }19451946    doc_texts = {1947        # Ignoring type since doc should be in the store and not a None1948        vector_store.get_by_ids([uid])[0].page_content1949        for uid in vector_store.store1950    }1951    assert doc_texts == {1952        "mutated document 1",1953        "mutated document 2",1954        "This is another document.",1955    }195619571958def test_indexing_with_no_docs(1959    record_manager: InMemoryRecordManager, vector_store: VectorStore1960) -> None:1961    """Check edge case when loader returns no new docs."""1962    loader = ToyLoader(documents=[])19631964    assert index(1965        loader,1966        record_manager,1967        vector_store,1968        cleanup="full",1969        key_encoder="sha256",1970    ) == {1971        "num_added": 0,1972        "num_deleted": 0,1973        "num_skipped": 0,1974        "num_updated": 0,1975    }197619771978async def test_aindexing_with_no_docs(1979    arecord_manager: InMemoryRecordManager, vector_store: VectorStore1980) -> None:1981    """Check edge case when loader returns no new docs."""1982    loader = ToyLoader(documents=[])19831984    assert await aindex(1985        loader,1986        arecord_manager,1987        vector_store,1988        cleanup="full",1989        key_encoder="sha256",1990    ) == {1991        "num_added": 0,1992        "num_deleted": 0,1993        "num_skipped": 0,1994        "num_updated": 0,1995    }199619971998def test_deduplication(1999    record_manager: InMemoryRecordManager, vector_store: VectorStore2000) -> None:

Code quality findings 2

Ensure functions have docstrings for documentation
info maintainability missing-docstring
def lazy_load(
Ensure functions have docstrings for documentation
info maintainability missing-docstring
async def alazy_load(

Get this view in your editor

Same data, no extra tab — call code_get_file + code_get_findings over MCP from Claude/Cursor/Copilot.

Generated by searchcode.com · powered by code_get_file + code_get_findings