1/*2 * Copyright (c) 2009-Present, Redis Ltd.3 * All rights reserved.4 * 5 * Copyright (c) 2024-present, Valkey contributors.6 * All rights reserved.7 *8 * Licensed under your choice of (a) the Redis Source Available License 2.09 * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the10 * GNU Affero General Public License v3 (AGPLv3).11 */1213#include "server.h"14#include "bio.h"15#include "redisassert.h"16#include "ebuckets.h"17#include "entry.h"18#include "vector.h"19#include <math.h>2021/* Threshold for HEXPIRE and HPERSIST to be considered whether it is worth to22 * update the expiration time of the hash object in global HFE DS. */23#define HASH_NEW_EXPIRE_DIFF_THRESHOLD 40002425/* Reserve 2 bits out of hash-field expiration time for possible future lightweight26 * indexing/categorizing of fields. It can be achieved by hacking HFE as follows:27 *28 *    HPEXPIREAT key [ 2^47 + USER_INDEX ] FIELDS numfields field [field …]29 *30 * Redis will also need to expose kind of HEXPIRESCAN and HEXPIRECOUNT for this31 * idea. Yet to be better defined.32 *33 * HFE_MAX_ABS_TIME_MSEC constraint must be enforced only at API level. Internally,34 * the expiration time can be up to EB_EXPIRE_TIME_MAX for future readiness.35 */36#define HFE_MAX_ABS_TIME_MSEC (EB_EXPIRE_TIME_MAX >> 2)3738typedef enum GetFieldRes {39    /* common (Used by hashTypeGet* value family) */40    GETF_OK = 0,            /* The field was found. */41    GETF_NOT_FOUND,         /* The field was not found. */42    GETF_EXPIRED,           /* Logically expired (Might be lazy deleted or not) */43    GETF_EXPIRED_HASH,      /* Delete hash since retrieved field was expired and44                             * it was the last field in the hash. */45} GetFieldRes;4647typedef listpackEntry CommonEntry; /* extend usage beyond lp */4849#define FIELDS_STACK_SIZE 645051/* A vec with an embedded stack buffer, used to collect field robj pointers52 * for subkey notifications without heap allocation in the common case. */53typedef struct fieldvec { vec v; void *buf[FIELDS_STACK_SIZE]; } fieldvec;5455static inline vec *fieldvecInit(fieldvec *fv, size_t cap) {56    vecInit(&fv->v, fv->buf, FIELDS_STACK_SIZE);57    vecReserve(&fv->v, cap);58    return &fv->v;59}6061/* hash field expiration (HFE) funcs */62static ExpireAction onFieldExpire(eItem item, void *ctx);63static ExpireMeta* hentryGetExpireMeta(const eItem field);64static void hexpireGenericCommand(client *c, long long basetime, int unit);65static void hfieldPersist(robj *hashObj, Entry *entry);66static void propagateHashFieldDeletion(redisDb *db, sds key, char *field, size_t fieldLen);6768/* hash dictType funcs */69static void dictEntryDestructor(dict *d, void *entry);70static size_t hashDictMetadataBytes(dict *d);71static size_t hashDictWithExpireMetadataBytes(dict *d);72static void hashDictWithExpireOnRelease(dict *d);73static kvobj* hashTypeLookupWriteOrCreate(client *c, robj *key);74static int hashTypeCanConvertTmplToListpack(robj *o);75static void hashTypeConvertTmplToListpackOrHT(robj *o, int lp_enc, int with_hfe);7677/*-----------------------------------------------------------------------------78 * Define dictType of hash79 *80 * - Stores fields as entry objects (field-value pairs) with optional expiration81 * - Note that small hashes are represented with listpacks82 * - Once expiration is set for a field, the dict instance and corresponding83 *   dictType are replaced with a dict containing metadata for Hash Field84 *   Expiration (HFE) and using dictType `entryHashDictTypeWithHFE`85 * - Dict uses no_value=1 since entry contains both field and value86 *----------------------------------------------------------------------------*/87dictType entryHashDictType = {88    dictSdsHash,                                /* lookup hash function */89    NULL,                                       /* key dup */90    NULL,                                       /* val dup */91    dictSdsKeyCompare,                          /* lookup key compare */92    dictEntryDestructor,                       /* key destructor */93    NULL,                                       /* val destructor (value is in entry) */94    .dictMetadataBytes = hashDictMetadataBytes,95    .no_value = 1,                              /* entry contains both field and value */96    .keys_are_odd = 1,                          /* entry pointers (SDS) are always odd */97};9899/* Define alternative dictType of hash with hash-field expiration (HFE) support */100dictType entryHashDictTypeWithHFE = {101    dictSdsHash,                                /* lookup hash function */102    NULL,                                       /* key dup */103    NULL,                                       /* val dup */104    dictSdsKeyCompare,                          /* lookup key compare */105    dictEntryDestructor,                       /* key destructor */106    NULL,                                       /* val destructor (value is in entry) */107    .dictMetadataBytes = hashDictWithExpireMetadataBytes,108    .onDictRelease = hashDictWithExpireOnRelease,109    .no_value = 1,                              /* entry contains both field and value */110    .keys_are_odd = 1,                          /* entry pointers (SDS) are always odd */111};112113/*-----------------------------------------------------------------------------114 * Hash Field Expiration (HFE) Feature115 *116 * Each hash instance maintains its own set of hash field expiration within its117 * private ebuckets DS. In order to support HFE active expire cycle across hash118 * instances, hashes with associated HFE will be also registered in a global119 * ebuckets DS with expiration time value that reflects their next minimum120 * time to expire (db->subexpires). The global HFE Active expiration will be121 * triggered from activeExpireCycle() function and in turn will invoke "local"122 * HFE Active sub-expiration for each hash instance that has expired fields.123 *----------------------------------------------------------------------------*/124EbucketsType subexpiresBucketsType = {125    .onDeleteItem = NULL,126    .getExpireMeta = hashGetExpireMeta,   /* get ExpireMeta attached to each hash */127    .itemsAddrAreOdd = 0,                 /* Addresses of dict are even */128};129130/* htExpireMetadata - ebuckets-type for hash fields with time-Expiration. ebuckets131 * instance Will be attached to each hash that has at least one field with expiry132 * time. */133EbucketsType hashFieldExpireBucketsType = {134    .onDeleteItem = NULL,135    .getExpireMeta = hentryGetExpireMeta, /* get ExpireMeta attached to each field */136    .itemsAddrAreOdd = 1,                 /* Addresses of hfield (entry/sds) are odd!! */137};138139/* OnFieldExpireCtx passed to OnFieldExpire() */140typedef struct OnFieldExpireCtx {141    robj *hashObj;142    redisDb *db;143    int activeEx; /* 1 for active expire, 0 for lazy expire */144    vec *vexpired; /* Expired fields vector */145} OnFieldExpireCtx;146147/* The implementation of hashes by dict was modified from storing fields as sds148 * strings to store "entry" objects (field-value pairs with optional expiration).149 * The entry structure unifies field and value into a single allocation, with150 * optional expiration metadata. This is simpler than the previous mstr approach151 * and provides better memory locality.152 */153154/* Used by hpersistCommand() */155typedef enum SetPersistRes {156    HFE_PERSIST_NO_FIELD =     -2,   /* No such hash-field */157    HFE_PERSIST_NO_TTL =       -1,   /* No TTL attached to the field */158    HFE_PERSIST_OK =            1159} SetPersistRes;160161static inline int isDictWithMetaHFE(dict *d) {162    return d->type == &entryHashDictTypeWithHFE;163}164165/*-----------------------------------------------------------------------------166 * setex* - Set field's expiration167 *168 * Setting expiration time to fields might be time-consuming and complex since169 * each update of expiration time, not only updates `ebuckets` of corresponding170 * hash, but also might update `ebuckets` of global HFE DS. It is required to opt171 * sequence of field updates with expirartion for a given hash, such that only172 * once done, the global HFE DS will get updated.173 *174 * To do so, follow the scheme:175 * 1. Call hashTypeSetExInit() to initialize the HashTypeSetEx struct.176 * 2. Call hashTypeSetEx() one time or more, for each field/expiration update.177 * 3. Call hashTypeSetExDone() for notification and update of global HFE.178 *----------------------------------------------------------------------------*/179180/* Returned value of hashTypeSetEx() */181typedef enum SetExRes {182    HSETEX_OK =                1,   /* Expiration time set/updated as expected */183    HSETEX_NO_FIELD =         -2,   /* No such hash-field */184    HSETEX_NO_CONDITION_MET =  0,   /* Specified NX | XX | GT | LT condition not met */185    HSETEX_DELETED =           2,   /* Field deleted because the specified time is in the past */186} SetExRes;187188/* Used by httlGenericCommand() */189typedef enum GetExpireTimeRes {190    HFE_GET_NO_FIELD =          -2, /* No such hash-field */191    HFE_GET_NO_TTL =            -1, /* No TTL attached to the field */192} GetExpireTimeRes;193194typedef enum ExpireSetCond {195    HFE_NX = 1<<0,196    HFE_XX = 1<<1,197    HFE_GT = 1<<2,198    HFE_LT = 1<<3199} ExpireSetCond;200201/* Used by hashTypeSetEx() for setting fields or their expiry  */202typedef struct HashTypeSetEx {203204    /*** config ***/205    ExpireSetCond expireSetCond;        /* [XX | NX | GT | LT] */206207    /*** metadata ***/208    uint64_t minExpire;                 /* if uninit EB_EXPIRE_TIME_INVALID */209    redisDb *db;210    robj *key, *hashObj;211    uint64_t minExpireFields;           /* Trace updated fields and their previous/new212                                         * minimum expiration time. If minimum recorded213                                         * is above minExpire of the hash, then we don't214                                         * have to update global HFE DS */215216    /* Optionally provide client for notification */217    client *c;218    const char *cmd;219} HashTypeSetEx;220221int hashTypeSetExInit(robj *key, kvobj *kvo, client *c, redisDb *db,222                      ExpireSetCond expireSetCond, HashTypeSetEx *ex);223224SetExRes hashTypeSetEx(robj *o, sds field, uint64_t expireAt, HashTypeSetEx *exInfo);225226void hashTypeSetExDone(HashTypeSetEx *e);227228/*-----------------------------------------------------------------------------229 * Accessor functions for dictType of hash230 *----------------------------------------------------------------------------*/231232static void dictEntryDestructor(dict *d, void *entry) {233    size_t usable;234    size_t *alloc_size = htGetMetadataSize(d);235236    /* If attached TTL to the field, then remove it from hash's private ebuckets. */237    if (entryGetExpiry(entry) != EB_EXPIRE_TIME_INVALID) {238        htMetadataEx *dictExpireMeta = htGetMetadataEx(d);239        ebRemove(&dictExpireMeta->hfe, &hashFieldExpireBucketsType, entry);240    }241242    entryFree(entry, &usable);243    *alloc_size -= usable;244245    /* Don't have to update global HFE DS. It's unnecessary. Implementing this246     * would introduce significant complexity and overhead for an operation that247     * isn't critical. In the worst case scenario, the hash will be efficiently248     * updated later by an active-expire operation, or it will be removed by the249     * hash's dbGenericDelete() function. */250}251252static size_t hashDictMetadataBytes(dict *d) {253    UNUSED(d);254    return sizeof(size_t);255}256257static size_t hashDictWithExpireMetadataBytes(dict *d) {258    UNUSED(d);259    /* expireMeta of the hash, ref to ebuckets and pointer to hash's key */260    return sizeof(htMetadataEx);261}262263static void hashDictWithExpireOnRelease(dict *d) {264    /* for sure allocated with metadata. Otherwise, this func won't be registered */265    htMetadataEx *dictExpireMeta = htGetMetadataEx(d);266    ebDestroy(&dictExpireMeta->hfe, &hashFieldExpireBucketsType, NULL);267}268269/*-----------------------------------------------------------------------------270 * Hash template registry internals271 *272 * Many hashes often share the exact same set of field names (e.g. rows of one273 * "schema"). Instead of repeating the field names in every key, a single274 * immutable template stores the names once and every matching hash references275 * it, keeping only its own values. This is a memory optimization aimed at large276 * populations of hashes with a stable, identical field set.277 *278 * A template is identified by its exact set of field names and is never279 * changed once created. So adding a field (HSET) or removing one (HDEL) does280 * not edit the template: the hash just switches to the template for the new281 * field set, reusing it if it already exists or creating it otherwise (see282 * hashTemplateGetOrCreateWithHash). This makes changing the field set more283 * costly than on a plain hash, so the feature is meant for many hashes that284 * keep the same fields mostly stable.285 *286 * Fields are kept sorted inside the template by sdscmplen (length, then bytes;287 * see hashTemplateValidateFields). A field lookup binary-searches the sorted288 * names for its index (hashTemplateFieldIndex), then reads the value stored at289 * that same index.290 *291 * Two value encodings reference a template:292 *   OBJ_ENCODING_TMPL_LP    - o->ptr is a listpack laid out as293 *                             [template_id (varint)][v0][v1]...[vN-1]. Values294 *                             are packed inline and the leading id maps back to295 *                             the template. Compact, used for small hashes.296 *                             Lookup: lpSeek(lp, idx + 1) (the +1 skips the id).297 *   OBJ_ENCODING_TMPL_ARRAY - o->ptr is a hashTemplateArray { tmpl_id; sds298 *                             values[] } holding the values in template field299 *                             order. Used once listpack limits are exceeded (or300 *                             there are many fields). Lookup: values[idx].301 *302 * Registry: htemplates->by_fields maps a sorted field set to its template, and303 * htemplates->by_id maps a small integer id to the template. TMPL_LP stores304 * this id inline rather than an 8-byte pointer, so a key's reference to its305 * template can be as little as ~2 bytes. RDB save uses it the same way: each306 * template is written once and every key just references it by this id.307 *308 * Lifetime and threading: each referencing key holds a key_refcount to its309 * template and a client can also hold a hold_refcount (e.g. a fieldset310 * registered with HIMPORT PREPARE). A single template may back a huge number of311 * keys and those keys are often freed in bulk in the background by a BIO312 * lazyfree thread (e.g. background trim after ASM). The registry is main thread313 * only, so a BIO thread must not touch it: instead the BIO lazyfree thread records314 * per-template key drop counts and publishes them to the main thread, which 315 * applies the drops and frees any template reaching zero refs in cron (see bio_pending_drops).316 *----------------------------------------------------------------------------*/317318#define HASH_TMPL_STACK_ENTRIES 128319320/* Global template registry; file-local alias of server.htemplates. */321static hashTemplateRegistry *htemplates = NULL;322323/* Forward declarations for template helpers. */324static hashTemplate *hashTemplateGetOrCreateWithHash(uint64_t hash, sds *fields, unsigned long long field_count);325static void hashTemplateDecrKeyRef(hashTemplate *tmpl);326static hashTemplate *hashTemplateLpGetTemplate(unsigned char *lp);327static uint64_t hashTemplateLpGetTemplateId(unsigned char *lp);328static unsigned char *hashTemplateLpCreate(hashTemplate *tmpl, sds *values);329static hashTemplateArray *hashTemplateArrayCreate(hashTemplate *tmpl, sds *values, int take);330static hashTemplate *hashTemplateArrayGetTemplate(hashTemplateArray *hta);331332/* Keys freed in BIO lazyfree thread sets a dict (template id->key_ref_count). 333 * Later, it is applied on main thread in cron, key refs are actually dropped. */334static redisAtomic uintptr_t bio_pending_drops = 0;335336static uint64_t pendingDropHash(const void *key) { return (uint64_t)(uintptr_t)key; }337static dictType pendingDropDictType = { pendingDropHash, NULL, NULL, NULL, NULL, NULL, NULL };338339/* BIO thread: Increment deleted key count for the template id. */340static void hashTemplateRecordPendingDrop(uint64_t id) {341    serverAssert(bioIsLazyfreeWorker());342343    uintptr_t cur;344    atomicExchangeAcquire(bio_pending_drops, 0, cur);345346    dict *d = cur ? (dict *)cur : dictCreate(&pendingDropDictType);347    dictEntry *e;348    dictEntry *de = dictAddRaw(d, (void *)(uintptr_t)id, &e);349    350    /* Increment deleted key count for the template id */351    if (de) 352        dictSetUnsignedIntegerVal(de, 1);353    else    354        dictSetUnsignedIntegerVal(e, dictGetUnsignedIntegerVal(e) + 1);355    atomicSetRelease(bio_pending_drops, (uintptr_t)d);356}357358/* Sum of the 'n' sds lengths, or -1 if any exceeds hash-max-listpack-value. */359static ssize_t hashTypeSdsArrayLpBytes(sds *arr, unsigned long long n) {360    size_t sum = 0;361    for (unsigned long long i = 0; i < n; i++) {362        size_t len = sdslen(arr[i]);363        if (len > server.hash_max_listpack_value) return -1;364        sum += len;365    }366    return (ssize_t)sum;367}368369/* Do 'n' sds strings fit a listpack? */370static int hashTypeSdsArrayFitsLp(sds *arr, unsigned long long n) {371    if (n > server.hash_max_listpack_entries) return 0;372    ssize_t sum = hashTypeSdsArrayLpBytes(arr, n);373    return sum >= 0 && lpSafeToAdd(NULL, sum);374}375376/* True if the hash uses one of the template encodings. */377static inline int hashTypeIsTemplate(const robj *o) {378    return o->encoding == OBJ_ENCODING_TMPL_LP ||379           o->encoding == OBJ_ENCODING_TMPL_ARRAY;380}381382/* by_id is a chunked array: fixed-size chunks of template pointers. */383#define TMPL_CHUNK_SIZE 128  /* template ids per chunk */384385typedef struct tmplIdChunk {386    hashTemplate *slots[TMPL_CHUNK_SIZE]; /* id % TMPL_CHUNK -> template, NULL if free */387    unsigned int used;               /* used slots; chunk freed when 0 */388} tmplIdChunk;389390/* Get chunk holding 'id', allocating it on demand. */391static tmplIdChunk *tmplIdGetOrCreateChunk(size_t id) {392    size_t chunk_idx = id / TMPL_CHUNK_SIZE;393    if (chunk_idx >= htemplates->by_id_cap) {394        size_t old = htemplates->by_id_cap;395        size_t ncap = old ? old * 2 : 8;396        while (ncap <= chunk_idx) ncap *= 2;397398        htemplates->by_id = zrealloc(htemplates->by_id, ncap * sizeof(tmplIdChunk *));399        memset(htemplates->by_id + old, 0, (ncap - old) * sizeof(tmplIdChunk *));400        htemplates->by_id_cap = ncap;401    }402    if (htemplates->by_id[chunk_idx] == NULL) {403        htemplates->by_id[chunk_idx] = zcalloc(sizeof(tmplIdChunk));404        htemplates->by_id_chunks++;405    }406    return htemplates->by_id[chunk_idx];407}408409/* Get lowest free id. Caller guarantees a gap exists. Chunks below410 * by_id_free_chunk_hint are full, the scan starts there. */411static size_t tmplIdGetLowestFree(void) {412    size_t chunk_idx = htemplates->by_id_free_chunk_hint;413#ifdef DEBUG_ASSERTIONS414    /* Chunks below the hint must be full. */415    for (size_t i = 0; i < chunk_idx; i++)416        serverAssert(htemplates->by_id[i] && htemplates->by_id[i]->used == TMPL_CHUNK_SIZE);417#endif418    while (chunk_idx < htemplates->by_id_cap && htemplates->by_id[chunk_idx] &&419           htemplates->by_id[chunk_idx]->used == TMPL_CHUNK_SIZE) {420        chunk_idx++;421    }422    htemplates->by_id_free_chunk_hint = chunk_idx;423    tmplIdChunk *chunk = chunk_idx < htemplates->by_id_cap ? htemplates->by_id[chunk_idx] : NULL;424    size_t id = chunk_idx * TMPL_CHUNK_SIZE;425    while (chunk && chunk->slots[id % TMPL_CHUNK_SIZE] != NULL) id++;426    return id;427}428429/* Store tmpl in by_id under the lowest free id and return that id (the caller430 * sets tmpl->id). Low ids keep the TMPL_LP listpack id ~2 bytes per key. */431static uint64_t tmplIdAllocate(hashTemplate *tmpl) {432    int no_gaps = dictSize(htemplates->by_fields) == htemplates->by_id_next;433    size_t id = no_gaps ? htemplates->by_id_next++ : tmplIdGetLowestFree();434    /* Every lower id is in use, the hint can move up. */435    if (no_gaps) htemplates->by_id_free_chunk_hint = id / TMPL_CHUNK_SIZE;436    tmplIdChunk *chunk = tmplIdGetOrCreateChunk(id);437    chunk->slots[id % TMPL_CHUNK_SIZE] = tmpl;438    chunk->used++;439    return id;440}441442/* Recycle a template ID when template is freed. */443static void tmplIdRecycle(uint64_t id) {444    size_t chunk_idx = id / TMPL_CHUNK_SIZE;445    tmplIdChunk *chunk = htemplates->by_id[chunk_idx];446    chunk->slots[id % TMPL_CHUNK_SIZE] = NULL;447    if (htemplates->by_id_free_chunk_hint > chunk_idx)448        htemplates->by_id_free_chunk_hint = chunk_idx;449    /* Free the chunk once it holds no live ids so the id space shrinks. */450    if (--chunk->used == 0) {451        zfree(chunk);452        htemplates->by_id[chunk_idx] = NULL;453        htemplates->by_id_chunks--;454    }455456    /* This runs from the registry's key destructor, dictSize()==1 means it is457     * the last one and the whole by_id array can be released. */458    if (dictSize(htemplates->by_fields) == 1) {459        for (size_t i = 0; i < htemplates->by_id_cap; i++)460            zfree(htemplates->by_id[i]);461        zfree(htemplates->by_id);462        htemplates->by_id = NULL;463        htemplates->by_id_cap = 0;464        htemplates->by_id_chunks = 0;465        htemplates->by_id_next = 0;466        htemplates->by_id_free_chunk_hint = 0;467    }468}469470/* Lookup template by ID. Returns NULL if invalid. */471hashTemplate *hashTemplateGetById(uint64_t id) {472    size_t chunk_idx = id / TMPL_CHUNK_SIZE;473    if (chunk_idx >= htemplates->by_id_cap) return NULL;474    tmplIdChunk *chunk = htemplates->by_id[chunk_idx];475    return chunk ? chunk->slots[id % TMPL_CHUNK_SIZE] : NULL;476}477478/* Defrag the template struct and re-point every reference479 * to it (by_id slot, by_fields key, by_fields_lp value).*/480void hashTemplateDefrag(hashTemplate *tmpl, dictEntry *bf, monotime endtime) {481    /* Field-name array. */482    sds *newfields = activeDefragAlloc(tmpl->fields);483    if (newfields) tmpl->fields = newfields;484485    /* Field names, checking the clock every 128 so a wide template cannot486     * blow the time budget. On timeout, defrag_field keeps the position, the487     * leftover is picked up when the template is scanned again. */488    unsigned long long i = tmpl->defrag_field;489    long iterations = 0;490    while (i < tmpl->field_count) {491        sds newsds = activeDefragSds(tmpl->fields[i]);492        if (newsds) tmpl->fields[i] = newsds;493        server.stat_active_defrag_scanned++;494        i++;495        if (++iterations > 128) {496            if (getMonotonicUs() > endtime) break;497            iterations = 0;498        }499    }500    tmpl->defrag_field = (i >= tmpl->field_count) ? 0 : i;501502    /* fields_lp blob (the by_fields_lp key). */503    dictEntry *lp = NULL;504    if (tmpl->fields_lp) {505        unsigned char *oldlp = tmpl->fields_lp;506        unsigned char *newlp = activeDefragAlloc(oldlp);507        if (newlp) {508            /* oldlp is freed, find its entry by pointer. */509            uint64_t hash = dictGetHash(htemplates->by_fields_lp, newlp);510            lp = dictFindByHashAndPtr(htemplates->by_fields_lp, oldlp, hash);511            serverAssert(lp);512            tmpl->fields_lp = newlp;513            dictSetKey(htemplates->by_fields_lp, lp, newlp);514        }515    }516517    /* The struct itself: by_id slot, by_fields key, by_fields_lp value. */518    uint64_t id = tmpl->id;519    hashTemplate *newtmpl = activeDefragAlloc(tmpl);520    if (!newtmpl) return;521522    tmplIdChunk *chunk = htemplates->by_id[id / TMPL_CHUNK_SIZE];523    chunk->slots[id % TMPL_CHUNK_SIZE] = newtmpl;524525    dictSetKey(htemplates->by_fields, bf, newtmpl);526    if (newtmpl->fields_lp) {527        if (!lp) lp = dictFind(htemplates->by_fields_lp, newtmpl->fields_lp);528        serverAssert(lp);529        dictSetVal(htemplates->by_fields_lp, lp, newtmpl);530    }531}532533/* Defrag by_id top array (once, at idx 0) and the chunk at 'idx'. */534int hashTemplateDefragByIdChunk(unsigned long chunk_idx) {535    if (htemplates->by_id == NULL || chunk_idx >= htemplates->by_id_cap) 536        return 0;537    if (chunk_idx == 0) {538        tmplIdChunk **newarr = activeDefragAlloc(htemplates->by_id);539        if (newarr) htemplates->by_id = newarr;540    }541    tmplIdChunk *chunk = htemplates->by_id[chunk_idx];542    if (chunk) {543        tmplIdChunk *newchunk = activeDefragAlloc(chunk);544        if (newchunk) htemplates->by_id[chunk_idx] = newchunk;545    }546    return 1;547}548549/* HIMPORT SET propagation usually sends RESTORE commands with the same few550 * field sets. ASM may send RESTORE commands with thousands of different field551 * sets because keys arrive in arbitrary order. The cache holds at most one blob552 * per template, so its size is naturally bounded by the template count. The553 * explicit cap only protects against misuse that creates an excessive number554 * of templates. */555#define HASH_TMPL_FIELDS_LP_CACHE_MAX_BYTES (16 * 1024 * 1024)556557/* Cache 'fields_lp' as tmpl's fields-listpack (blob -> template), taking558 * ownership on success. Returns 0 without consuming fields_lp if tmpl is already559 * indexed or the cache is full. */560int hashTemplateIndexFieldsLp(hashTemplate *tmpl, unsigned char *fields_lp) {561    if (tmpl->fields_lp) return 0;562563    size_t cached_bytes = htemplates->fields_lp_cache_bytes;564    serverAssert(cached_bytes <= HASH_TMPL_FIELDS_LP_CACHE_MAX_BYTES);565566    size_t bytes = lpBytes(fields_lp);567    if (bytes > HASH_TMPL_FIELDS_LP_CACHE_MAX_BYTES - cached_bytes)568        return 0;569570    tmpl->fields_lp = fields_lp;571    tmpl->fields_lp_last_used = server.mstime;572    serverAssert(dictAdd(htemplates->by_fields_lp, fields_lp, tmpl) == DICT_OK);573    htemplates->fields_lp_cache_bytes += bytes;574    htemplates->total_mem_size += bytes;575    return 1;576}577578/* Build a fresh fields listpack (caller owns it). */579static unsigned char *hashTemplateBuildFieldsLp(hashTemplate *tmpl) {580    listpackEntry stack_ent[HASH_TMPL_STACK_ENTRIES];581    listpackEntry *ent = (tmpl->field_count <= HASH_TMPL_STACK_ENTRIES) ?582                         stack_ent : zmalloc(sizeof(listpackEntry) * tmpl->field_count);583    for (unsigned long long i = 0; i < tmpl->field_count; i++) {584        ent[i].sval = (unsigned char *)tmpl->fields[i];585        ent[i].slen = sdslen(tmpl->fields[i]);586    }587    unsigned char *lp = lpNewWithEntries(ent, tmpl->field_count);588    if (ent != stack_ent) zfree(ent);589    return lp;590}591592/* Return tmpl's fields listpack, building it on first use. '*cache' requests593 * indexing the blob for RESTORE lookup and is cleared if the cache is full,594 * in which case the caller owns the returned blob. */595unsigned char *hashTemplateGetFieldsLp(hashTemplate *tmpl, int *cache) {596    if (!*cache) return hashTemplateBuildFieldsLp(tmpl);597    tmpl->fields_lp_last_used = server.mstime;598    if (tmpl->fields_lp) return tmpl->fields_lp;599    unsigned char *lp = hashTemplateBuildFieldsLp(tmpl);600    if (!hashTemplateIndexFieldsLp(tmpl, lp)) *cache = 0;601    return lp;602}603604/* Lookup a template by its fields listpack. Returns NULL if not indexed. */605hashTemplate *hashTemplateGetByFieldsLp(unsigned char *fields_lp) {606    dictEntry *de = dictFind(htemplates->by_fields_lp, fields_lp);607    if (!de) return NULL;608    hashTemplate *tmpl = dictGetVal(de);609    tmpl->fields_lp_last_used = server.mstime;610    return tmpl;611}612613/* Compute SipHash for a single field. */614static uint64_t computeFieldHash(sds field) {615    return dictGenHashFunction(field, sdslen(field));616}617618/* Compute commutative hash for a set of fields.619 * Uses sum of per-field SipHashes so fields can be620 * incrementally added/removed:621 *   add:    hash += computeFieldHash(new_field)622 *   remove: hash -= computeFieldHash(removed_field) */623static uint64_t computeFieldsHash(sds *fields, unsigned long long field_count) {624    uint64_t hash = 0;625    for (unsigned long long i = 0; i < field_count; i++)626        hash += computeFieldHash(fields[i]);627    return hash;628}629630/* Template registry dict callbacks. Key is hashTemplate*, value is same. */631static uint64_t templateFieldsHashFunc(const void *key) {632    const hashTemplate *tmpl = key;633    return tmpl->hash;634}635636static int templateFieldsKeyCompare(dictCmpCache *cache, 637                                    const void *k1,638                                    const void *k2)639{640    UNUSED(cache);641    const hashTemplate *t1 = k1;642    const hashTemplate *t2 = k2;643644    if (t1->hash != t2->hash) return 0;645    if (t1->field_count != t2->field_count) return 0;646647    for (unsigned long long i = 0; i < t1->field_count; i++) {648        if (sdscmplen(t1->fields[i], t2->fields[i]) != 0)649            return 0;650    }651    return 1;652}653654/* Drop tmpl's cached fields listpack. */655static void hashTemplateDropFieldsLp(hashTemplate *tmpl) {656    if (tmpl->fields_lp == NULL) return;657658    unsigned char *lp = tmpl->fields_lp;659    size_t bytes = lpBytes(lp);660    serverAssert(dictDelete(htemplates->by_fields_lp, lp) == DICT_OK);661    serverAssert(htemplates->fields_lp_cache_bytes >= bytes);662    htemplates->fields_lp_cache_bytes -= bytes;663    htemplates->total_mem_size -= bytes;664    lpFree(lp);665    tmpl->fields_lp = NULL;666}667668static void templateFieldsKeyDestructor(dict *d, void *key) {669    UNUSED(d);670    hashTemplate *tmpl = key;671    tmplIdRecycle(tmpl->id);672    hashTemplateDropFieldsLp(tmpl);673    for (unsigned long long i = 0; i < tmpl->field_count; i++)674        sdsfree(tmpl->fields[i]);675    zfree(tmpl->fields);676    htemplates->total_mem_size -= tmpl->mem_size;677    zfree(tmpl);678}679680static dictType templateFieldsDictType = {681    templateFieldsHashFunc,        /* hash function */682    NULL,                          /* key dup */683    NULL,                          /* val dup */684    templateFieldsKeyCompare,      /* key compare */685    templateFieldsKeyDestructor,   /* key destructor (key=tmpl) */686    NULL,                          /* val destructor (val=same as key) */687    NULL                           /* allow to expand */688};689690/* by_fields_lp dict: key is the fields listpack, value is the hashTemplate* */691static uint64_t templateFieldsLpHashFunc(const void *key) {692    return dictGenHashFunction(key, lpBytes((unsigned char *)key));693}694695static int templateFieldsLpKeyCompare(dictCmpCache *cache,696                                      const void *k1, const void *k2)697{698    UNUSED(cache);699    size_t l1 = lpBytes((unsigned char *)k1), l2 = lpBytes((unsigned char *)k2);700    return (l1 == l2) && (memcmp(k1, k2, l1) == 0);701}702703static dictType templateFieldsLpDictType = {704    templateFieldsLpHashFunc,      /* hash function */705    NULL,                          /* key dup */706    NULL,                          /* val dup */707    templateFieldsLpKeyCompare,    /* key compare */708    NULL,                          /* key destructor (blob owned by template) */709    NULL,                          /* val destructor */710    NULL                           /* allow to expand */711};712713/* Initialize hash templates registry. */714void hashTemplatesInit(void) {715    if (htemplates) return;716    htemplates = zcalloc(sizeof(hashTemplateRegistry));717    htemplates->by_fields = dictCreate(&templateFieldsDictType);718    htemplates->by_fields_lp = dictCreate(&templateFieldsLpDictType);719    server.htemplates = htemplates;720}721722/* Create a new hash tmpl, fields must be pre-sorted. */723static hashTemplate *hashTemplateCreateInternal(uint64_t hash, sds *fields,724                                                unsigned long long field_count) 725{726    /* Fields must be strictly ascending; catch unsorted/dup sets in test builds. */727    debugServerAssert(hashTemplateValidateFields(fields, field_count));728729    hashTemplate *tmpl = zmalloc(sizeof(*tmpl));730    tmpl->hash = hash;731    tmpl->hold_refcount = 0;732    tmpl->key_refcount = 0;733    tmpl->field_count = field_count;734    tmpl->fields = zmalloc(sizeof(sds) * field_count);735    tmpl->fields_lp = NULL; /* Lazy built on first save/lookup due to RESTORE. */736    tmpl->fields_lp_last_used = 0;737    tmpl->defrag_field = 0;738    tmpl->mem_size = sizeof(*tmpl) + sizeof(sds) * field_count;739740    for (unsigned long long i = 0; i < field_count; i++) {741        tmpl->fields[i] = sdsdup(fields[i]);742        tmpl->mem_size += sdsZmallocSize(tmpl->fields[i]);743    }744    htemplates->total_mem_size += tmpl->mem_size;745746    /* Whether DUMP can serialize the field names as one listpack blob. */747    tmpl->fits_in_listpack = hashTypeSdsArrayFitsLp(tmpl->fields, tmpl->field_count);748    tmpl->id = tmplIdAllocate(tmpl);749    return tmpl;750}751752/* Get or create a template (computes the fields-hash). fields must be pre-sorted. */753hashTemplate *hashTemplateGetOrCreate(sds *fields, unsigned long long field_count) {754    return hashTemplateGetOrCreateWithHash(755        computeFieldsHash(fields, field_count), fields, field_count);756}757758/* Get or create a template, reusing a fields-hash the caller already has. The759 * fields-hash is commutative (a sum of per-field hashes), so an HSET/HDEL that760 * adds or removes one field can update it in O(1) (hash +/- that field's hash)761 * and pass it here instead of rescanning every field. fields must be pre-sorted. */762static hashTemplate *hashTemplateGetOrCreateWithHash(uint64_t hash, sds *fields,763                                              unsigned long long field_count) {764    /* Fields must be strictly ascending; catch unsorted/dup sets in test builds. */765    debugServerAssert(hashTemplateValidateFields(fields, field_count));766767    hashTemplate query = {768        .hash = hash,769        .field_count = field_count,770        .fields = fields771    };772773    dictEntry *de = dictFind(htemplates->by_fields, &query);774    if (de) return dictGetKey(de);775776    hashTemplate *tmpl = hashTemplateCreateInternal(hash, fields, field_count);777    dictAdd(htemplates->by_fields, tmpl, NULL);778    return tmpl;779}780781/* Registry lookup without creating on miss. fields must be pre-sorted. */782static hashTemplate *hashTemplateFindByFields(uint64_t hash, sds *fields,783                                              unsigned long long field_count) {784    debugServerAssert(hashTemplateValidateFields(fields, field_count));785786    hashTemplate query = { .hash = hash, .field_count = field_count, .fields = fields };787    dictEntry *de = dictFind(htemplates->by_fields, &query);788    return de ? dictGetKey(de) : NULL;789}790791/* Bump key_refcount when a hash key starts using tmpl. */792void hashTemplateIncrKeyRef(hashTemplate *tmpl) {793    tmpl->key_refcount++;794    htemplates->total_key_refs++;795}796797/* Bump hold_refcount for a non-key holder (a client's prepared fieldset from798 * HIMPORT PREPARE, or an in-progress RDB load). */799void hashTemplateIncrHoldRef(hashTemplate *tmpl) {800    tmpl->hold_refcount++;801}802803/* Free the template when no key and no holder reference it. */804static void hashTemplateFreeIfUnreferenced(hashTemplate *tmpl) {805    if (tmpl->key_refcount == 0 && tmpl->hold_refcount == 0)806        dictDelete(htemplates->by_fields, tmpl);807}808809/* Drop one holdref (a client's prepared fieldset, or an in-progress RDB load).810 * When the last holder leaves, free tmpl if no keys reference it either. */811void hashTemplateDecrHoldRef(hashTemplate *tmpl) {812    serverAssert(tmpl->hold_refcount > 0);813    tmpl->hold_refcount--;814    if (tmpl->hold_refcount > 0) return;815816    hashTemplateFreeIfUnreferenced(tmpl);817}818819/* True if the calling thread is the one allowed to mutate the registry: the820 * main thread or a thread holding the module GIL. Any other thread (a BIO821 * lazyfree job) must defer the drop using hashTemplateRecordPendingDrop() */822static inline int canThreadWriteRegistry(void) {823    return pthread_equal(pthread_self(), server.main_thread_id) || moduleThreadHoldsGIL();824}825826/* Decrement key_refcount when freeing a hash key. */827static void hashTemplateDecrKeyRef(hashTemplate *tmpl) {828    serverAssert(canThreadWriteRegistry());829    serverAssert(tmpl);830    serverAssert(tmpl->key_refcount > 0);831    htemplates->total_key_refs--;832    if (--tmpl->key_refcount == 0) hashTemplateFreeIfUnreferenced(tmpl);833}834835/* Apply the key ref drops recorded by the BIO lazyfree thread. Processes836 * incrementally with a time limit to avoid blocking the main thread when837 * draining large batches (e.g. after FLUSHALL). */838static void hashTemplateApplyPendingDrops(void) {839    static dict *batch = NULL;840    static dictIterator it;841842    /* Acquire new batch if not already draining. */843    if (batch == NULL) {844        uintptr_t cur;845        atomicExchangeAcquire(bio_pending_drops, 0, cur);846        if (cur == 0) return;847        batch = (dict *)cur;848        dictInitSafeIterator(&it, batch);849    }850851    /* Time limit: spread work across multiple cron cycles to avoid spikes. */852    long long start = ustime();853    long long timelimit = 1000000 / server.hz / 20;  /* 5% of a hz cycle */854    if (timelimit <= 0) timelimit = 1;855856    int i = 0;857    dictEntry *de;858    while ((de = dictNext(&it)) != NULL) {859        hashTemplate *tmpl = hashTemplateGetById((uint64_t)(uintptr_t)dictGetKey(de));860        uint64_t n = dictGetUnsignedIntegerVal(de);861        serverAssert(tmpl != NULL && tmpl->key_refcount >= n);862        htemplates->total_key_refs -= n;863        tmpl->key_refcount -= n;864        if (tmpl->key_refcount == 0) hashTemplateFreeIfUnreferenced(tmpl);865        /* Check the clock every 64 drops. */866        if ((++i & 63) == 0 && ustime() - start > timelimit) return;867    }868    dictResetIterator(&it);869    dictRelease(batch);870    batch = NULL;871}872873/* Idle time (ms) after which an unused fields_lp blob is deleted which was874 * generated due to DUMP/RESTORE propagation master->repica or ASM */875#define HASH_TEMPLATE_FIELDS_LP_IDLE_MS 5000876877/* Max idle blobs collected per batch. */878#define FIELDS_LP_BATCH 64879880typedef struct {881    hashTemplate *tmpls[FIELDS_LP_BATCH];882    int collected; /* idle blobs gathered into tmpls[] */883    int scanned;   /* blobs scanned this window */884} fieldsLpReclaimCtx;885886/* dictScan callback: collect templates whose fields_lp is idle. */887static void fieldsLpReclaimCollect(void *privdata, const dictEntry *de, dictEntry **plink) {888    UNUSED(plink);889    fieldsLpReclaimCtx *ctx = privdata;890    if (ctx->collected >= FIELDS_LP_BATCH) return;891892    hashTemplate *tmpl = dictGetVal((dictEntry *)de);893    if (tmpl->fields_lp == NULL) return;894    ctx->scanned++;895    if (server.mstime - tmpl->fields_lp_last_used < HASH_TEMPLATE_FIELDS_LP_IDLE_MS) return; /* still in use */896    ctx->tmpls[ctx->collected++] = tmpl;897}898899/* Reclaim fields_lp blobs idle longer than HASH_TEMPLATE_FIELDS_LP_IDLE_MS.900 * These blobs are built lazily during DUMP/RESTORE/ASM for O(1) template lookup;901 * once idle they are dead weight (rebuilt on the next DUMP if needed). */902static void hashTemplatesCleanupFieldsLpCron(void) {903    static unsigned long cursor = 0;904    dict *d = htemplates ? htemplates->by_fields_lp : NULL;905    if (!d || dictSize(d) == 0) return;906907    long long start = ustime();908    long long timelimit = 1000000 / server.hz / 200;  /* 0.5% of a hz cycle (~0.5ms) */909    if (timelimit <= 0) timelimit = 1;910911    while (1) {912        /* Collect a small batch, then free it */913        fieldsLpReclaimCtx ctx = { .collected = 0 };914        int steps = 0;915916        do {917            cursor = dictScan(d, cursor, fieldsLpReclaimCollect, &ctx);918        } while (cursor != 0 && ++steps < 16 && ctx.collected < FIELDS_LP_BATCH);919920        /* Blobs present but none idle: nothing to reclaim now, stop (e.g. ASM). */921        if (ctx.collected == 0 && ctx.scanned > 0) break;922923        for (int i = 0; i < ctx.collected; i++) {924            hashTemplate *tmpl = ctx.tmpls[i];925            hashTemplateDropFieldsLp(tmpl);926        }927928        if (cursor == 0) break; /* completed a full sweep */929        if (ustime() - start > timelimit) break;   /* out of time budget */930    }931}932933/* Template registry maintenance, once per serverCron cycle. */934void hashTemplatesCron(void) {935    if (!htemplates) return;936    hashTemplateApplyPendingDrops();937    hashTemplatesCleanupFieldsLpCron();938}939940/* Get number of templates in the registry. */941size_t hashTemplateRegistrySize(void) {942    if (!htemplates || !htemplates->by_fields) return 0;943    return dictSize(htemplates->by_fields);944}945946/* Get total number of template-based keys (sum of all key_refcounts). */947size_t hashTemplateKeyCount(void) {948    if (!htemplates) return 0;949    return htemplates->total_key_refs;950}951952/* Memory held by the shared template registry, reported as953 * used_memory_hash_templates (INFO) and hash.templates (MEMORY STATS). */954size_t hashTemplatesMemUsage(void) {955    if (!htemplates) return 0;956    return htemplates->total_mem_size + sizeof(*htemplates) +957           dictMemUsage(htemplates->by_fields) +958           dictMemUsage(htemplates->by_fields_lp) +959           htemplates->by_id_cap * sizeof(tmplIdChunk *) +960           htemplates->by_id_chunks * sizeof(tmplIdChunk);961}962963/* Find field index in tmpl using binary search (fields are sorted).964 * Returns the index (>= 0) if found, otherwise -(insert_pos + 1) where965 * insert_pos is the position to splice field into to keep fields sorted. */966long long hashTemplateFieldIndex(hashTemplate *tmpl, sds field) {967    long long lo = 0, hi = (long long)tmpl->field_count - 1;968    while (lo <= hi) {969        long long mid = lo + (hi - lo) / 2;970        int cmp = sdscmplen(field, tmpl->fields[mid]);971        if (cmp == 0) return mid;972        if (cmp < 0) hi = mid - 1;973        else lo = mid + 1;974    }975    return -(lo + 1);976}977978/* Return 1 if fields are strictly ascending (sorted, no duplicates), else 0. */979int hashTemplateValidateFields(sds *fields, unsigned long long field_count) {980    for (unsigned long long i = 1; i < field_count; i++)981        if (sdscmplen(fields[i - 1], fields[i]) >= 0)982            return 0;983    return 1;984}985986/*-----------------------------------------------------------------------------987 * HashTemplateLp functions (OBJ_ENCODING_TMPL_LP)988 *989 * Listpack format: [template_id (varint)][value1][value2]...990 * o->ptr points directly to the listpack991 *----------------------------------------------------------------------------*/992993/* Read the template ID stored as the first listpack entry. */994static uint64_t hashTemplateLpGetTemplateId(unsigned char *lp) {995    unsigned char *p = lpFirst(lp);996    serverAssert(p != NULL); /* id entry always present in a well-formed TMPL_LP */997    long long id;998    if (!lpGetIntegerValue(p, &id))999        serverPanic("TMPL_LP listpack header is not an integer template ID");1000    return (uint64_t)id;1001}10021003static hashTemplate *hashTemplateLpGetTemplate(unsigned char *lp) {1004    hashTemplate *tmpl = hashTemplateGetById(hashTemplateLpGetTemplateId(lp));1005    if (tmpl == NULL)1006        serverPanic("TMPL_LP listpack references unknown template ID");1007    return tmpl;1008}10091010/* Return the shared template of a template-encoded hash (TMPL_LP or TMPL_ARRAY). */1011hashTemplate *hashTypeGetTemplate(robj *o) {1012    serverAssert(hashTypeIsTemplate(o));1013    return (o->encoding == OBJ_ENCODING_TMPL_LP) ?1014        hashTemplateLpGetTemplate(o->ptr) :1015        hashTemplateArrayGetTemplate(o->ptr);1016}10171018/* Replace template ID in listpack. */1019unsigned char *hashTemplateLpSetTemplate(unsigned char *lp, hashTemplate *tmpl) {1020    unsigned char *p = lpFirst(lp);1021    serverAssert(p != NULL); /* id entry always present in a well-formed TMPL_LP */1022    unsigned char *v = lpReplaceInteger(lp, &p, (long long)tmpl->id);1023    serverAssert(v != NULL);1024    return v;1025}10261027/* Get pointer to first value entry (skip template ID). */1028static unsigned char *hashTemplateLpFirstValue(unsigned char *lp) {1029    unsigned char *p = lpFirst(lp);  /* template ID entry */1030    serverAssert(p != NULL);         /* id entry always present */1031    unsigned char *v = lpNext(lp, p);1032    serverAssert(v != NULL);         /* a TMPL_LP always holds >= 1 value */1033    return v;1034}10351036/* Seek the value entry at template field index 'idx' (entry 0 is the template1037 * ID, so value idx lives at idx+1). A well-formed TMPL_LP always has this1038 * entry; so assert here rather than hand a NULL cursor. */1039static unsigned char *hashTemplateLpSeekValue(unsigned char *lp, long long idx) {1040    unsigned char *p = lpSeek(lp, idx + 1);1041    serverAssert(p != NULL);1042    return p;1043}10441045/* Walk a TMPL_LP listpack once, filling vptrs[i] with a pointer to the entry1046 * holding the value of field i (i in [0, field_count)). Lets callers that need1047 * many values index them in O(1) instead of an O(field_count) lpSeek per draw1048 * (e.g. HRANDFIELD with a large count). */1049static void hashTemplateLpCollectValuePtrs(unsigned char *lp, unsigned char **vptrs,1050                                           unsigned long long field_count) {1051    unsigned char *p = hashTemplateLpFirstValue(lp);1052    for (unsigned long long i = 0; i < field_count; i++) {1053        serverAssert(p != NULL);1054        vptrs[i] = p;1055        p = lpNext(lp, p);1056    }1057}10581059/* Create listpack with template ID and values. Increments key_refcount. */1060static unsigned char *hashTemplateLpCreate(hashTemplate *tmpl, sds *values) {1061    hashTemplateIncrKeyRef(tmpl);10621063    unsigned long long n = tmpl->field_count;1064    /* +1 for template ID entry */1065    listpackEntry stack_entries[HASH_TMPL_STACK_ENTRIES + 1];1066    listpackEntry *entries = (n + 1 <= HASH_TMPL_STACK_ENTRIES + 1) ?1067                             stack_entries :1068                             zmalloc(sizeof(listpackEntry) * (n + 1));10691070    /* First entry: template ID */1071    entries[0].lval = (long long)tmpl->id;1072    entries[0].sval = NULL;10731074    /* Remaining entries: values */1075    for (unsigned long long i = 0; i < n; i++) {1076        entries[i + 1].sval = (unsigned char *)values[i];1077        entries[i + 1].slen = sdslen(values[i]);1078    }10791080    unsigned char *lp = lpNewWithEntries(entries, n + 1);1081    serverAssert(lp != NULL);1082    if (entries != stack_entries) zfree(entries);10831084    return lp;1085}10861087typedef struct {1088    sds field;1089    unsigned char *vstr;1090    size_t vlen;1091    long long vll;1092} hashTypeFvPair;10931094/* qsort comparator: order hashTypeFvPair by field name. */1095static int hashTypeFvPairCmp(const void *a, const void *b) {1096    return sdscmplen(((const hashTypeFvPair *)a)->field,1097                     ((const hashTypeFvPair *)b)->field);1098}10991100/* Create TMPL_LP from field-value pairs */1101static unsigned char *hashTemplateLpCreateFromPairs(hashTemplate *tmpl,1102                                                    hashTypeFvPair *pairs,1103                                                    unsigned long long n) {1104    hashTemplateIncrKeyRef(tmpl);11051106    /* +1 for template ID entry */1107    listpackEntry stack_entries[HASH_TMPL_STACK_ENTRIES + 1];1108    listpackEntry *entries = (n + 1 <= HASH_TMPL_STACK_ENTRIES + 1) ?1109                             stack_entries :1110                             zmalloc(sizeof(listpackEntry) * (n + 1));11111112    entries[0].lval = (long long)tmpl->id;  /* template ID */1113    entries[0].sval = NULL;1114    for (unsigned long long i = 0; i < n; i++) {1115        if (pairs[i].vstr) {1116            entries[i + 1].sval = pairs[i].vstr;1117            entries[i + 1].slen = pairs[i].vlen;1118        } else {1119            entries[i + 1].sval = NULL;      /* int-encoded value */1120            entries[i + 1].lval = pairs[i].vll;1121        }1122    }11231124    unsigned char *lp = lpNewWithEntries(entries, n + 1);1125    serverAssert(lp != NULL);1126    if (entries != stack_entries) zfree(entries);11271128    return lp;1129}11301131/* Release a template listpack's key ref and free the listpack. May run1132 * in a BIO lazyfree thread and it may just record the ref count drop to be 1133 * collected by main thread later in hashTemplateApplyPendingDrops() */1134void hashTemplateLpFree(unsigned char *lp) {1135    uint64_t id = hashTemplateLpGetTemplateId(lp);1136    if (!canThreadWriteRegistry()) {1137        hashTemplateRecordPendingDrop(id);1138    } else {1139        hashTemplate *tmpl = hashTemplateGetById(id);1140        if (tmpl == NULL)1141            serverPanic("TMPL_LP listpack references unknown template ID");1142        hashTemplateDecrKeyRef(tmpl);1143    }1144    lpFree(lp);1145}11461147/*-----------------------------------------------------------------------------1148 * hashTemplateArray functions (OBJ_ENCODING_TMPL_ARRAY)1149 *----------------------------------------------------------------------------*/11501151static hashTemplate *hashTemplateArrayGetTemplate(hashTemplateArray *hta) {1152    hashTemplate *tmpl = hashTemplateGetById(hta->tmpl_id);1153    serverAssert(tmpl != NULL);1154    serverAssert(tmpl->field_count == hta->field_count);1155    return tmpl;1156}11571158/* Create a new hashTemplateArray. Increments key_refcount.1159 * If take is set, takes ownership of the SDS strings in values array.1160 * Otherwise copies them with sdsdup. */1161static hashTemplateArray *hashTemplateArrayCreate(hashTemplate *tmpl, sds *values, int take) {1162    unsigned long long n = tmpl->field_count;1163    size_t usable;1164    hashTemplateArray *hta = zmalloc_usable(sizeof(*hta) + sizeof(sds) * n, &usable);1165    hta->tmpl_id = tmpl->id;1166    hta->field_count = n;1167    hta->alloc_size = usable;11681169    for (unsigned long long i = 0; i < n; i++) {1170        hta->values[i] = take ? values[i] : sdsdup(values[i]);1171        hta->alloc_size += sdsAllocSize(hta->values[i]);1172    }11731174    hashTemplateIncrKeyRef(tmpl);1175    return hta;1176}11771178static hashTemplateArray *hashTemplateArrayResize(hashTemplateArray *hta, unsigned long long field_count) {1179    size_t old_usable, new_usable;1180    hta = zrealloc_usable(hta, sizeof(*hta) + sizeof(sds) * field_count,1181                          &new_usable, &old_usable);1182    hta->alloc_size += new_usable;1183    hta->alloc_size -= old_usable;1184    return hta;1185}11861187/* Free a hashTemplateArray (release key ref and free data). May run in a BIO1188 * lazyfree thread: uses the tmpl_id/field_count, never the registry. */1189void hashTemplateArrayFree(hashTemplateArray *hta) {1190#ifdef DEBUG_ASSERTIONS1191    size_t values_size = 0;1192    for (unsigned long long i = 0; i < hta->field_count; i++)1193        values_size += sdsAllocSize(hta->values[i]);1194    debugServerAssert(hta->alloc_size - values_size == zmalloc_usable_size(hta));1195#endif11961197    for (unsigned long long i = 0; i < hta->field_count; i++)1198        sdsfree(hta->values[i]);11991200    if (!canThreadWriteRegistry())1201        hashTemplateRecordPendingDrop(hta->tmpl_id);1202    else1203        hashTemplateDecrKeyRef(hashTemplateGetById(hta->tmpl_id));12041205    zfree(hta);1206}12071208/* Build a hash object that shares `tmpl` and stores the given values.1209 * Two template-backed encodings exist:1210 *   TMPL_LP    - values packed in a listpack (compact, small hashes).1211 *   TMPL_ARRAY - values stored as an sds array (used when listpack limits1212 *                would be exceeded).1213 * If `take` is set, ownership of the value sds strings is transferred here. */1214robj *createHashObjectFromTemplate(hashTemplate *tmpl, sds *values, int take) {1215    robj *o;12161217    /* TMPL_LP if the values fit a listpack (field names live in the template). */1218    if (hashTypeSdsArrayFitsLp(values, tmpl->field_count)) {1219        o = createObject(OBJ_HASH, hashTemplateLpCreate(tmpl, values));1220        o->encoding = OBJ_ENCODING_TMPL_LP;1221        /* The listpack copied the value bytes; if we own them, free them now. */1222        if (take)1223            for (unsigned long long i = 0; i < tmpl->field_count; i++)1224                sdsfree(values[i]);1225    } else {1226        o = createObject(OBJ_HASH, hashTemplateArrayCreate(tmpl, values, take));1227        o->encoding = OBJ_ENCODING_TMPL_ARRAY;1228    }1229    return o;1230}12311232/* Move a template hash to `new_tmpl` whose field set must already match the1233 * values held by the key. */1234static void hashTypeTmplSwitch(robj *o, hashTemplate *old_tmpl, hashTemplate *new_tmpl) {1235    hashTemplateIncrKeyRef(new_tmpl);1236    if (o->encoding == OBJ_ENCODING_TMPL_LP) {1237        o->ptr = hashTemplateLpSetTemplate(o->ptr, new_tmpl);1238    } else {1239        serverAssert(o->encoding == OBJ_ENCODING_TMPL_ARRAY);1240        hashTemplateArray *hta = o->ptr;1241        hta->tmpl_id = new_tmpl->id;1242        hta->field_count = new_tmpl->field_count;1243    }1244    hashTemplateDecrKeyRef(old_tmpl);1245}12461247/* Overwrite the value of the field at template index `idx`. Returns 1 if1248 * `value` was adopted. */1249static int hashTypeTmplWriteValueAt(robj *o, long long idx, sds value, int take) {1250    if (o->encoding == OBJ_ENCODING_TMPL_LP) {1251        unsigned char *lp = o->ptr;1252        unsigned char *p = hashTemplateLpSeekValue(lp, idx);1253        o->ptr = lpReplace(lp, &p, (unsigned char *)value, sdslen(value));1254        serverAssert(o->ptr != NULL);1255        return 0;1256    } else {1257        serverAssert(o->encoding == OBJ_ENCODING_TMPL_ARRAY);1258        hashTemplateArray *hta = o->ptr;1259        serverAssert((unsigned long long)idx < hta->field_count);1260        hta->alloc_size -= sdsAllocSize(hta->values[idx]);1261        sdsfree(hta->values[idx]);1262        hta->values[idx] = take ? value : sdsdup(value);1263        hta->alloc_size += sdsAllocSize(hta->values[idx]);1264        return take != 0;1265    }1266}12671268/* A field an HSET-style command adds to a template hash. */1269typedef struct {1270    sds field;1271    sds value;1272    unsigned long long insert_pos;1273    int arg_idx;          /* argv order, so the last of duplicate fields sets the value */1274} tmplNewField;12751276/* qsort comparator: field name ascending, then argv position. */1277static int tmplNewFieldCmp(const void *a, const void *b) {1278    const tmplNewField *x = a, *y = b;1279    int cmp = sdscmplen(x->field, y->field);1280    return cmp ? cmp : x->arg_idx - y->arg_idx;1281}12821283/* Add new fields to a template hash with a single template switch. `new_fields`1284 * must be sorted by field name and have no duplicates, and insert_pos must be1285 * the field's final position in the new template. Values are copied. */1286static void hashTypeTmplAddFields(redisDb *db, robj *o, hashTemplate *tmpl,1287                                  tmplNewField *new_fields, int num_new_fields)1288{1289    unsigned long long old_field_count = tmpl->field_count;12901291    /* Check if the listpack needs to be converted to array */1292    if (o->encoding == OBJ_ENCODING_TMPL_LP &&1293        old_field_count + num_new_fields > server.hash_max_listpack_entries)1294    {1295        hashTypeConvert(db, o, OBJ_ENCODING_TMPL_ARRAY);1296    }12971298    /* Merge old and new names into the new template's field array. */1299    unsigned long long total_field_count = old_field_count + num_new_fields;1300    sds stack_fields[HASH_TMPL_STACK_ENTRIES];1301    sds *fields_arr = (total_field_count <= HASH_TMPL_STACK_ENTRIES) ? stack_fields :1302                                                                     zmalloc(sizeof(sds) * total_field_count);1303    uint64_t hash = tmpl->hash;1304    unsigned long long old_pos = 0;1305    int added = 0;1306    for (unsigned long long i = 0; i < total_field_count; i++) {1307        if (added < num_new_fields && new_fields[added].insert_pos == i) {1308            fields_arr[i] = new_fields[added].field;1309            hash += computeFieldHash(new_fields[added].field); /* incremental set hash */1310            added++;1311        } else {1312            serverAssert(old_pos < old_field_count);1313            fields_arr[i] = tmpl->fields[old_pos++];1314        }1315    }1316    serverAssert(added == num_new_fields && old_pos == old_field_count);13171318    hashTemplate *new_tmpl = hashTemplateGetOrCreateWithHash(hash, fields_arr, total_field_count);1319    if (fields_arr != stack_fields) zfree(fields_arr);1320    1321    /* Update the values */1322    if (o->encoding == OBJ_ENCODING_TMPL_LP) {1323        unsigned char *lp = o->ptr;1324        for (int i = 0; i < num_new_fields; i++) {1325            sds v = new_fields[i].value;1326            serverAssert(new_fields[i].insert_pos <= old_field_count + (unsigned long long)i);1327            /* Check if this is an insert or an append */1328            if (new_fields[i].insert_pos < old_field_count + (unsigned long long)i) {1329                unsigned char *p = hashTemplateLpSeekValue(lp, new_fields[i].insert_pos);1330                lp = lpInsertString(lp, (unsigned char *)v, sdslen(v), p, LP_BEFORE, NULL);1331            } else {1332                lp = lpAppend(lp, (unsigned char *)v, sdslen(v));1333            }1334            serverAssert(lp != NULL);1335        }1336        o->ptr = lp;1337    } else {1338        serverAssert(o->encoding == OBJ_ENCODING_TMPL_ARRAY);1339        serverAssert(num_new_fields > 0);1340        hashTemplateArray *hta = hashTemplateArrayResize(o->ptr, total_field_count);13411342        /* Open a gap: move the old values from the first insert position1343         * onwards to the end of the array. Then fill each slot in order,1344         * either with the next new value or with the next old one. */1345        unsigned long long first = new_fields[0].insert_pos;1346        serverAssert(first <= old_field_count);1347        memmove(&hta->values[first + num_new_fields], &hta->values[first],1348                sizeof(sds) * (old_field_count - first));1349        old_pos = first + num_new_fields;1350        added = 0;1351        for (unsigned long long i = first; added < num_new_fields; i++) {1352            serverAssert(i < total_field_count);1353            if (new_fields[added].insert_pos == i) {1354                hta->values[i] = sdsdup(new_fields[added].value);1355                hta->alloc_size += sdsAllocSize(hta->values[i]);1356                added++;1357            } else {1358                serverAssert(old_pos < total_field_count);1359                hta->values[i] = hta->values[old_pos++];1360            }1361        }1362        o->ptr = hta;1363    }1364    hashTypeTmplSwitch(o, tmpl, new_tmpl);1365}13661367/* Set the `n` field-value pairs at argv[i * 2] and argv[i * 2 + 1] on a template1368 * hash: an existing field's value is overwritten in place and the fields the1369 * template lacks are all added in a single template switch.1370 * Returns the number of fields added. */1371static int hashTypeTmplSetFields(redisDb *db, robj *o, robj **argv, int n) {1372    hashTemplate *tmpl = hashTypeGetTemplate(o);1373    tmplNewField stack_new_fields[HASH_TMPL_STACK_ENTRIES];1374    tmplNewField *new_fields = (n <= HASH_TMPL_STACK_ENTRIES) ? stack_new_fields :1375                               zmalloc(sizeof(tmplNewField) * n);1376    int num_new_fields = 0;13771378    /* One pass: overwrite the fields that exist, collect the ones that do not.1379     * Binary searching here is the only field lookup the command does. */1380    for (int i = 0; i < n; i++) {1381        sds field = argv[i * 2]->ptr;1382        sds value = argv[i * 2 + 1]->ptr;13831384        long long idx = hashTemplateFieldIndex(tmpl, field);1385        if (idx >= 0) {1386            hashTypeTmplWriteValueAt(o, idx, value, 0); /* Existing field */1387            continue;1388        }13891390        /* Temporarily set insert pos for the existing template.1391         * Later, it will be adjusted for the new template below */1392        new_fields[num_new_fields].insert_pos = -idx - 1;1393        new_fields[num_new_fields].field = field;1394        new_fields[num_new_fields].value = value;1395        new_fields[num_new_fields].arg_idx = i;1396        num_new_fields++;1397    }13981399    if (num_new_fields > 0) {1400        /* Sort and drop duplicates, keeping the last occurrence of a field1401         * given more than once, so its last value wins. A single field needs1402         * neither. */1403        if (num_new_fields > 1) {1404            qsort(new_fields, num_new_fields, sizeof(*new_fields), tmplNewFieldCmp);1405            int unique = 0;1406            for (int i = 0; i < num_new_fields; i++) {1407                /* Skip the entry if the next one is the same field. */1408                if (i + 1 < num_new_fields &&1409                    sdscmplen(new_fields[i].field, new_fields[i + 1].field) == 0)1410                {1411                    continue;1412                }1413                new_fields[unique++] = new_fields[i];1414            }1415            num_new_fields = unique;1416        }14171418        /* So far insert_pos is where the field goes among the OLD template fields.1419         * Every new field placed before it shifts it right by one, so its1420         * final position in the new template is insert_pos + k. */1421        for (int k = 0; k < num_new_fields; k++) {1422            serverAssert(new_fields[k].insert_pos <= tmpl->field_count);1423            new_fields[k].insert_pos += k;1424        }14251426        hashTypeTmplAddFields(db, o, tmpl, new_fields, num_new_fields);1427    }14281429    if (new_fields != stack_new_fields) zfree(new_fields);1430    return num_new_fields;1431}14321433/* Reply with the value of the field at template index `idx`. */1434static void addTmplValueToReply(client *c, robj *o, long long idx) {1435    if (o->encoding == OBJ_ENCODING_TMPL_LP) {1436        unsigned int vlen;1437        long long vll;1438        unsigned char *p = hashTemplateLpSeekValue(o->ptr, idx);1439        unsigned char *vstr = lpGetValue(p, &vlen, &vll);1440        if (vstr)1441            addReplyBulkCBuffer(c, vstr, vlen);1442        else1443            addReplyBulkLongLong(c, vll);1444    } else {1445        serverAssert(o->encoding == OBJ_ENCODING_TMPL_ARRAY);1446        hashTemplateArray *hta = o->ptr;1447        serverAssert((unsigned long long)idx < hta->field_count);1448        sds value = hta->values[idx];1449        addReplyBulkCBuffer(c, value, sdslen(value));1450    }1451}14521453/* qsort comparator for template field indexes. */1454static int tmplIdxCmp(const void *a, const void *b) {1455    long long x = *(const long long *)a;1456    long long y = *(const long long *)b;1457    return (x > y) - (x < y);1458}14591460/* Delete fields from a template hash with a single template switch. `del_indexes`1461 * holds the template indexes of the fields to delete, in any order. Each index1462 * must be valid and given once. */1463static void hashTypeTmplDropFields(robj *o, hashTemplate *tmpl,1464                                   long long *del_indexes,1465                                   unsigned long long num_del_fields)1466{1467    unsigned long long old_field_count = tmpl->field_count;1468    serverAssert(num_del_fields > 0 && num_del_fields <= old_field_count);14691470    /* The walks below expect ascending indexes. */1471    if (num_del_fields > 1)1472        qsort(del_indexes, num_del_fields, sizeof(*del_indexes), tmplIdxCmp);14731474    if (num_del_fields == old_field_count) {1475        /* Losing the last field turns the hash into a plain empty listpack.1476         * Caller will delete the key. */1477        if (o->encoding == OBJ_ENCODING_TMPL_LP) {1478            hashTemplateLpFree(o->ptr);1479        } else {1480            serverAssert(o->encoding == OBJ_ENCODING_TMPL_ARRAY);1481            hashTemplateArrayFree(o->ptr);1482        }1483        o->ptr = lpNew(0);1484        o->encoding = OBJ_ENCODING_LISTPACK;1485        return;1486    }14871488    /* The surviving fields, in order, are the new field set. */1489    unsigned long long new_field_count = old_field_count - num_del_fields;1490    fieldvec fvfields;1491    vec *vfields = fieldvecInit(&fvfields, new_field_count);1492    uint64_t hash = tmpl->hash;1493    unsigned long long kept = 0, dropped = 0;14941495    for (unsigned long long i = 0; i < old_field_count; i++) {1496        if (dropped < num_del_fields && del_indexes[dropped] == (long long)i) {1497            hash -= computeFieldHash(tmpl->fields[i]); /* incremental set hash */1498            dropped++;1499        } else {1500            vecPush(vfields, tmpl->fields[i]);1501        }1502    }1503    serverAssert(vecSize(vfields) == new_field_count && dropped == num_del_fields);15041505    hashTemplate *new_tmpl = hashTemplateGetOrCreateWithHash(hash, (sds *)vecData(vfields), new_field_count);1506    vecRelease(vfields);15071508    /* Update the values */1509    if (o->encoding == OBJ_ENCODING_TMPL_LP) {1510        /* Collect the dropped entries in one walk, then delete them in one pass. */1511        unsigned char *lp = o->ptr;1512        fieldvec fvps;1513        vec *vps = fieldvecInit(&fvps, num_del_fields);1514        /* Seek the first dropped entry; each next one is `distance` entries1515         * after the previous, so step forward instead of a fresh seek. */1516        vecPush(vps, hashTemplateLpSeekValue(lp, del_indexes[0]));1517        for (unsigned long long i = 1; i < num_del_fields; i++) {1518            unsigned long long distance = del_indexes[i] - del_indexes[i - 1];1519            unsigned char *p = lpNextN(lp, vecGet(vps, i - 1), distance);1520            serverAssert(p != NULL);1521            vecPush(vps, p);1522        }1523        o->ptr = lpBatchDelete(lp, (unsigned char **)vecData(vps), num_del_fields);1524        vecRelease(vps);1525    } else {1526        serverAssert(o->encoding == OBJ_ENCODING_TMPL_ARRAY);1527        hashTemplateArray *hta = o->ptr;1528        /* Same walk as the field loop above, this time over the values. */1529        kept = dropped = 0;1530        for (unsigned long long i = 0; i < old_field_count; i++) {1531            if (dropped < num_del_fields && del_indexes[dropped] == (long long)i) {1532                hta->alloc_size -= sdsAllocSize(hta->values[i]);1533                sdsfree(hta->values[i]);1534                dropped++;1535            } else {1536                hta->values[kept++] = hta->values[i];1537            }1538        }1539        serverAssert(kept == new_field_count && dropped == num_del_fields);1540        o->ptr = hashTemplateArrayResize(hta, new_field_count);1541    }1542    hashTypeTmplSwitch(o, tmpl, new_tmpl);1543}15441545/* Delete the `n` fields in argv[] from a template hash in a single template1546 * switch, pushing the fields actually deleted into `vdeleted` in argv order.1547 * With `c`, each value is replied first (HGETDEL), null for a miss or repeat. */1548static void hashTypeTmplDeleteFields(robj *o, robj **argv, int n, vec *vdeleted, client *c) {1549    hashTemplate *tmpl = hashTypeGetTemplate(o);1550    unsigned long long old_field_count = tmpl->field_count;1551    unsigned long long num_del_fields = 0;1552    /* Fields are collected first and dropped all at once at the end. Since the 1553     * hash stays untouched during the scan, a field given twice in argv would be 1554     * found twice: delmark[] marks the fields seen in an earlier argv position. */1555    unsigned char stack_delmark[HASH_TMPL_STACK_ENTRIES] = {0};1556    unsigned char *delmark = NULL;1557    long long stack_del_indexes[HASH_TMPL_STACK_ENTRIES];1558    long long *del_indexes = (n <= HASH_TMPL_STACK_ENTRIES) ? stack_del_indexes :1559                                                              zmalloc(sizeof(long long) * n);1560    for (int i = 0; i < n; i++) {1561        long long idx = hashTemplateFieldIndex(tmpl, argv[i]->ptr);1562        /* Skip missing fields or if a field given twice in argv. */1563        if (idx < 0 || (delmark && delmark[idx])) {1564            if (c) addReplyNull(c);1565            continue;1566        }1567        /* Reply while the values are still in place. */1568        if (c) addTmplValueToReply(c, o, idx);1569        1570        /* Mark the field as deleted */1571        if (n > 1) {1572            if (delmark == NULL) {1573                delmark = (old_field_count <= HASH_TMPL_STACK_ENTRIES) ?1574                          stack_delmark : zcalloc(old_field_count);1575            }1576            serverAssert((unsigned long long)idx < old_field_count);1577            delmark[idx] = 1;1578        }1579        1580        /* Mark field idx to delete */1581        del_indexes[num_del_fields++] = idx;1582        vecPush(vdeleted, argv[i]);1583    }1584    1585    /* Delete the fields */1586    if (num_del_fields > 0)1587        hashTypeTmplDropFields(o, tmpl, del_indexes, num_del_fields);15881589    if (delmark && delmark != stack_delmark) zfree(delmark);1590    if (del_indexes != stack_del_indexes) zfree(del_indexes);1591}15921593/*-----------------------------------------------------------------------------1594 * listpackEx functions1595 *----------------------------------------------------------------------------*/1596/*1597 * If any of hash field expiration command is called on a listpack hash object1598 * for the first time, we convert it to OBJ_ENCODING_LISTPACK_EX encoding.1599 * We allocate "struct listpackEx" which holds listpack pointer and expiry1600 * metadata. In the listpack string, we append another TTL entry for each field1601 * value pair. From now on, listpack will have triplets in it: field-value-ttl.1602 * If TTL is not set for a field, we store 'zero' as the TTL value. 'zero' is1603 * encoded as two bytes in the listpack. Memory overhead of a non-existing TTL1604 * will be two bytes per field.1605 *1606 * Fields in the listpack will be ordered by TTL. Field with the smallest expiry1607 * time will be the first item. Fields without TTL will be at the end of the1608 * listpack. This way, it is easier/faster to find expired items.1609 */16101611#define HASH_LP_NO_TTL 016121613struct listpackEx *listpackExCreate(void) {1614    listpackEx *lpt = zcalloc(sizeof(*lpt));1615    lpt->meta.trash = 1;1616    lpt->lp = NULL;1617    return lpt;1618}16191620static void listpackExFree(listpackEx *lpt) {1621    lpFree(lpt->lp);1622    zfree(lpt);1623}16241625struct lpFindArgs {1626    uint64_t max_to_search; /* [in] Max number of tuples to search */1627    uint64_t expire_time;   /* [in] Find the tuple that has a TTL larger than expire_time */1628    unsigned char *p;       /* [out] First item of the tuple that has a TTL larger than expire_time */1629    int expired;            /* [out] Number of tuples that have TTLs less than expire_time */1630    int index;              /* Internally used */1631    unsigned char *fptr;    /* Internally used, temp ptr */1632};16331634/* Callback for lpFindCb(). Used to find number of expired fields as part of1635 * active expiry or when trying to find the position for the new field according1636 * to its expiry time.*/1637static int cbFindInListpack(const unsigned char *lp, unsigned char *p,1638                            void *user, unsigned char *s, long long slen)1639{1640    (void) lp;1641    struct lpFindArgs *r = user;16421643    r->index++;16441645    if (r->max_to_search == 0)1646        return 0; /* Break the loop and return */16471648    if (r->index % 3 == 1) {1649        r->fptr = p;  /* First item of the tuple. */1650    } else if (r->index % 3 == 0) {1651        serverAssert(!s);16521653        /* Third item of a tuple is expiry time */1654        if (slen == HASH_LP_NO_TTL || (uint64_t) slen >= r->expire_time) {1655            r->p = r->fptr;1656            return 0; /* Break the loop and return */1657        }1658        r->expired++;1659        r->max_to_search--;1660    }16611662    return 1;1663}16641665/* Returns number of expired fields. */1666static uint64_t listpackExExpireDryRun(const robj *o) {1667    serverAssert(o->encoding == OBJ_ENCODING_LISTPACK_EX);16681669    listpackEx *lpt = o->ptr;16701671    struct lpFindArgs r = {1672        .max_to_search = UINT64_MAX,1673        .expire_time = commandTimeSnapshot(),1674    };16751676    lpFindCb(lpt->lp, NULL, &r, cbFindInListpack, 0);1677    return r.expired;1678}16791680/* Returns the expiration time of the item with the nearest expiration. */1681static uint64_t listpackExGetMinExpire(robj *o) {1682    serverAssert(o->encoding == OBJ_ENCODING_LISTPACK_EX);16831684    long long expireAt;1685    unsigned char *fptr;1686    listpackEx *lpt = o->ptr;16871688    /* As fields are ordered by expire time, first field will have the smallest1689     * expiry time. Third element is the expiry time of the first field */1690    fptr = lpSeek(lpt->lp, 2);1691    if (fptr != NULL) {1692        serverAssert(lpGetIntegerValue(fptr, &expireAt));16931694        /* Check if this is a non-volatile field. */1695        if (expireAt != HASH_LP_NO_TTL)1696            return expireAt;1697    }16981699    return EB_EXPIRE_TIME_INVALID;1700}17011702/* Walk over fields and delete the expired ones. */1703void listpackExExpire(redisDb *db, kvobj *kv, ExpireInfo *info) {1704    OnFieldExpireCtx *ctx = info->ctx;1705    serverAssert(kv->encoding == OBJ_ENCODING_LISTPACK_EX);1706    uint64_t expired = 0, min = EB_EXPIRE_TIME_INVALID;1707    unsigned char *ptr;1708    listpackEx *lpt = kv->ptr;17091710    ptr = lpFirst(lpt->lp);17111712    sds key = kvobjGetKey(kv);17131714    while (ptr != NULL && (info->itemsExpired < info->maxToExpire)) {1715        long long val;1716        int64_t flen;1717        unsigned char intbuf[LP_INTBUF_SIZE], *fref;17181719        fref = lpGet(ptr, &flen, intbuf);17201721        ptr = lpNext(lpt->lp, ptr);1722        serverAssert(ptr);1723        ptr = lpNext(lpt->lp, ptr);1724        serverAssert(ptr && lpGetIntegerValue(ptr, &val));17251726        /* Fields are ordered by expiry time. If we reached to a non-expired1727         * or a non-volatile field, we know rest is not yet expired. A field is1728         * expired iff its expire-time is strictly less than `now`, matching1729         * lazy-expiry semantics (see hashTypeIsExpired()) and ebExpire(). */1730        if (val == HASH_LP_NO_TTL || (uint64_t) val >= info->now)1731            break;17321733        /* Collect expired field for subkey notification. */1734        if (ctx->vexpired) {1735            char *fstr = (char *)(fref ? fref : intbuf);1736            vecPush(ctx->vexpired, createStringObject(fstr, flen));1737        }17381739        propagateHashFieldDeletion(db, key, (char *)((fref) ? fref : intbuf), flen);1740        server.stat_expired_subkeys++;1741        if (ctx->activeEx) server.stat_expired_subkeys_active++;17421743        ptr = lpNext(lpt->lp, ptr);17441745        info->itemsExpired++;1746        expired++;1747    }17481749    if (expired) {1750        size_t oldsize = 0;1751        if (server.memory_tracking_enabled)1752            oldsize = kvobjAllocSize(kv);1753        lpt->lp = lpDeleteRange(lpt->lp, 0, expired * 3);1754        if (server.memory_tracking_enabled)1755            updateSlotAllocSize(db, getKeySlot(key), kv, oldsize, kvobjAllocSize(kv));17561757        /* update keysizes */1758        unsigned long l = lpLength(lpt->lp) / 3;1759        updateKeysizesHist(db, OBJ_HASH, l + expired, l);1760    }17611762    min = hashTypeGetMinExpire(kv, 1 /*accurate*/);1763    info->nextExpireTime = min;1764}17651766static void listpackExAddInternal(robj *o, listpackEntry ent[3]) {1767    listpackEx *lpt = o->ptr;17681769    /* Shortcut, just append at the end if this is a non-volatile field. */1770    if (ent[2].lval == HASH_LP_NO_TTL) {1771        lpt->lp = lpBatchAppend(lpt->lp, ent, 3);1772        return;1773    }17741775    struct lpFindArgs r = {1776            .max_to_search = UINT64_MAX,1777            .expire_time = ent[2].lval,1778    };17791780    /* Check if there is a field with a larger TTL. */1781    lpFindCb(lpt->lp, NULL, &r, cbFindInListpack, 0);17821783    /* If list is empty or there is no field with a larger TTL, result will be1784     * NULL. Otherwise, just insert before the found item.*/1785    if (r.p)1786        lpt->lp = lpBatchInsert(lpt->lp, r.p, LP_BEFORE, ent, 3, NULL);1787    else1788        lpt->lp = lpBatchAppend(lpt->lp, ent, 3);1789}17901791/* Add new field ordered by expire time. */1792void listpackExAddNew(robj *o, char *field, size_t flen,1793                      char *value, size_t vlen, uint64_t expireAt) {1794    listpackEntry ent[3] = {1795        {.sval = (unsigned char*) field, .slen = flen},1796        {.sval = (unsigned char*) value, .slen = vlen},1797        {.lval = expireAt}1798    };17991800    listpackExAddInternal(o, ent);1801}18021803/* If expiry time is changed, this function will place field into the correct1804 * position. First, it deletes the field and re-inserts to the listpack ordered1805 * by expiry time. */1806static void listpackExUpdateExpiry(robj *o, sds field,1807                                   unsigned char *fptr,1808                                   unsigned char *vptr,1809                                   uint64_t expire_at) {1810    unsigned int slen = 0;1811    long long val = 0;1812    unsigned char tmp[512] = {0};1813    unsigned char *valstr;1814    sds tmpval = NULL;1815    listpackEx *lpt = o->ptr;18161817    /* Copy value */1818    valstr = lpGetValue(vptr, &slen, &val);1819    if (valstr) {1820        /* Normally, item length in the listpack is limited by1821         * 'hash-max-listpack-value' config. It is unlikely, but it might be1822         * larger than sizeof(tmp). */1823        if (slen > sizeof(tmp))1824            tmpval = sdsnewlen(valstr, slen);1825        else1826            memcpy(tmp, valstr, slen);1827    }18281829    /* Delete field name, value and expiry time */1830    lpt->lp = lpDeleteRangeWithEntry(lpt->lp, &fptr, 3);18311832    listpackEntry ent[3] = {{0}};18331834    ent[0].sval = (unsigned char*) field;1835    ent[0].slen = sdslen(field);18361837    if (valstr) {1838        ent[1].sval = tmpval ? (unsigned char *) tmpval : tmp;1839        ent[1].slen = slen;1840    } else {1841        ent[1].lval = val;1842    }1843    ent[2].lval = expire_at;18441845    listpackExAddInternal(o, ent);1846    sdsfree(tmpval);1847}18481849/* Update field expire time. */1850SetExRes hashTypeSetExpiryListpack(HashTypeSetEx *ex, sds field,1851                                   unsigned char *fptr, unsigned char *vptr,1852                                   unsigned char *tptr, uint64_t expireAt)1853{1854    long long expireTime;1855    uint64_t prevExpire = EB_EXPIRE_TIME_INVALID;18561857    serverAssert(lpGetIntegerValue(tptr, &expireTime));18581859    if (expireTime != HASH_LP_NO_TTL) {1860        prevExpire = (uint64_t) expireTime;1861    }18621863    /* Special value of EXPIRE_TIME_INVALID indicates field should be persisted.*/1864    if (expireAt == EB_EXPIRE_TIME_INVALID) {1865        /* Return error if already there is no ttl. */1866        if (prevExpire == EB_EXPIRE_TIME_INVALID)1867            return HSETEX_NO_CONDITION_MET;1868        listpackExUpdateExpiry(ex->hashObj, field, fptr, vptr, HASH_LP_NO_TTL);1869        return HSETEX_OK;1870    }18711872    if (prevExpire == EB_EXPIRE_TIME_INVALID) {1873        /* For fields without expiry, LT condition is considered valid */1874        if (ex->expireSetCond & (HFE_XX | HFE_GT))1875            return HSETEX_NO_CONDITION_MET;1876    } else {1877        if (((ex->expireSetCond == HFE_GT) && (prevExpire >= expireAt)) ||1878            ((ex->expireSetCond == HFE_LT) && (prevExpire <= expireAt)) ||1879            (ex->expireSetCond == HFE_NX) )1880            return HSETEX_NO_CONDITION_MET;18811882        /* Track of minimum expiration time (only later update global HFE DS) */1883        if (ex->minExpireFields > prevExpire)1884            ex->minExpireFields = prevExpire;1885    }18861887    /* If expired, then delete the field and propagate the deletion.1888     * If replica, continue like the field is valid */1889    if (unlikely(checkAlreadyExpired(expireAt))) {1890        propagateHashFieldDeletion(ex->db, ex->key->ptr, field, sdslen(field));1891        hashTypeDelete(ex->hashObj, field);1892        server.stat_expired_subkeys++;1893        return HSETEX_DELETED;1894    }18951896    if (ex->minExpireFields > expireAt)1897        ex->minExpireFields = expireAt;18981899    listpackExUpdateExpiry(ex->hashObj, field, fptr, vptr, expireAt);1900    return HSETEX_OK;1901}19021903/* Returns 1 if expired */1904int hashTypeIsExpired(const robj *o, uint64_t expireAt) {1905    if (server.allow_access_expired) return 0;19061907    if (o->encoding == OBJ_ENCODING_LISTPACK_EX) {1908        if (expireAt == HASH_LP_NO_TTL)1909            return 0;1910    } else if (o->encoding == OBJ_ENCODING_HT) {1911        if (expireAt == EB_EXPIRE_TIME_INVALID)1912            return 0;1913    } else {1914        serverPanic("Unknown encoding: %d", o->encoding);1915    }19161917    return (mstime_t) expireAt < commandTimeSnapshot();1918}19191920/* Returns listpack pointer of the object. */1921unsigned char *hashTypeListpackGetLp(robj *o) {1922    if (o->encoding == OBJ_ENCODING_LISTPACK)1923        return o->ptr;1924    else if (o->encoding == OBJ_ENCODING_LISTPACK_EX)1925        return ((listpackEx*)o->ptr)->lp;19261927    serverPanic("Unknown encoding: %d", o->encoding);1928}19291930/*-----------------------------------------------------------------------------1931 * Hash type API1932 *----------------------------------------------------------------------------*/19331934/* Check the length of a number of objects to see if we need to convert a1935 * listpack to a real hash. Note that we only check string encoded objects1936 * as their string length can be queried in constant time. */1937void hashTypeTryConversion(redisDb *db, kvobj *o, robj **argv, int start, int end) {1938    int tmpl_lp = (o->encoding == OBJ_ENCODING_TMPL_LP);1939    int target_enc = tmpl_lp ? OBJ_ENCODING_TMPL_ARRAY : OBJ_ENCODING_HT;19401941    /* TMPL_ARRAY and HT don't need conversion checks. */1942    if (o->encoding != OBJ_ENCODING_LISTPACK &&1943        o->encoding != OBJ_ENCODING_LISTPACK_EX &&1944        o->encoding != OBJ_ENCODING_TMPL_LP)1945    {1946        return;1947    }19481949    /* Determine target encoding for conversion. */1950    if (!tmpl_lp) {1951        /* Check field count limit for regular listpack. */1952        size_t new_fields = (end - start + 1) / 2;1953        if (new_fields > server.hash_max_listpack_entries) {1954            hashTypeConvert(db, o, OBJ_ENCODING_HT);1955            dictExpand(o->ptr, new_fields);1956            return;1957        }1958    }19591960    /* Check value sizes (and field sizes for non-TMPL_LP). */1961    size_t sum = 0;1962    for (int i = start; i <= end; i++) {1963        if (tmpl_lp && ((i - start) % 2 == 0))1964            continue;1965        if (!sdsEncodedObject(argv[i]))1966            continue;1967        size_t len = sdslen(argv[i]->ptr);1968        if (len > server.hash_max_listpack_value) {1969            hashTypeConvert(db, o, target_enc);1970            return;1971        }1972        sum += len;1973    }1974    unsigned char *lp = (o->encoding == OBJ_ENCODING_LISTPACK_EX) ?1975                        ((listpackEx*)o->ptr)->lp : o->ptr;1976    if (!lpSafeToAdd(lp, sum))1977        hashTypeConvert(db, o, target_enc);1978}19791980/* Get the value from a listpack encoded hash, identified by field. */1981GetFieldRes hashTypeGetFromListpack(robj *o, sds field,1982                            unsigned char **vstr,1983                            unsigned int *vlen,1984                            long long *vll,1985                            uint64_t *expiredAt)1986{1987    *expiredAt = EB_EXPIRE_TIME_INVALID;1988    unsigned char *zl, *fptr = NULL, *vptr = NULL;19891990    if (o->encoding == OBJ_ENCODING_LISTPACK) {1991        zl = o->ptr;1992        fptr = lpFirst(zl);1993        if (fptr != NULL) {1994            fptr = lpFind(zl, fptr, (unsigned char*)field, sdslen(field), 1);1995            if (fptr != NULL) {1996                /* Grab pointer to the value (fptr points to the field) */1997                vptr = lpNext(zl, fptr);1998                serverAssert(vptr != NULL);1999            }2000        }