Warning: Allocation result must be checked for NULL before use to prevent null pointer dereference.
e->free_page_buckets = calloc(cf->page_buckets, sizeof(store_page *));
1/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */23#include "config.h"4// FIXME: config.h?5#include <stdint.h>6#include <stdbool.h>7// end FIXME8#include <stdlib.h>9#include <limits.h>10#include <pthread.h>11#include <sys/types.h>12#include <sys/stat.h>13#include <sys/uio.h>14#include <fcntl.h>15#include <unistd.h>16#include <stdio.h>17#include <string.h>18#include <assert.h>19#include "extstore.h"2021// TODO: better if an init option turns this on/off.22#ifdef EXTSTORE_DEBUG23#define E_DEBUG(...) \24 do { \25 fprintf(stderr, __VA_ARGS__); \26 } while (0)27#else28#define E_DEBUG(...)29#endif3031#define STAT_L(e) pthread_mutex_lock(&e->stats_mutex);32#define STAT_UL(e) pthread_mutex_unlock(&e->stats_mutex);33#define STAT_INCR(e, stat, amount) { \34 pthread_mutex_lock(&e->stats_mutex); \35 e->stats.stat += amount; \36 pthread_mutex_unlock(&e->stats_mutex); \37}3839#define STAT_DECR(e, stat, amount) { \40 pthread_mutex_lock(&e->stats_mutex); \41 e->stats.stat -= amount; \42 pthread_mutex_unlock(&e->stats_mutex); \43}4445typedef struct __store_wbuf {46 struct __store_wbuf *next;47 char *buf;48 char *buf_pos;49 unsigned int free;50 unsigned int size;51 unsigned int offset; /* offset into page this write starts at */52 bool full; /* done writing to this page */53 bool flushed; /* whether wbuf has been flushed to disk */54} _store_wbuf;5556typedef struct _store_page {57 pthread_mutex_t mutex; /* Need to be held for most operations */58 uint64_t obj_count; /* _delete can decrease post-closing */59 uint64_t bytes_used; /* _delete can decrease post-closing */60 uint64_t offset; /* starting address of page within fd */61 unsigned int version;62 unsigned int refcount;63 unsigned int allocated;64 unsigned int written; /* item offsets can be past written if wbuf not flushed */65 unsigned int bucket; /* which bucket the page is linked into */66 unsigned int free_bucket; /* which bucket this page returns to when freed */67 int fd;68 unsigned short id;69 bool active; /* actively being written to */70 bool closed; /* closed and draining before free */71 bool free; /* on freelist */72 _store_wbuf *wbuf; /* currently active wbuf from the stack */73 struct _store_page *next;74} store_page;7576typedef struct store_engine store_engine;77typedef struct {78 pthread_mutex_t mutex;79 pthread_cond_t cond;80 obj_io *queue;81 obj_io *queue_tail;82 store_engine *e;83 unsigned int depth; // queue depth84} store_io_thread;8586// sub-struct for maintenance related tasks.87struct store_maint {88 pthread_mutex_t mutex;89};9091struct store_engine {92 pthread_mutex_t mutex; /* covers internal stacks and variables */93 store_page *pages; /* directly addressable page list */94 _store_wbuf *wbuf_stack; /* wbuf freelist */95 obj_io *io_stack; /* IO's to use with submitting wbuf's */96 store_io_thread *io_threads;97 store_io_thread *bg_thread; /* dedicated thread for write submit / compact ops */98 store_page **page_buckets; /* stack of pages currently allocated to each bucket */99 store_page **free_page_buckets; /* stack of use-case isolated free pages */100 size_t page_size;101 unsigned int version; /* global version counter */102 unsigned int last_io_thread; /* round robin the IO threads */103 unsigned int io_threadcount; /* count of IO threads */104 unsigned int page_count;105 unsigned int page_free; /* unallocated pages */106 unsigned int page_bucketcount; /* count of potential page buckets */107 unsigned int free_page_bucketcount; /* count of free page buckets */108 unsigned int io_depth; /* FIXME: Might cache into thr struct */109 pthread_mutex_t stats_mutex;110 struct extstore_stats stats;111 struct store_maint maint;112};113114// FIXME: code is duplicated from thread.c since extstore.c doesn't pull in115// the memcached ecosystem. worth starting a cross-utility header with static116// definitions/macros?117// keeping a minimal func here for now.118#define THR_NAME_MAXLEN 16119static void thread_setname(pthread_t thread, const char *name) {120assert(strlen(name) < THR_NAME_MAXLEN);121#if defined(__linux__) && defined(HAVE_PTHREAD_SETNAME_NP)122pthread_setname_np(thread, name);123#endif124}125#undef THR_NAME_MAXLEN126127static _store_wbuf *wbuf_new(size_t size) {128 _store_wbuf *b = calloc(1, sizeof(_store_wbuf));129 if (b == NULL)130 return NULL;131 b->buf = calloc(size, sizeof(char));132 if (b->buf == NULL) {133 free(b);134 return NULL;135 }136 b->buf_pos = b->buf;137 b->free = size;138 b->size = size;139 return b;140}141142static store_io_thread *_get_io_thread(store_engine *e) {143 int tid = -1;144 long long int low = LLONG_MAX;145 pthread_mutex_lock(&e->mutex);146 // find smallest queue. ignoring lock since being wrong isn't fatal.147 // TODO: if average queue depth can be quickly tracked, can break as soon148 // as we see a thread that's less than average, and start from last_io_thread149 for (int x = 0; x < e->io_threadcount; x++) {150 if (e->io_threads[x].depth == 0) {151 tid = x;152 break;153 } else if (e->io_threads[x].depth < low) {154 tid = x;155 low = e->io_threads[x].depth;156 }157 }158 pthread_mutex_unlock(&e->mutex);159160 return &e->io_threads[tid];161}162163static uint64_t _next_version(store_engine *e) {164 return e->version++;165}166// internal only method for freeing a page up167static void _free_page(store_engine *e, store_page *p);168169static void *extstore_io_thread(void *arg);170171/* Copies stats internal to engine and computes any derived values */172void extstore_get_stats(void *ptr, struct extstore_stats *st) {173 store_engine *e = (store_engine *)ptr;174 STAT_L(e);175 memcpy(st, &e->stats, sizeof(struct extstore_stats));176 STAT_UL(e);177178 // grab pages_free/pages_used179 pthread_mutex_lock(&e->mutex);180 st->pages_free = e->page_free;181 st->pages_used = e->page_count - e->page_free;182 pthread_mutex_unlock(&e->mutex);183 st->io_queue = 0;184 for (int x = 0; x < e->io_threadcount; x++) {185 pthread_mutex_lock(&e->io_threads[x].mutex);186 st->io_queue += e->io_threads[x].depth;187 pthread_mutex_unlock(&e->io_threads[x].mutex);188 }189 // calculate bytes_fragmented.190 // note that open and yet-filled pages count against fragmentation.191 st->bytes_fragmented = st->pages_used * e->page_size -192 st->bytes_used;193}194195void extstore_get_page_data(void *ptr, struct extstore_stats *st) {196 store_engine *e = (store_engine *)ptr;197 pthread_mutex_lock(&e->maint.mutex);198 struct extstore_page_data *pd = st->page_data;199200 for (int i = 0; i < e->page_count; i++) {201 store_page *p = &e->pages[i];202 pthread_mutex_lock(&p->mutex);203204 pd[p->id].free_bucket = p->free_bucket;205 pd[p->id].version = p->version;206 pd[p->id].bytes_used = p->bytes_used;207 if (p->active) {208 pd[p->id].active = true;209 }210 if (p->active || p->free) {211 pthread_mutex_unlock(&p->mutex);212 continue;213 }214 if (p->obj_count > 0 && !p->closed) {215 pd[p->id].bucket = p->bucket;216 }217 if ((p->obj_count == 0 || p->closed) && p->refcount == 0) {218 _free_page(e, p);219 }220 pthread_mutex_unlock(&p->mutex);221 }222223 pthread_mutex_unlock(&e->maint.mutex);224}225226const char *extstore_err(enum extstore_res res) {227 const char *rv = "unknown error";228 switch (res) {229 case EXTSTORE_INIT_BAD_WBUF_SIZE:230 rv = "page_size must be divisible by wbuf_size";231 break;232 case EXTSTORE_INIT_NEED_MORE_WBUF:233 rv = "wbuf_count must be >= page_buckets";234 break;235 case EXTSTORE_INIT_NEED_MORE_BUCKETS:236 rv = "page_buckets must be > 0";237 break;238 case EXTSTORE_INIT_PAGE_WBUF_ALIGNMENT:239 rv = "page_size and wbuf_size must be divisible by 1024*1024*2";240 break;241 case EXTSTORE_INIT_TOO_MANY_PAGES:242 rv = "page_count must total to < 65536. Increase page_size or lower path sizes";243 break;244 case EXTSTORE_INIT_OOM:245 rv = "failed calloc for engine";246 break;247 case EXTSTORE_INIT_OPEN_FAIL:248 rv = "failed to open file";249 break;250 case EXTSTORE_INIT_THREAD_FAIL:251 break;252 }253 return rv;254}255256// TODO: #define's for DEFAULT_BUCKET, FREE_VERSION, etc257void *extstore_init(struct extstore_conf_file *fh, struct extstore_conf *cf,258 enum extstore_res *res) {259 int i;260 struct extstore_conf_file *f = NULL;261 pthread_t thread;262263 if (cf->page_size % cf->wbuf_size != 0) {264 *res = EXTSTORE_INIT_BAD_WBUF_SIZE;265 return NULL;266 }267 // Should ensure at least one write buffer per potential page268 if (cf->page_buckets > cf->wbuf_count) {269 *res = EXTSTORE_INIT_NEED_MORE_WBUF;270 return NULL;271 }272 if (cf->page_buckets < 1) {273 *res = EXTSTORE_INIT_NEED_MORE_BUCKETS;274 return NULL;275 }276277 // TODO: More intelligence around alignment of flash erasure block sizes278 if (cf->page_size % (1024 * 1024 * 2) != 0 ||279 cf->wbuf_size % (1024 * 1024 * 2) != 0) {280 *res = EXTSTORE_INIT_PAGE_WBUF_ALIGNMENT;281 return NULL;282 }283284 store_engine *e = calloc(1, sizeof(store_engine));285 if (e == NULL) {286 *res = EXTSTORE_INIT_OOM;287 return NULL;288 }289290 e->page_size = cf->page_size;291 uint64_t temp_page_count = 0;292 for (f = fh; f != NULL; f = f->next) {293 f->fd = open(f->file, O_RDWR | O_CREAT, 0644);294 if (f->fd < 0) {295 *res = EXTSTORE_INIT_OPEN_FAIL;296#ifdef EXTSTORE_DEBUG297 perror("extstore open");298#endif299 free(e);300 return NULL;301 }302 // use an fcntl lock to help avoid double starting.303 struct flock lock;304 lock.l_type = F_WRLCK;305 lock.l_start = 0;306 lock.l_whence = SEEK_SET;307 lock.l_len = 0;308 if (fcntl(f->fd, F_SETLK, &lock) < 0) {309 *res = EXTSTORE_INIT_OPEN_FAIL;310 free(e);311 return NULL;312 }313 if (ftruncate(f->fd, 0) < 0) {314 *res = EXTSTORE_INIT_OPEN_FAIL;315 free(e);316 return NULL;317 }318319 temp_page_count += f->page_count;320 f->offset = 0;321 }322323 if (temp_page_count >= UINT16_MAX) {324 *res = EXTSTORE_INIT_TOO_MANY_PAGES;325 free(e);326 return NULL;327 }328 e->page_count = temp_page_count;329330 e->pages = calloc(e->page_count, sizeof(store_page));331 if (e->pages == NULL) {332 *res = EXTSTORE_INIT_OOM;333 // FIXME: loop-close. make error label334 free(e);335 return NULL;336 }337338 // interleave the pages between devices339 f = NULL; // start at the first device.340 for (i = 0; i < e->page_count; i++) {341 // find next device with available pages342 while (1) {343 // restart the loop344 if (f == NULL || f->next == NULL) {345 f = fh;346 } else {347 f = f->next;348 }349 if (f->page_count) {350 f->page_count--;351 break;352 }353 }354 pthread_mutex_init(&e->pages[i].mutex, NULL);355 e->pages[i].id = i;356 e->pages[i].fd = f->fd;357 e->pages[i].free_bucket = f->free_bucket;358 e->pages[i].offset = f->offset;359 e->pages[i].free = true;360 f->offset += e->page_size;361 }362363 // free page buckets allows the app to organize devices by use case364 e->free_page_buckets = calloc(cf->page_buckets, sizeof(store_page *));365 e->free_page_bucketcount = cf->page_buckets;366367 e->page_free = e->page_count;368 for (i = e->page_count-1; i >= 0; i--) {369 int fb = e->pages[i].free_bucket;370 e->pages[i].next = e->free_page_buckets[fb];371 e->free_page_buckets[fb] = &e->pages[i];372 }373374 // 0 is magic "page is freed" version375 e->version = 1;376377 // scratch data for stats. TODO: malloc failure handle378 e->stats.page_data =379 calloc(e->page_count, sizeof(struct extstore_page_data));380 e->stats.page_count = e->page_count;381 e->stats.page_size = e->page_size;382383 // page buckets lazily have pages assigned into them384 e->page_buckets = calloc(cf->page_buckets, sizeof(store_page *));385 e->page_bucketcount = cf->page_buckets;386387 // allocate write buffers388 // also IO's to use for shipping to IO thread389 for (i = 0; i < cf->wbuf_count; i++) {390 _store_wbuf *w = wbuf_new(cf->wbuf_size);391 obj_io *io = calloc(1, sizeof(obj_io));392 /* TODO: on error, loop again and free stack. */393 w->next = e->wbuf_stack;394 e->wbuf_stack = w;395 io->next = e->io_stack;396 e->io_stack = io;397 }398399 pthread_mutex_init(&e->mutex, NULL);400 pthread_mutex_init(&e->stats_mutex, NULL);401 pthread_mutex_init(&e->maint.mutex, NULL);402403 e->io_depth = cf->io_depth;404405 // spawn threads406 e->io_threads = calloc(cf->io_threadcount, sizeof(store_io_thread));407 for (i = 0; i < cf->io_threadcount; i++) {408 pthread_mutex_init(&e->io_threads[i].mutex, NULL);409 pthread_cond_init(&e->io_threads[i].cond, NULL);410 e->io_threads[i].e = e;411 // FIXME: error handling412 pthread_create(&thread, NULL, extstore_io_thread, &e->io_threads[i]);413 thread_setname(thread, "mc-ext-io");414 }415 e->io_threadcount = cf->io_threadcount;416417 // dedicated IO thread for certain non-hotpath functions.418 e->bg_thread = calloc(1, sizeof(store_io_thread));419 e->bg_thread->e = e;420 pthread_mutex_init(&e->bg_thread->mutex, NULL);421 pthread_cond_init(&e->bg_thread->cond, NULL);422 pthread_create(&thread, NULL, extstore_io_thread, e->bg_thread);423 thread_setname(thread, "mc-ext-bgio");424425 return (void *)e;426}427428// Call without *e locked, not a fast function.429static void _evict_page(store_engine *e, unsigned int bucket,430 unsigned int free_bucket) {431 struct extstore_stats st;432 st.page_data = calloc(e->page_count, sizeof(struct extstore_page_data));433 extstore_get_page_data(e, &st);434 uint64_t low_version = ULLONG_MAX;435 unsigned int low_page = 0;436437 // find lowest version of anything in free_bucket OR 0438 // unless free_bucket is 0439 for (int i = 0; i < e->page_count; i++) {440 // must belong to 0 or the requested free_bucket441 if (st.page_data[i].free_bucket &&442 st.page_data[i].free_bucket != free_bucket) {443 continue;444 }445446 // found a free page, don't evict.447 if (st.page_data[i].version == 0) {448 low_version = ULLONG_MAX;449 break;450 }451452 // find the lowest version.453 if (!st.page_data[i].active &&454 st.page_data[i].version < low_version) {455 low_page = i;456 low_version = st.page_data[i].version;457 }458 }459460 if (low_version != ULLONG_MAX) {461 extstore_evict_page(e, low_page, low_version);462 }463 free(st.page_data);464}465466// call with *e locked467static store_page *_allocate_page(store_engine *e, unsigned int bucket,468 unsigned int free_bucket) {469 E_DEBUG("EXTSTORE: allocating new page [bucket:%u]\n", bucket);470 assert(!e->page_buckets[bucket] || e->page_buckets[bucket]->allocated == e->page_size);471 store_page *tmp = NULL;472 if (e->free_page_buckets[free_bucket] != NULL) {473 assert(e->page_free > 0);474 tmp = e->free_page_buckets[free_bucket];475 e->free_page_buckets[free_bucket] = tmp->next;476 } else if (e->free_page_buckets[0] != NULL) {477 // fall back to default bucket.478 assert(e->page_free > 0);479 tmp = e->free_page_buckets[0];480 e->free_page_buckets[0] = tmp->next;481 }482 if (tmp != NULL) {483 tmp->next = e->page_buckets[bucket];484 e->page_buckets[bucket] = tmp;485 tmp->active = true;486 tmp->free = false;487 tmp->closed = false;488 tmp->version = _next_version(e);489 tmp->bucket = bucket;490 e->page_free--;491 STAT_INCR(e, page_allocs, 1);492 }493494 if (tmp)495 E_DEBUG("EXTSTORE: got page %u [free:%u]\n", tmp->id, e->page_free);496 return tmp;497}498499// call with *p locked. locks *e500static void _allocate_wbuf(store_engine *e, store_page *p) {501 _store_wbuf *wbuf = NULL;502 assert(!p->wbuf);503 pthread_mutex_lock(&e->mutex);504 if (e->wbuf_stack) {505 wbuf = e->wbuf_stack;506 e->wbuf_stack = wbuf->next;507 wbuf->next = 0;508 }509 pthread_mutex_unlock(&e->mutex);510 if (wbuf) {511 wbuf->offset = p->allocated;512 p->allocated += wbuf->size;513 wbuf->free = wbuf->size;514 wbuf->buf_pos = wbuf->buf;515 wbuf->full = false;516 wbuf->flushed = false;517518 p->wbuf = wbuf;519 }520}521522/* callback after wbuf is flushed. can only remove wbuf's from the head onward523 * if successfully flushed, which complicates this routine. each callback524 * attempts to free the wbuf stack, which is finally done when the head wbuf's525 * callback happens.526 * It's rare flushes would happen out of order.527 */528static void _wbuf_cb(void *ep, obj_io *io, int ret) {529 store_engine *e = (store_engine *)ep;530 store_page *p = &e->pages[io->page_id];531 _store_wbuf *w = (_store_wbuf *) io->data;532533 // TODO: Examine return code. Not entirely sure how to handle errors.534 // Naive first-pass should probably cause the page to close/free.535 w->flushed = true;536 pthread_mutex_lock(&p->mutex);537 assert(p->wbuf != NULL && p->wbuf == w);538 assert(p->written == w->offset);539 p->written += w->size;540 p->wbuf = NULL;541542 if (p->written == e->page_size)543 p->active = false;544545 // return the wbuf546 pthread_mutex_lock(&e->mutex);547 w->next = e->wbuf_stack;548 e->wbuf_stack = w;549 // also return the IO we just used.550 io->next = e->io_stack;551 e->io_stack = io;552 pthread_mutex_unlock(&e->mutex);553 pthread_mutex_unlock(&p->mutex);554}555556/* Wraps pages current wbuf in an io and submits to IO thread.557 * Called with p locked, locks e.558 */559static void _submit_wbuf(store_engine *e, store_page *p) {560 _store_wbuf *w;561 pthread_mutex_lock(&e->mutex);562 obj_io *io = e->io_stack;563 e->io_stack = io->next;564 pthread_mutex_unlock(&e->mutex);565 w = p->wbuf;566567 // zero out the end of the wbuf to allow blind readback of data.568 memset(w->buf + (w->size - w->free), 0, w->free);569570 io->next = NULL;571 io->mode = OBJ_IO_WRITE;572 io->page_id = p->id;573 io->data = w;574 io->offset = w->offset;575 io->len = w->size;576 io->buf = w->buf;577 io->cb = _wbuf_cb;578579 extstore_submit_bg(e, io);580}581582/* engine write function; takes engine, item_io.583 * fast fail if no available write buffer (flushing)584 * lock engine context, find active page, unlock585 * if page full, submit page/buffer to io thread.586 *587 * write is designed to be flaky; if page full, caller must try again to get588 * new page. best if used from a background thread that can harmlessly retry.589 */590591int extstore_write_request(void *ptr, unsigned int bucket,592 unsigned int free_bucket, obj_io *io) {593 store_engine *e = (store_engine *)ptr;594 store_page *p;595 int ret = -1;596 if (bucket >= e->page_bucketcount)597 return ret;598599 pthread_mutex_lock(&e->mutex);600 p = e->page_buckets[bucket];601 if (!p) {602 p = _allocate_page(e, bucket, free_bucket);603 }604 pthread_mutex_unlock(&e->mutex);605 if (!p) {606 _evict_page(e, bucket, free_bucket);607 return ret;608 }609610 pthread_mutex_lock(&p->mutex);611612 // FIXME: can't null out page_buckets!!!613 // page is full, clear bucket and retry later.614 if (!p->active ||615 ((!p->wbuf || p->wbuf->full) && p->allocated >= e->page_size)) {616 pthread_mutex_unlock(&p->mutex);617 pthread_mutex_lock(&e->mutex);618 store_page *temp_p = _allocate_page(e, bucket, free_bucket);619 pthread_mutex_unlock(&e->mutex);620 if (!temp_p) {621 _evict_page(e, bucket, free_bucket);622 }623 return ret;624 }625626 // if io won't fit, submit IO for wbuf and find new one.627 if (p->wbuf && p->wbuf->free < io->len && !p->wbuf->full) {628 _submit_wbuf(e, p);629 p->wbuf->full = true;630 }631632 if (!p->wbuf && p->allocated < e->page_size) {633 _allocate_wbuf(e, p);634 }635636 // hand over buffer for caller to copy into637 // leaves p locked.638 if (p->wbuf && !p->wbuf->full && p->wbuf->free >= io->len) {639 io->buf = p->wbuf->buf_pos;640 io->page_id = p->id;641 return 0;642 }643644 pthread_mutex_unlock(&p->mutex);645 // p->written is incremented post-wbuf flush646 return ret;647}648649/* _must_ be called after a successful write_request.650 * fills the rest of io structure.651 */652void extstore_write(void *ptr, obj_io *io) {653 store_engine *e = (store_engine *)ptr;654 store_page *p = &e->pages[io->page_id];655656 io->offset = p->wbuf->offset + (p->wbuf->size - p->wbuf->free);657 io->page_version = p->version;658 p->wbuf->buf_pos += io->len;659 p->wbuf->free -= io->len;660 p->bytes_used += io->len;661 p->obj_count++;662 STAT_L(e);663 e->stats.bytes_written += io->len;664 e->stats.bytes_used += io->len;665 e->stats.objects_written++;666 e->stats.objects_used++;667 STAT_UL(e);668669 pthread_mutex_unlock(&p->mutex);670}671672/* engine submit function; takes engine, item_io stack.673 * lock io_thread context and add stack674 * signal io thread to wake.675 * return success.676 */677static int _extstore_submit(void *ptr, obj_io *io, store_io_thread *t) {678 unsigned int depth = 0;679 obj_io *tio = io;680 obj_io *tail = NULL;681 while (tio != NULL) {682 tail = tio; // keep updating potential tail.683 depth++;684 tio = tio->next;685 }686687 pthread_mutex_lock(&t->mutex);688689 t->depth += depth;690 if (t->queue == NULL) {691 t->queue = io;692 t->queue_tail = tail;693 } else {694 // Have to put the *io stack at the end of current queue.695 assert(tail->next == NULL);696 assert(t->queue_tail->next == NULL);697 t->queue_tail->next = io;698 t->queue_tail = tail;699 }700701 pthread_mutex_unlock(&t->mutex);702703 //pthread_mutex_lock(&t->mutex);704 pthread_cond_signal(&t->cond);705 //pthread_mutex_unlock(&t->mutex);706 return 0;707}708709int extstore_submit(void *ptr, obj_io *io) {710 store_engine *e = (store_engine *)ptr;711 store_io_thread *t = _get_io_thread(e);712 return _extstore_submit(ptr, io, t);713}714715int extstore_submit_bg(void *ptr, obj_io *io) {716 store_engine *e = (store_engine *)ptr;717 store_io_thread *t = e->bg_thread;718 return _extstore_submit(ptr, io, t);719}720721/* engine note delete function: takes engine, page id, size?722 * note that an item in this page is no longer valid723 */724int extstore_delete(void *ptr, unsigned int page_id, uint64_t page_version,725 unsigned int count, unsigned int bytes) {726 store_engine *e = (store_engine *)ptr;727 // FIXME: validate page_id in bounds728 store_page *p = &e->pages[page_id];729 int ret = 0;730731 pthread_mutex_lock(&p->mutex);732 if (!p->closed && p->version == page_version) {733 if (p->bytes_used >= bytes) {734 p->bytes_used -= bytes;735 } else {736 p->bytes_used = 0;737 }738739 if (p->obj_count >= count) {740 p->obj_count -= count;741 } else {742 p->obj_count = 0; // caller has bad accounting?743 }744 STAT_L(e);745 e->stats.bytes_used -= bytes;746 e->stats.objects_used -= count;747 STAT_UL(e);748749 if (p->obj_count == 0 && p->refcount == 0 && !p->active) {750 _free_page(e, p);751 }752 } else {753 ret = -1;754 }755 pthread_mutex_unlock(&p->mutex);756 return ret;757}758759int extstore_check(void *ptr, unsigned int page_id, uint64_t page_version) {760 store_engine *e = (store_engine *)ptr;761 store_page *p = &e->pages[page_id];762 int ret = 0;763764 pthread_mutex_lock(&p->mutex);765 if (p->version != page_version)766 ret = -1;767 pthread_mutex_unlock(&p->mutex);768 return ret;769}770771/* allows a compactor to say "we're done with this page, kill it." */772void extstore_close_page(void *ptr, unsigned int page_id, uint64_t page_version) {773 store_engine *e = (store_engine *)ptr;774 store_page *p = &e->pages[page_id];775776 pthread_mutex_lock(&p->mutex);777 if (!p->closed && !p->active && p->version == page_version) {778 p->closed = true;779 if (p->refcount == 0) {780 _free_page(e, p);781 }782 }783 pthread_mutex_unlock(&p->mutex);784}785786/* signal that we've forcefully ejected rather than gracefully closed */787void extstore_evict_page(void *ptr, unsigned int page_id, uint64_t page_version) {788 store_engine *e = (store_engine *)ptr;789 store_page *p = &e->pages[page_id];790791 pthread_mutex_lock(&p->mutex);792 if (!p->closed && !p->active && p->version == page_version) {793 E_DEBUG("EXTSTORE: evicting page [%d] [v: %llu]\n",794 p->id, (unsigned long long) p->version);795796 p->closed = true;797 STAT_L(e);798 e->stats.page_evictions++;799 e->stats.objects_evicted += p->obj_count;800 e->stats.bytes_evicted += p->bytes_used;801 STAT_UL(e);802 if (p->refcount == 0) {803 _free_page(e, p);804 }805 }806 pthread_mutex_unlock(&p->mutex);807}808809/* Finds an attached wbuf that can satisfy the read.810 * Since wbufs can potentially be flushed to disk out of order, they are only811 * removed as the head of the list successfully flushes to disk.812 */813// call with *p locked814// FIXME: protect from reading past wbuf815static inline int _read_from_wbuf(store_page *p, obj_io *io) {816 _store_wbuf *wbuf = p->wbuf;817 assert(wbuf != NULL);818 assert(io->offset < p->written + wbuf->size);819 if (io->iov == NULL) {820 memcpy(io->buf, wbuf->buf + (io->offset - wbuf->offset), io->len);821 } else {822 int x;823 unsigned int off = io->offset - wbuf->offset;824 // need to loop fill iovecs825 for (x = 0; x < io->iovcnt; x++) {826 struct iovec *iov = &io->iov[x];827 memcpy(iov->iov_base, wbuf->buf + off, iov->iov_len);828 off += iov->iov_len;829 }830 }831 return io->len;832}833834/* engine IO thread; takes engine context835 * manage writes/reads836 * runs IO callbacks inline after each IO837 */838// FIXME: protect from reading past page839static void *extstore_io_thread(void *arg) {840 store_io_thread *me = (store_io_thread *)arg;841 store_engine *e = me->e;842 while (1) {843 obj_io *io_stack = NULL;844 pthread_mutex_lock(&me->mutex);845 if (me->queue == NULL) {846 pthread_cond_wait(&me->cond, &me->mutex);847 }848849 // Pull and disconnect a batch from the queue850 // Chew small batches from the queue so the IO thread picker can keep851 // the IO queue depth even, instead of piling on threads one at a time852 // as they gobble a queue.853 if (me->queue != NULL) {854 int i;855 obj_io *end = NULL;856 io_stack = me->queue;857 end = io_stack;858 for (i = 1; i < e->io_depth; i++) {859 if (end->next) {860 end = end->next;861 } else {862 me->queue_tail = end->next;863 break;864 }865 }866 me->depth -= i;867 me->queue = end->next;868 end->next = NULL;869 }870 pthread_mutex_unlock(&me->mutex);871872 obj_io *cur_io = io_stack;873 while (cur_io) {874 // We need to note next before the callback in case the obj_io875 // gets reused.876 obj_io *next = cur_io->next;877 int ret = 0;878 int do_op = 1;879 store_page *p = &e->pages[cur_io->page_id];880 // TODO: loop if not enough bytes were read/written.881 switch (cur_io->mode) {882 case OBJ_IO_READ:883 // Page is currently open. deal if read is past the end.884 pthread_mutex_lock(&p->mutex);885 if (!p->free && !p->closed && p->version == cur_io->page_version) {886 if (p->active && cur_io->offset >= p->written) {887 ret = _read_from_wbuf(p, cur_io);888 do_op = 0;889 } else {890 p->refcount++;891 }892 STAT_L(e);893 e->stats.bytes_read += cur_io->len;894 e->stats.objects_read++;895 STAT_UL(e);896 } else {897 do_op = 0;898 ret = -2; // TODO: enum in IO for status?899 }900 pthread_mutex_unlock(&p->mutex);901 if (do_op) {902#if !defined(HAVE_PREAD) || !defined(HAVE_PREADV)903 // TODO: lseek offset is natively 64-bit on OS X, but904 // perhaps not on all platforms? Else use lseek64()905 ret = lseek(p->fd, p->offset + cur_io->offset, SEEK_SET);906 if (ret >= 0) {907 if (cur_io->iov == NULL) {908 ret = read(p->fd, cur_io->buf, cur_io->len);909 } else {910 ret = readv(p->fd, cur_io->iov, cur_io->iovcnt);911 }912 }913#else914 if (cur_io->iov == NULL) {915 ret = pread(p->fd, cur_io->buf, cur_io->len, p->offset + cur_io->offset);916 } else {917 ret = preadv(p->fd, cur_io->iov, cur_io->iovcnt, p->offset + cur_io->offset);918 }919#endif920 }921 break;922 case OBJ_IO_WRITE:923 do_op = 0;924 // FIXME: Should hold refcount during write. doesn't925 // currently matter since page can't free while active.926 ret = pwrite(p->fd, cur_io->buf, cur_io->len, p->offset + cur_io->offset);927 break;928 }929 if (ret == 0) {930 E_DEBUG("read returned nothing\n");931 }932933#ifdef EXTSTORE_DEBUG934 if (ret == -1) {935 perror("read/write op failed");936 }937#endif938 cur_io->cb(e, cur_io, ret);939 if (do_op) {940 pthread_mutex_lock(&p->mutex);941 p->refcount--;942 pthread_mutex_unlock(&p->mutex);943 }944 cur_io = next;945 }946 }947948 return NULL;949}950951// call with *p locked.952static void _free_page(store_engine *e, store_page *p) {953 store_page *tmp = NULL;954 store_page *prev = NULL;955 E_DEBUG("EXTSTORE: freeing page %u\n", p->id);956 STAT_L(e);957 e->stats.objects_used -= p->obj_count;958 e->stats.bytes_used -= p->bytes_used;959 e->stats.page_reclaims++;960 STAT_UL(e);961 pthread_mutex_lock(&e->mutex);962 // unlink page from bucket list963 tmp = e->page_buckets[p->bucket];964 while (tmp) {965 if (tmp == p) {966 if (prev) {967 prev->next = tmp->next;968 } else {969 e->page_buckets[p->bucket] = tmp->next;970 }971 tmp->next = NULL;972 break;973 }974 prev = tmp;975 tmp = tmp->next;976 }977 // reset most values978 p->version = 0;979 p->obj_count = 0;980 p->bytes_used = 0;981 p->allocated = 0;982 p->written = 0;983 p->bucket = 0;984 p->active = false;985 p->closed = false;986 p->free = true;987 // add to page stack988 p->next = e->free_page_buckets[p->free_bucket];989 e->free_page_buckets[p->free_bucket] = p;990 e->page_free++;991 E_DEBUG("EXTSTORE: pages free %u\n", e->page_free);992 pthread_mutex_unlock(&e->mutex);993}
Same data, no extra tab — call code_get_file + code_get_findings over MCP from Claude/Cursor/Copilot.
Generated by searchcode.com · powered by code_get_file + code_get_findings