|
Packit Service |
584ef9 |
/* Copyright 2016 Netflix.
|
|
Packit Service |
584ef9 |
*
|
|
Packit Service |
584ef9 |
* Use and distribution licensed under the BSD license. See
|
|
Packit Service |
584ef9 |
* the LICENSE file for full text.
|
|
Packit Service |
584ef9 |
*/
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
|
|
Packit Service |
584ef9 |
#include "memcached.h"
|
|
Packit Service |
584ef9 |
#include <sys/stat.h>
|
|
Packit Service |
584ef9 |
#include <sys/socket.h>
|
|
Packit Service |
584ef9 |
#include <sys/resource.h>
|
|
Packit Service |
584ef9 |
#include <fcntl.h>
|
|
Packit Service |
584ef9 |
#include <netinet/in.h>
|
|
Packit Service |
584ef9 |
#include <errno.h>
|
|
Packit Service |
584ef9 |
#include <stdlib.h>
|
|
Packit Service |
584ef9 |
#include <stdio.h>
|
|
Packit Service |
584ef9 |
#include <signal.h>
|
|
Packit Service |
584ef9 |
#include <string.h>
|
|
Packit Service |
584ef9 |
#include <time.h>
|
|
Packit Service |
584ef9 |
#include <assert.h>
|
|
Packit Service |
584ef9 |
#include <unistd.h>
|
|
Packit Service |
584ef9 |
#include <poll.h>
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
#define LARGEST_ID POWER_LARGEST
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
typedef struct {
|
|
Packit Service |
584ef9 |
void *c; /* original connection structure. still with source thread attached. */
|
|
Packit Service |
584ef9 |
int sfd; /* client fd. */
|
|
Packit Service |
584ef9 |
bipbuf_t *buf; /* output buffer */
|
|
Packit Service |
584ef9 |
char *cbuf; /* current buffer */
|
|
Packit Service |
584ef9 |
} crawler_client_t;
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
typedef struct _crawler_module_t crawler_module_t;
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
typedef void (*crawler_eval_func)(crawler_module_t *cm, item *it, uint32_t hv, int slab_cls);
|
|
Packit Service |
584ef9 |
typedef int (*crawler_init_func)(crawler_module_t *cm, void *data); // TODO: init args?
|
|
Packit Service |
584ef9 |
typedef void (*crawler_deinit_func)(crawler_module_t *cm); // TODO: extra args?
|
|
Packit Service |
584ef9 |
typedef void (*crawler_doneclass_func)(crawler_module_t *cm, int slab_cls);
|
|
Packit Service |
584ef9 |
typedef void (*crawler_finalize_func)(crawler_module_t *cm);
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
typedef struct {
|
|
Packit Service |
584ef9 |
crawler_init_func init; /* run before crawl starts */
|
|
Packit Service |
584ef9 |
crawler_eval_func eval; /* runs on an item. */
|
|
Packit Service |
584ef9 |
crawler_doneclass_func doneclass; /* runs once per sub-crawler completion. */
|
|
Packit Service |
584ef9 |
crawler_finalize_func finalize; /* runs once when all sub-crawlers are done. */
|
|
Packit Service |
584ef9 |
bool needs_lock; /* whether or not we need the LRU lock held when eval is called */
|
|
Packit Service |
584ef9 |
bool needs_client; /* whether or not to grab onto the remote client */
|
|
Packit Service |
584ef9 |
} crawler_module_reg_t;
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
struct _crawler_module_t {
|
|
Packit Service |
584ef9 |
void *data; /* opaque data pointer */
|
|
Packit Service |
584ef9 |
crawler_client_t c;
|
|
Packit Service |
584ef9 |
crawler_module_reg_t *mod;
|
|
Packit Service |
584ef9 |
};
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
static int crawler_expired_init(crawler_module_t *cm, void *data);
|
|
Packit Service |
584ef9 |
static void crawler_expired_doneclass(crawler_module_t *cm, int slab_cls);
|
|
Packit Service |
584ef9 |
static void crawler_expired_finalize(crawler_module_t *cm);
|
|
Packit Service |
584ef9 |
static void crawler_expired_eval(crawler_module_t *cm, item *search, uint32_t hv, int i);
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
crawler_module_reg_t crawler_expired_mod = {
|
|
Packit Service |
584ef9 |
.init = crawler_expired_init,
|
|
Packit Service |
584ef9 |
.eval = crawler_expired_eval,
|
|
Packit Service |
584ef9 |
.doneclass = crawler_expired_doneclass,
|
|
Packit Service |
584ef9 |
.finalize = crawler_expired_finalize,
|
|
Packit Service |
584ef9 |
.needs_lock = true,
|
|
Packit Service |
584ef9 |
.needs_client = false
|
|
Packit Service |
584ef9 |
};
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
static void crawler_metadump_eval(crawler_module_t *cm, item *search, uint32_t hv, int i);
|
|
Packit Service |
584ef9 |
static void crawler_metadump_finalize(crawler_module_t *cm);
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
crawler_module_reg_t crawler_metadump_mod = {
|
|
Packit Service |
584ef9 |
.init = NULL,
|
|
Packit Service |
584ef9 |
.eval = crawler_metadump_eval,
|
|
Packit Service |
584ef9 |
.doneclass = NULL,
|
|
Packit Service |
584ef9 |
.finalize = crawler_metadump_finalize,
|
|
Packit Service |
584ef9 |
.needs_lock = false,
|
|
Packit Service |
584ef9 |
.needs_client = true
|
|
Packit Service |
584ef9 |
};
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
crawler_module_reg_t *crawler_mod_regs[3] = {
|
|
Packit Service |
584ef9 |
&crawler_expired_mod,
|
|
Packit Service |
584ef9 |
&crawler_expired_mod,
|
|
Packit Service |
584ef9 |
&crawler_metadump_mod
|
|
Packit Service |
584ef9 |
};
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
static int lru_crawler_client_getbuf(crawler_client_t *c);
|
|
Packit Service |
584ef9 |
crawler_module_t active_crawler_mod;
|
|
Packit Service |
584ef9 |
enum crawler_run_type active_crawler_type;
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
static crawler crawlers[LARGEST_ID];
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
static int crawler_count = 0;
|
|
Packit Service |
584ef9 |
static volatile int do_run_lru_crawler_thread = 0;
|
|
Packit Service |
584ef9 |
static int lru_crawler_initialized = 0;
|
|
Packit Service |
584ef9 |
static pthread_mutex_t lru_crawler_lock = PTHREAD_MUTEX_INITIALIZER;
|
|
Packit Service |
584ef9 |
static pthread_cond_t lru_crawler_cond = PTHREAD_COND_INITIALIZER;
|
|
Packit Service |
584ef9 |
#ifdef EXTSTORE
|
|
Packit Service |
584ef9 |
/* TODO: pass this around */
|
|
Packit Service |
584ef9 |
static void *storage;
|
|
Packit Service |
584ef9 |
#endif
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
/* Will crawl all slab classes a minimum of once per hour */
|
|
Packit Service |
584ef9 |
#define MAX_MAINTCRAWL_WAIT 60 * 60
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
/*** LRU CRAWLER THREAD ***/
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
#define LRU_CRAWLER_WRITEBUF 8192
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
static void lru_crawler_close_client(crawler_client_t *c) {
|
|
Packit Service |
584ef9 |
//fprintf(stderr, "CRAWLER: Closing client\n");
|
|
Packit Service |
584ef9 |
sidethread_conn_close(c->c);
|
|
Packit Service |
584ef9 |
c->c = NULL;
|
|
Packit Service |
584ef9 |
c->cbuf = NULL;
|
|
Packit Service |
584ef9 |
bipbuf_free(c->buf);
|
|
Packit Service |
584ef9 |
c->buf = NULL;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
static void lru_crawler_release_client(crawler_client_t *c) {
|
|
Packit Service |
584ef9 |
//fprintf(stderr, "CRAWLER: Closing client\n");
|
|
Packit Service |
584ef9 |
redispatch_conn(c->c);
|
|
Packit Service |
584ef9 |
c->c = NULL;
|
|
Packit Service |
584ef9 |
c->cbuf = NULL;
|
|
Packit Service |
584ef9 |
bipbuf_free(c->buf);
|
|
Packit Service |
584ef9 |
c->buf = NULL;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
static int crawler_expired_init(crawler_module_t *cm, void *data) {
|
|
Packit Service |
584ef9 |
struct crawler_expired_data *d;
|
|
Packit Service |
584ef9 |
if (data != NULL) {
|
|
Packit Service |
584ef9 |
d = data;
|
|
Packit Service |
584ef9 |
d->is_external = true;
|
|
Packit Service |
584ef9 |
cm->data = data;
|
|
Packit Service |
584ef9 |
} else {
|
|
Packit Service |
584ef9 |
// allocate data.
|
|
Packit Service |
584ef9 |
d = calloc(1, sizeof(struct crawler_expired_data));
|
|
Packit Service |
584ef9 |
if (d == NULL) {
|
|
Packit Service |
584ef9 |
return -1;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
// init lock.
|
|
Packit Service |
584ef9 |
pthread_mutex_init(&d->lock, NULL);
|
|
Packit Service |
584ef9 |
d->is_external = false;
|
|
Packit Service |
584ef9 |
d->start_time = current_time;
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
cm->data = d;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
pthread_mutex_lock(&d->lock);
|
|
Packit Service |
584ef9 |
memset(&d->crawlerstats, 0, sizeof(crawlerstats_t) * POWER_LARGEST);
|
|
Packit Service |
584ef9 |
for (int x = 0; x < POWER_LARGEST; x++) {
|
|
Packit Service |
584ef9 |
d->crawlerstats[x].start_time = current_time;
|
|
Packit Service |
584ef9 |
d->crawlerstats[x].run_complete = false;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&d->lock);
|
|
Packit Service |
584ef9 |
return 0;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
static void crawler_expired_doneclass(crawler_module_t *cm, int slab_cls) {
|
|
Packit Service |
584ef9 |
struct crawler_expired_data *d = (struct crawler_expired_data *) cm->data;
|
|
Packit Service |
584ef9 |
pthread_mutex_lock(&d->lock);
|
|
Packit Service |
584ef9 |
d->crawlerstats[slab_cls].end_time = current_time;
|
|
Packit Service |
584ef9 |
d->crawlerstats[slab_cls].run_complete = true;
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&d->lock);
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
static void crawler_expired_finalize(crawler_module_t *cm) {
|
|
Packit Service |
584ef9 |
struct crawler_expired_data *d = (struct crawler_expired_data *) cm->data;
|
|
Packit Service |
584ef9 |
pthread_mutex_lock(&d->lock);
|
|
Packit Service |
584ef9 |
d->end_time = current_time;
|
|
Packit Service |
584ef9 |
d->crawl_complete = true;
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&d->lock);
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
if (!d->is_external) {
|
|
Packit Service |
584ef9 |
free(d);
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
/* I pulled this out to make the main thread clearer, but it reaches into the
|
|
Packit Service |
584ef9 |
* main thread's values too much. Should rethink again.
|
|
Packit Service |
584ef9 |
*/
|
|
Packit Service |
584ef9 |
static void crawler_expired_eval(crawler_module_t *cm, item *search, uint32_t hv, int i) {
|
|
Packit Service |
584ef9 |
struct crawler_expired_data *d = (struct crawler_expired_data *) cm->data;
|
|
Packit Service |
584ef9 |
pthread_mutex_lock(&d->lock);
|
|
Packit Service |
584ef9 |
crawlerstats_t *s = &d->crawlerstats[i];
|
|
Packit Service |
584ef9 |
int is_flushed = item_is_flushed(search);
|
|
Packit Service |
584ef9 |
#ifdef EXTSTORE
|
|
Packit Service |
584ef9 |
bool is_valid = true;
|
|
Packit Service |
584ef9 |
if (search->it_flags & ITEM_HDR) {
|
|
Packit Service |
584ef9 |
item_hdr *hdr = (item_hdr *)ITEM_data(search);
|
|
Packit Service |
584ef9 |
if (extstore_check(storage, hdr->page_id, hdr->page_version) != 0)
|
|
Packit Service |
584ef9 |
is_valid = false;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
#endif
|
|
Packit Service |
584ef9 |
if ((search->exptime != 0 && search->exptime < current_time)
|
|
Packit Service |
584ef9 |
|| is_flushed
|
|
Packit Service |
584ef9 |
#ifdef EXTSTORE
|
|
Packit Service |
584ef9 |
|| !is_valid
|
|
Packit Service |
584ef9 |
#endif
|
|
Packit Service |
584ef9 |
) {
|
|
Packit Service |
584ef9 |
crawlers[i].reclaimed++;
|
|
Packit Service |
584ef9 |
s->reclaimed++;
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
if (settings.verbose > 1) {
|
|
Packit Service |
584ef9 |
int ii;
|
|
Packit Service |
584ef9 |
char *key = ITEM_key(search);
|
|
Packit Service |
584ef9 |
fprintf(stderr, "LRU crawler found an expired item (flags: %d, slab: %d): ",
|
|
Packit Service |
584ef9 |
search->it_flags, search->slabs_clsid);
|
|
Packit Service |
584ef9 |
for (ii = 0; ii < search->nkey; ++ii) {
|
|
Packit Service |
584ef9 |
fprintf(stderr, "%c", key[ii]);
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
fprintf(stderr, "\n");
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
if ((search->it_flags & ITEM_FETCHED) == 0 && !is_flushed) {
|
|
Packit Service |
584ef9 |
crawlers[i].unfetched++;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
#ifdef EXTSTORE
|
|
Packit Service |
584ef9 |
STORAGE_delete(storage, search);
|
|
Packit Service |
584ef9 |
#endif
|
|
Packit Service |
584ef9 |
do_item_unlink_nolock(search, hv);
|
|
Packit Service |
584ef9 |
do_item_remove(search);
|
|
Packit Service |
584ef9 |
} else {
|
|
Packit Service |
584ef9 |
s->seen++;
|
|
Packit Service |
584ef9 |
refcount_decr(search);
|
|
Packit Service |
584ef9 |
if (search->exptime == 0) {
|
|
Packit Service |
584ef9 |
s->noexp++;
|
|
Packit Service |
584ef9 |
} else if (search->exptime - current_time > 3599) {
|
|
Packit Service |
584ef9 |
s->ttl_hourplus++;
|
|
Packit Service |
584ef9 |
} else {
|
|
Packit Service |
584ef9 |
rel_time_t ttl_remain = search->exptime - current_time;
|
|
Packit Service |
584ef9 |
int bucket = ttl_remain / 60;
|
|
Packit Service |
584ef9 |
if (bucket <= 60) {
|
|
Packit Service |
584ef9 |
s->histo[bucket]++;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&d->lock);
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
static void crawler_metadump_eval(crawler_module_t *cm, item *it, uint32_t hv, int i) {
|
|
Packit Service |
584ef9 |
//int slab_id = CLEAR_LRU(i);
|
|
Packit Service |
584ef9 |
char keybuf[KEY_MAX_URI_ENCODED_LENGTH];
|
|
Packit Service |
584ef9 |
int is_flushed = item_is_flushed(it);
|
|
Packit Service |
584ef9 |
/* Ignore expired content. */
|
|
Packit Service |
584ef9 |
if ((it->exptime != 0 && it->exptime < current_time)
|
|
Packit Service |
584ef9 |
|| is_flushed) {
|
|
Packit Service |
584ef9 |
refcount_decr(it);
|
|
Packit Service |
584ef9 |
return;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
// TODO: uriencode directly into the buffer.
|
|
Packit Service |
584ef9 |
uriencode(ITEM_key(it), keybuf, it->nkey, KEY_MAX_URI_ENCODED_LENGTH);
|
|
Packit Service |
584ef9 |
int total = snprintf(cm->c.cbuf, 4096,
|
|
Packit Service |
584ef9 |
"key=%s exp=%ld la=%llu cas=%llu fetch=%s cls=%u size=%lu\n",
|
|
Packit Service |
584ef9 |
keybuf,
|
|
Packit Service |
584ef9 |
(it->exptime == 0) ? -1 : (long)(it->exptime + process_started),
|
|
Packit Service |
584ef9 |
(unsigned long long)(it->time + process_started),
|
|
Packit Service |
584ef9 |
(unsigned long long)ITEM_get_cas(it),
|
|
Packit Service |
584ef9 |
(it->it_flags & ITEM_FETCHED) ? "yes" : "no",
|
|
Packit Service |
584ef9 |
ITEM_clsid(it),
|
|
Packit Service |
584ef9 |
(unsigned long) ITEM_ntotal(it));
|
|
Packit Service |
584ef9 |
refcount_decr(it);
|
|
Packit Service |
584ef9 |
// TODO: some way of tracking the errors. these are very unlikely though.
|
|
Packit Service |
584ef9 |
if (total >= LRU_CRAWLER_WRITEBUF - 1 || total <= 0) {
|
|
Packit Service |
584ef9 |
/* Failed to write, don't push it. */
|
|
Packit Service |
584ef9 |
return;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
bipbuf_push(cm->c.buf, total);
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
static void crawler_metadump_finalize(crawler_module_t *cm) {
|
|
Packit Service |
584ef9 |
if (cm->c.c != NULL) {
|
|
Packit Service |
584ef9 |
// Ensure space for final message.
|
|
Packit Service |
584ef9 |
lru_crawler_client_getbuf(&cm->c);
|
|
Packit Service |
584ef9 |
memcpy(cm->c.cbuf, "END\r\n", 5);
|
|
Packit Service |
584ef9 |
bipbuf_push(cm->c.buf, 5);
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
static int lru_crawler_poll(crawler_client_t *c) {
|
|
Packit Service |
584ef9 |
unsigned char *data;
|
|
Packit Service |
584ef9 |
unsigned int data_size = 0;
|
|
Packit Service |
584ef9 |
struct pollfd to_poll[1];
|
|
Packit Service |
584ef9 |
to_poll[0].fd = c->sfd;
|
|
Packit Service |
584ef9 |
to_poll[0].events = POLLOUT;
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
int ret = poll(to_poll, 1, 1000);
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
if (ret < 0) {
|
|
Packit Service |
584ef9 |
// fatal.
|
|
Packit Service |
584ef9 |
return -1;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
if (ret == 0) return 0;
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
if (to_poll[0].revents & POLLIN) {
|
|
Packit Service |
584ef9 |
char buf[1];
|
|
Packit Service |
584ef9 |
int res = ((conn*)c->c)->read(c->c, buf, 1);
|
|
Packit Service |
584ef9 |
if (res == 0 || (res == -1 && (errno != EAGAIN && errno != EWOULDBLOCK))) {
|
|
Packit Service |
584ef9 |
lru_crawler_close_client(c);
|
|
Packit Service |
584ef9 |
return -1;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
if ((data = bipbuf_peek_all(c->buf, &data_size)) != NULL) {
|
|
Packit Service |
584ef9 |
if (to_poll[0].revents & (POLLHUP|POLLERR)) {
|
|
Packit Service |
584ef9 |
lru_crawler_close_client(c);
|
|
Packit Service |
584ef9 |
return -1;
|
|
Packit Service |
584ef9 |
} else if (to_poll[0].revents & POLLOUT) {
|
|
Packit Service |
584ef9 |
int total = ((conn*)c->c)->write(c->c, data, data_size);
|
|
Packit Service |
584ef9 |
if (total == -1) {
|
|
Packit Service |
584ef9 |
if (errno != EAGAIN && errno != EWOULDBLOCK) {
|
|
Packit Service |
584ef9 |
lru_crawler_close_client(c);
|
|
Packit Service |
584ef9 |
return -1;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
} else if (total == 0) {
|
|
Packit Service |
584ef9 |
lru_crawler_close_client(c);
|
|
Packit Service |
584ef9 |
return -1;
|
|
Packit Service |
584ef9 |
} else {
|
|
Packit Service |
584ef9 |
bipbuf_poll(c->buf, total);
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
return 0;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
/* Grab some space to work with, if none exists, run the poll() loop and wait
|
|
Packit Service |
584ef9 |
* for it to clear up or close.
|
|
Packit Service |
584ef9 |
* Return NULL if closed.
|
|
Packit Service |
584ef9 |
*/
|
|
Packit Service |
584ef9 |
static int lru_crawler_client_getbuf(crawler_client_t *c) {
|
|
Packit Service |
584ef9 |
void *buf = NULL;
|
|
Packit Service |
584ef9 |
if (c->c == NULL) return -1;
|
|
Packit Service |
584ef9 |
/* not enough space. */
|
|
Packit Service |
584ef9 |
while ((buf = bipbuf_request(c->buf, LRU_CRAWLER_WRITEBUF)) == NULL) {
|
|
Packit Service |
584ef9 |
// TODO: max loops before closing.
|
|
Packit Service |
584ef9 |
int ret = lru_crawler_poll(c);
|
|
Packit Service |
584ef9 |
if (ret < 0) return ret;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
c->cbuf = buf;
|
|
Packit Service |
584ef9 |
return 0;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
static void lru_crawler_class_done(int i) {
|
|
Packit Service |
584ef9 |
crawlers[i].it_flags = 0;
|
|
Packit Service |
584ef9 |
crawler_count--;
|
|
Packit Service |
584ef9 |
do_item_unlinktail_q((item *)&crawlers[i]);
|
|
Packit Service |
584ef9 |
do_item_stats_add_crawl(i, crawlers[i].reclaimed,
|
|
Packit Service |
584ef9 |
crawlers[i].unfetched, crawlers[i].checked);
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_locks[i]);
|
|
Packit Service |
584ef9 |
if (active_crawler_mod.mod->doneclass != NULL)
|
|
Packit Service |
584ef9 |
active_crawler_mod.mod->doneclass(&active_crawler_mod, i);
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
static void *item_crawler_thread(void *arg) {
|
|
Packit Service |
584ef9 |
int i;
|
|
Packit Service |
584ef9 |
int crawls_persleep = settings.crawls_persleep;
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
pthread_mutex_lock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
pthread_cond_signal(&lru_crawler_cond);
|
|
Packit Service |
584ef9 |
settings.lru_crawler = true;
|
|
Packit Service |
584ef9 |
if (settings.verbose > 2)
|
|
Packit Service |
584ef9 |
fprintf(stderr, "Starting LRU crawler background thread\n");
|
|
Packit Service |
584ef9 |
while (do_run_lru_crawler_thread) {
|
|
Packit Service |
584ef9 |
pthread_cond_wait(&lru_crawler_cond, &lru_crawler_lock);
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
while (crawler_count) {
|
|
Packit Service |
584ef9 |
item *search = NULL;
|
|
Packit Service |
584ef9 |
void *hold_lock = NULL;
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
for (i = POWER_SMALLEST; i < LARGEST_ID; i++) {
|
|
Packit Service |
584ef9 |
if (crawlers[i].it_flags != 1) {
|
|
Packit Service |
584ef9 |
continue;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
/* Get memory from bipbuf, if client has no space, flush. */
|
|
Packit Service |
584ef9 |
if (active_crawler_mod.c.c != NULL) {
|
|
Packit Service |
584ef9 |
int ret = lru_crawler_client_getbuf(&active_crawler_mod.c);
|
|
Packit Service |
584ef9 |
if (ret != 0) {
|
|
Packit Service |
584ef9 |
lru_crawler_class_done(i);
|
|
Packit Service |
584ef9 |
continue;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
} else if (active_crawler_mod.mod->needs_client) {
|
|
Packit Service |
584ef9 |
lru_crawler_class_done(i);
|
|
Packit Service |
584ef9 |
continue;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
pthread_mutex_lock(&lru_locks[i]);
|
|
Packit Service |
584ef9 |
search = do_item_crawl_q((item *)&crawlers[i]);
|
|
Packit Service |
584ef9 |
if (search == NULL ||
|
|
Packit Service |
584ef9 |
(crawlers[i].remaining && --crawlers[i].remaining < 1)) {
|
|
Packit Service |
584ef9 |
if (settings.verbose > 2)
|
|
Packit Service |
584ef9 |
fprintf(stderr, "Nothing left to crawl for %d\n", i);
|
|
Packit Service |
584ef9 |
lru_crawler_class_done(i);
|
|
Packit Service |
584ef9 |
continue;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
uint32_t hv = hash(ITEM_key(search), search->nkey);
|
|
Packit Service |
584ef9 |
/* Attempt to hash item lock the "search" item. If locked, no
|
|
Packit Service |
584ef9 |
* other callers can incr the refcount
|
|
Packit Service |
584ef9 |
*/
|
|
Packit Service |
584ef9 |
if ((hold_lock = item_trylock(hv)) == NULL) {
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_locks[i]);
|
|
Packit Service |
584ef9 |
continue;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
/* Now see if the item is refcount locked */
|
|
Packit Service |
584ef9 |
if (refcount_incr(search) != 2) {
|
|
Packit Service |
584ef9 |
refcount_decr(search);
|
|
Packit Service |
584ef9 |
if (hold_lock)
|
|
Packit Service |
584ef9 |
item_trylock_unlock(hold_lock);
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_locks[i]);
|
|
Packit Service |
584ef9 |
continue;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
crawlers[i].checked++;
|
|
Packit Service |
584ef9 |
/* Frees the item or decrements the refcount. */
|
|
Packit Service |
584ef9 |
/* Interface for this could improve: do the free/decr here
|
|
Packit Service |
584ef9 |
* instead? */
|
|
Packit Service |
584ef9 |
if (!active_crawler_mod.mod->needs_lock) {
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_locks[i]);
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
active_crawler_mod.mod->eval(&active_crawler_mod, search, hv, i);
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
if (hold_lock)
|
|
Packit Service |
584ef9 |
item_trylock_unlock(hold_lock);
|
|
Packit Service |
584ef9 |
if (active_crawler_mod.mod->needs_lock) {
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_locks[i]);
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
if (crawls_persleep-- <= 0 && settings.lru_crawler_sleep) {
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
usleep(settings.lru_crawler_sleep);
|
|
Packit Service |
584ef9 |
pthread_mutex_lock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
crawls_persleep = settings.crawls_persleep;
|
|
Packit Service |
584ef9 |
} else if (!settings.lru_crawler_sleep) {
|
|
Packit Service |
584ef9 |
// TODO: only cycle lock every N?
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
pthread_mutex_lock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
if (active_crawler_mod.mod != NULL) {
|
|
Packit Service |
584ef9 |
if (active_crawler_mod.mod->finalize != NULL)
|
|
Packit Service |
584ef9 |
active_crawler_mod.mod->finalize(&active_crawler_mod);
|
|
Packit Service |
584ef9 |
while (active_crawler_mod.c.c != NULL && bipbuf_used(active_crawler_mod.c.buf)) {
|
|
Packit Service |
584ef9 |
lru_crawler_poll(&active_crawler_mod.c);
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
// Double checking in case the client closed during the poll
|
|
Packit Service |
584ef9 |
if (active_crawler_mod.c.c != NULL) {
|
|
Packit Service |
584ef9 |
lru_crawler_release_client(&active_crawler_mod.c);
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
active_crawler_mod.mod = NULL;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
if (settings.verbose > 2)
|
|
Packit Service |
584ef9 |
fprintf(stderr, "LRU crawler thread sleeping\n");
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
STATS_LOCK();
|
|
Packit Service |
584ef9 |
stats_state.lru_crawler_running = false;
|
|
Packit Service |
584ef9 |
STATS_UNLOCK();
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
if (settings.verbose > 2)
|
|
Packit Service |
584ef9 |
fprintf(stderr, "LRU crawler thread stopping\n");
|
|
Packit Service |
584ef9 |
settings.lru_crawler = false;
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
return NULL;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
static pthread_t item_crawler_tid;
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
int stop_item_crawler_thread(bool wait) {
|
|
Packit Service |
584ef9 |
int ret;
|
|
Packit Service |
584ef9 |
pthread_mutex_lock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
if (do_run_lru_crawler_thread == 0) {
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
return 0;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
do_run_lru_crawler_thread = 0;
|
|
Packit Service |
584ef9 |
pthread_cond_signal(&lru_crawler_cond);
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
if (wait && (ret = pthread_join(item_crawler_tid, NULL)) != 0) {
|
|
Packit Service |
584ef9 |
fprintf(stderr, "Failed to stop LRU crawler thread: %s\n", strerror(ret));
|
|
Packit Service |
584ef9 |
return -1;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
return 0;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
/* Lock dance to "block" until thread is waiting on its condition:
|
|
Packit Service |
584ef9 |
* caller locks mtx. caller spawns thread.
|
|
Packit Service |
584ef9 |
* thread blocks on mutex.
|
|
Packit Service |
584ef9 |
* caller waits on condition, releases lock.
|
|
Packit Service |
584ef9 |
* thread gets lock, sends signal.
|
|
Packit Service |
584ef9 |
* caller can't wait, as thread has lock.
|
|
Packit Service |
584ef9 |
* thread waits on condition, releases lock
|
|
Packit Service |
584ef9 |
* caller wakes on condition, gets lock.
|
|
Packit Service |
584ef9 |
* caller immediately releases lock.
|
|
Packit Service |
584ef9 |
* thread is now safely waiting on condition before the caller returns.
|
|
Packit Service |
584ef9 |
*/
|
|
Packit Service |
584ef9 |
int start_item_crawler_thread(void) {
|
|
Packit Service |
584ef9 |
int ret;
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
if (settings.lru_crawler)
|
|
Packit Service |
584ef9 |
return -1;
|
|
Packit Service |
584ef9 |
pthread_mutex_lock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
do_run_lru_crawler_thread = 1;
|
|
Packit Service |
584ef9 |
if ((ret = pthread_create(&item_crawler_tid, NULL,
|
|
Packit Service |
584ef9 |
item_crawler_thread, NULL)) != 0) {
|
|
Packit Service |
584ef9 |
fprintf(stderr, "Can't create LRU crawler thread: %s\n",
|
|
Packit Service |
584ef9 |
strerror(ret));
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
return -1;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
/* Avoid returning until the crawler has actually started */
|
|
Packit Service |
584ef9 |
pthread_cond_wait(&lru_crawler_cond, &lru_crawler_lock);
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
return 0;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
/* 'remaining' is passed in so the LRU maintainer thread can scrub the whole
|
|
Packit Service |
584ef9 |
* LRU every time.
|
|
Packit Service |
584ef9 |
*/
|
|
Packit Service |
584ef9 |
static int do_lru_crawler_start(uint32_t id, uint32_t remaining) {
|
|
Packit Service |
584ef9 |
uint32_t sid = id;
|
|
Packit Service |
584ef9 |
int starts = 0;
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
pthread_mutex_lock(&lru_locks[sid]);
|
|
Packit Service |
584ef9 |
if (crawlers[sid].it_flags == 0) {
|
|
Packit Service |
584ef9 |
if (settings.verbose > 2)
|
|
Packit Service |
584ef9 |
fprintf(stderr, "Kicking LRU crawler off for LRU %u\n", sid);
|
|
Packit Service |
584ef9 |
crawlers[sid].nbytes = 0;
|
|
Packit Service |
584ef9 |
crawlers[sid].nkey = 0;
|
|
Packit Service |
584ef9 |
crawlers[sid].it_flags = 1; /* For a crawler, this means enabled. */
|
|
Packit Service |
584ef9 |
crawlers[sid].next = 0;
|
|
Packit Service |
584ef9 |
crawlers[sid].prev = 0;
|
|
Packit Service |
584ef9 |
crawlers[sid].time = 0;
|
|
Packit Service |
584ef9 |
if (remaining == LRU_CRAWLER_CAP_REMAINING) {
|
|
Packit Service |
584ef9 |
remaining = do_get_lru_size(sid);
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
/* Values for remaining:
|
|
Packit Service |
584ef9 |
* remaining = 0
|
|
Packit Service |
584ef9 |
* - scan all elements, until a NULL is reached
|
|
Packit Service |
584ef9 |
* - if empty, NULL is reached right away
|
|
Packit Service |
584ef9 |
* remaining = n + 1
|
|
Packit Service |
584ef9 |
* - first n elements are parsed (or until a NULL is reached)
|
|
Packit Service |
584ef9 |
*/
|
|
Packit Service |
584ef9 |
if (remaining) remaining++;
|
|
Packit Service |
584ef9 |
crawlers[sid].remaining = remaining;
|
|
Packit Service |
584ef9 |
crawlers[sid].slabs_clsid = sid;
|
|
Packit Service |
584ef9 |
crawlers[sid].reclaimed = 0;
|
|
Packit Service |
584ef9 |
crawlers[sid].unfetched = 0;
|
|
Packit Service |
584ef9 |
crawlers[sid].checked = 0;
|
|
Packit Service |
584ef9 |
do_item_linktail_q((item *)&crawlers[sid]);
|
|
Packit Service |
584ef9 |
crawler_count++;
|
|
Packit Service |
584ef9 |
starts++;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_locks[sid]);
|
|
Packit Service |
584ef9 |
if (starts) {
|
|
Packit Service |
584ef9 |
STATS_LOCK();
|
|
Packit Service |
584ef9 |
stats_state.lru_crawler_running = true;
|
|
Packit Service |
584ef9 |
stats.lru_crawler_starts++;
|
|
Packit Service |
584ef9 |
STATS_UNLOCK();
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
return starts;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
static int lru_crawler_set_client(crawler_module_t *cm, void *c, const int sfd) {
|
|
Packit Service |
584ef9 |
crawler_client_t *crawlc = &cm->c;
|
|
Packit Service |
584ef9 |
if (crawlc->c != NULL) {
|
|
Packit Service |
584ef9 |
return -1;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
crawlc->c = c;
|
|
Packit Service |
584ef9 |
crawlc->sfd = sfd;
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
crawlc->buf = bipbuf_new(1024 * 128);
|
|
Packit Service |
584ef9 |
if (crawlc->buf == NULL) {
|
|
Packit Service |
584ef9 |
return -2;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
return 0;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
int lru_crawler_start(uint8_t *ids, uint32_t remaining,
|
|
Packit Service |
584ef9 |
const enum crawler_run_type type, void *data,
|
|
Packit Service |
584ef9 |
void *c, const int sfd) {
|
|
Packit Service |
584ef9 |
int starts = 0;
|
|
Packit Service |
584ef9 |
bool is_running;
|
|
Packit Service |
584ef9 |
static rel_time_t block_ae_until = 0;
|
|
Packit Service |
584ef9 |
pthread_mutex_lock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
STATS_LOCK();
|
|
Packit Service |
584ef9 |
is_running = stats_state.lru_crawler_running;
|
|
Packit Service |
584ef9 |
STATS_UNLOCK();
|
|
Packit Service |
584ef9 |
if (do_run_lru_crawler_thread == 0) {
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
return -2;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
if (is_running &&
|
|
Packit Service |
584ef9 |
!(type == CRAWLER_AUTOEXPIRE && active_crawler_type == CRAWLER_AUTOEXPIRE)) {
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
block_ae_until = current_time + 60;
|
|
Packit Service |
584ef9 |
return -1;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
if (type == CRAWLER_AUTOEXPIRE && block_ae_until > current_time) {
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
return -1;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
/* Configure the module */
|
|
Packit Service |
584ef9 |
if (!is_running) {
|
|
Packit Service |
584ef9 |
assert(crawler_mod_regs[type] != NULL);
|
|
Packit Service |
584ef9 |
active_crawler_mod.mod = crawler_mod_regs[type];
|
|
Packit Service |
584ef9 |
active_crawler_type = type;
|
|
Packit Service |
584ef9 |
if (active_crawler_mod.mod->init != NULL) {
|
|
Packit Service |
584ef9 |
active_crawler_mod.mod->init(&active_crawler_mod, data);
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
if (active_crawler_mod.mod->needs_client) {
|
|
Packit Service |
584ef9 |
if (c == NULL || sfd == 0) {
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
return -2;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
if (lru_crawler_set_client(&active_crawler_mod, c, sfd) != 0) {
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
return -2;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
/* we allow the autocrawler to restart sub-LRU's before completion */
|
|
Packit Service |
584ef9 |
for (int sid = POWER_SMALLEST; sid < POWER_LARGEST; sid++) {
|
|
Packit Service |
584ef9 |
if (ids[sid])
|
|
Packit Service |
584ef9 |
starts += do_lru_crawler_start(sid, remaining);
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
if (starts) {
|
|
Packit Service |
584ef9 |
pthread_cond_signal(&lru_crawler_cond);
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
return starts;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
/*
|
|
Packit Service |
584ef9 |
* Also only clear the crawlerstats once per sid.
|
|
Packit Service |
584ef9 |
*/
|
|
Packit Service |
584ef9 |
enum crawler_result_type lru_crawler_crawl(char *slabs, const enum crawler_run_type type,
|
|
Packit Service |
584ef9 |
void *c, const int sfd, unsigned int remaining) {
|
|
Packit Service |
584ef9 |
char *b = NULL;
|
|
Packit Service |
584ef9 |
uint32_t sid = 0;
|
|
Packit Service |
584ef9 |
int starts = 0;
|
|
Packit Service |
584ef9 |
uint8_t tocrawl[POWER_LARGEST];
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
/* FIXME: I added this while debugging. Don't think it's needed? */
|
|
Packit Service |
584ef9 |
memset(tocrawl, 0, sizeof(uint8_t) * POWER_LARGEST);
|
|
Packit Service |
584ef9 |
if (strcmp(slabs, "all") == 0) {
|
|
Packit Service |
584ef9 |
for (sid = 0; sid < POWER_LARGEST; sid++) {
|
|
Packit Service |
584ef9 |
tocrawl[sid] = 1;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
} else {
|
|
Packit Service |
584ef9 |
for (char *p = strtok_r(slabs, ",", &b);
|
|
Packit Service |
584ef9 |
p != NULL;
|
|
Packit Service |
584ef9 |
p = strtok_r(NULL, ",", &b)) {
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
if (!safe_strtoul(p, &sid) || sid < POWER_SMALLEST
|
|
Packit Service |
584ef9 |
|| sid >= MAX_NUMBER_OF_SLAB_CLASSES) {
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
return CRAWLER_BADCLASS;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
tocrawl[sid | TEMP_LRU] = 1;
|
|
Packit Service |
584ef9 |
tocrawl[sid | HOT_LRU] = 1;
|
|
Packit Service |
584ef9 |
tocrawl[sid | WARM_LRU] = 1;
|
|
Packit Service |
584ef9 |
tocrawl[sid | COLD_LRU] = 1;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
starts = lru_crawler_start(tocrawl, remaining, type, NULL, c, sfd);
|
|
Packit Service |
584ef9 |
if (starts == -1) {
|
|
Packit Service |
584ef9 |
return CRAWLER_RUNNING;
|
|
Packit Service |
584ef9 |
} else if (starts == -2) {
|
|
Packit Service |
584ef9 |
return CRAWLER_ERROR; /* FIXME: not very helpful. */
|
|
Packit Service |
584ef9 |
} else if (starts) {
|
|
Packit Service |
584ef9 |
return CRAWLER_OK;
|
|
Packit Service |
584ef9 |
} else {
|
|
Packit Service |
584ef9 |
return CRAWLER_NOTSTARTED;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
/* If we hold this lock, crawler can't wake up or move */
|
|
Packit Service |
584ef9 |
void lru_crawler_pause(void) {
|
|
Packit Service |
584ef9 |
pthread_mutex_lock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
void lru_crawler_resume(void) {
|
|
Packit Service |
584ef9 |
pthread_mutex_unlock(&lru_crawler_lock);
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
|
|
Packit Service |
584ef9 |
int init_lru_crawler(void *arg) {
|
|
Packit Service |
584ef9 |
if (lru_crawler_initialized == 0) {
|
|
Packit Service |
584ef9 |
#ifdef EXTSTORE
|
|
Packit Service |
584ef9 |
storage = arg;
|
|
Packit Service |
584ef9 |
#endif
|
|
Packit Service |
584ef9 |
active_crawler_mod.c.c = NULL;
|
|
Packit Service |
584ef9 |
active_crawler_mod.mod = NULL;
|
|
Packit Service |
584ef9 |
active_crawler_mod.data = NULL;
|
|
Packit Service |
584ef9 |
lru_crawler_initialized = 1;
|
|
Packit Service |
584ef9 |
}
|
|
Packit Service |
584ef9 |
return 0;
|
|
Packit Service |
584ef9 |
}
|