X-Git-Url: http://git.indexdata.com/?p=idzebra-moved-to-github.git;a=blobdiff_plain;f=index%2Fkey_block.c;h=d127d556ee82c6b3c2de7c39df91ece09f1310b1;hp=318fbb0a142ef236350ec66a2bec9e383d2ca374;hb=a5c8c78e8671af863fc61b2ad8b24f92f827f7b2;hpb=9cb67fae9997acfd1399515584c6ab5222f3eaf1 diff --git a/index/key_block.c b/index/key_block.c index 318fbb0..d127d55 100644 --- a/index/key_block.c +++ b/index/key_block.c @@ -1,8 +1,5 @@ -/* $Id: key_block.c,v 1.2 2006-11-21 14:54:12 adam Exp $ - Copyright (C) 1995-2006 - Index Data ApS - -This file is part of the Zebra server. +/* This file is part of the Zebra server. + Copyright (C) 1994-2011 Index Data Zebra is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free @@ -20,8 +17,12 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ +#if HAVE_CONFIG_H +#include +#endif #include #include +#include #include #include @@ -29,9 +30,9 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include #endif -#include -#include "index.h" #include "key_block.h" +#include +#include struct zebra_key_block { char **key_buf; @@ -40,8 +41,9 @@ struct zebra_key_block { size_t key_buf_used; int key_file_no; char *key_tmp_dir; -#if YAZ_POSIX_THREADS + int use_threads; char **alt_buf; +#if YAZ_POSIX_THREADS char **thread_key_buf; size_t thread_ptr_top; size_t thread_ptr_i; @@ -63,8 +65,6 @@ struct encode_info { char buf[ENCODE_BUFLEN]; }; -static int log_level = 0; - #define USE_SHELLSORT 0 #if USE_SHELLSORT @@ -99,15 +99,21 @@ static void encode_key_init(struct encode_info *i) i->decode_handle = iscz1_start(); } -static void encode_key_write (char *k, struct encode_info *i, FILE *outf) +static void encode_key_write(const char *k, struct encode_info *i, FILE *outf) { struct it_key key; char *bp = i->buf, *bp0; const char *src = (char *) &key; + size_t klen = strlen(k); + + if (fwrite (k, klen+1, 1, outf) != 1) + { + yaz_log (YLOG_FATAL|YLOG_ERRNO, "fwrite"); + zebra_exit("encode_key_write"); + } + + k = k + klen+1; - /* copy term to output buf */ - while ((*bp++ = *k++)) - ; /* and copy & align key so we can mangle */ memcpy (&key, k+1, sizeof(struct it_key)); /* *k is insert/delete */ @@ -181,25 +187,38 @@ static void *thread_func(void *vp) } #endif -zebra_key_block_t key_block_create(int mem, const char *key_tmp_dir) +zebra_key_block_t key_block_create(int mem, const char *key_tmp_dir, + int use_threads) { zebra_key_block_t p = xmalloc(sizeof(*p)); +#if YAZ_POSIX_THREADS + /* we'll be making two memory areas so cut in half */ + if (use_threads) + mem = mem / 2; +#endif p->key_buf = (char**) xmalloc (mem); p->ptr_top = mem/sizeof(char*); p->ptr_i = 0; p->key_buf_used = 0; p->key_tmp_dir = xstrdup(key_tmp_dir); p->key_file_no = 0; + p->alt_buf = 0; + p->use_threads = 0; + if (use_threads) + { #if YAZ_POSIX_THREADS - p->alt_buf = (char**) xmalloc (mem); - p->is_sorting = 0; - p->exit_flag = 0; - pthread_mutex_init(&p->mutex, 0); - pthread_cond_init(&p->work_available, 0); - pthread_cond_init(&p->cond_sorting, 0); - pthread_create(&p->thread_id, 0, thread_func, p); + p->use_threads = use_threads; + p->is_sorting = 0; + p->exit_flag = 0; + pthread_mutex_init(&p->mutex, 0); + pthread_cond_init(&p->work_available, 0); + pthread_cond_init(&p->cond_sorting, 0); + pthread_create(&p->thread_id, 0, thread_func, p); + p->alt_buf = (char**) xmalloc (mem); #endif + } + yaz_log(YLOG_DEBUG, "key_block_create t=%d", p->use_threads); return p; } @@ -208,24 +227,27 @@ void key_block_destroy(zebra_key_block_t *pp) zebra_key_block_t p = *pp; if (p) { + if (p->use_threads) + { #if YAZ_POSIX_THREADS - pthread_mutex_lock(&p->mutex); - - while (p->is_sorting) - pthread_cond_wait(&p->cond_sorting, &p->mutex); - - p->exit_flag = 1; - - pthread_cond_broadcast(&p->work_available); - - pthread_mutex_unlock(&p->mutex); - pthread_join(p->thread_id, 0); - pthread_cond_destroy(&p->work_available); - pthread_cond_destroy(&p->cond_sorting); - pthread_mutex_destroy(&p->mutex); - - xfree(p->alt_buf); + pthread_mutex_lock(&p->mutex); + + while (p->is_sorting) + pthread_cond_wait(&p->cond_sorting, &p->mutex); + + p->exit_flag = 1; + + pthread_cond_broadcast(&p->work_available); + + pthread_mutex_unlock(&p->mutex); + pthread_join(p->thread_id, 0); + pthread_cond_destroy(&p->work_available); + pthread_cond_destroy(&p->cond_sorting); + pthread_mutex_destroy(&p->mutex); + #endif + xfree(p->alt_buf); + } xfree(p->key_buf); xfree(p->key_tmp_dir); xfree(p); @@ -233,7 +255,7 @@ void key_block_destroy(zebra_key_block_t *pp) } } -void key_block_write(zebra_key_block_t p, SYSNO sysno, struct it_key *key_in, +void key_block_write(zebra_key_block_t p, zint sysno, struct it_key *key_in, int cmd, const char *str_buf, size_t str_len, zint staticrank, int static_rank_enable) { @@ -266,7 +288,10 @@ void key_block_write(zebra_key_block_t p, SYSNO sysno, struct it_key *key_in, ((char*)p->key_buf)[(p->key_buf_used)++] = cmd; if (static_rank_enable) + { + assert(staticrank >= 0); key_out.mem[j++] = staticrank; + } if (key_in->mem[1]) /* filter specified record ID */ key_out.mem[j++] = key_in->mem[1]; @@ -283,18 +308,20 @@ void key_block_write(zebra_key_block_t p, SYSNO sysno, struct it_key *key_in, void key_block_flush_int(zebra_key_block_t p, - char **key_buf, size_t ptr_top, size_t ptr_i) + char **key_buf, size_t ptr_top, size_t ptr_i) { FILE *outf; char out_fname[200]; char *prevcp, *cp; struct encode_info encode_info; + if (ptr_i == 0) + return ; + (p->key_file_no)++; - yaz_log(YLOG_LOG, "sorting section %d", (p->key_file_no)); - yaz_log(log_level, " sort_buff at %p n=%d", - key_buf + ptr_top - ptr_i,ptr_i); + yaz_log(YLOG_DEBUG, "sorting section %d", (p->key_file_no)); + assert(ptr_i > 0); #if USE_SHELLSORT shellsort(key_buf + ptr_top - ptr_i, ptr_i, @@ -310,7 +337,7 @@ void key_block_flush_int(zebra_key_block_t p, yaz_log (YLOG_FATAL|YLOG_ERRNO, "fopen %s", out_fname); zebra_exit("key_block_flush"); } - yaz_log(YLOG_LOG, "writing section %d", p->key_file_no); + yaz_log(YLOG_DEBUG, "writing section %d", p->key_file_no); prevcp = cp = (key_buf)[ptr_top - ptr_i]; encode_key_init (&encode_info); @@ -335,47 +362,46 @@ void key_block_flush_int(zebra_key_block_t p, yaz_log (YLOG_FATAL|YLOG_ERRNO, "fclose %s", out_fname); zebra_exit("key_block_flush"); } - yaz_log(YLOG_LOG, "finished section %d", p->key_file_no); + yaz_log(YLOG_DEBUG, "finished section %d", p->key_file_no); } void key_block_flush(zebra_key_block_t p, int is_final) - /* optimizing: if final=1, and no files written yet */ - /* push the keys directly to merge, sidestepping the */ - /* temp file altogether. Speeds small updates */ { -#if YAZ_POSIX_THREADS - char **tmp; -#endif if (!p) return; -#if YAZ_POSIX_THREADS - pthread_mutex_lock(&p->mutex); - - while (p->is_sorting) - pthread_cond_wait(&p->cond_sorting, &p->mutex); - - p->is_sorting = 1; - - p->thread_ptr_top = p->ptr_top; - p->thread_ptr_i = p->ptr_i; - p->thread_key_buf = p->key_buf; - - tmp = p->key_buf; - p->key_buf = p->alt_buf; - p->alt_buf = tmp; - - pthread_cond_signal(&p->work_available); - - if (is_final) + if (p->use_threads) { +#if YAZ_POSIX_THREADS + char **tmp; + + pthread_mutex_lock(&p->mutex); + while (p->is_sorting) pthread_cond_wait(&p->cond_sorting, &p->mutex); - } - pthread_mutex_unlock(&p->mutex); -#else - key_block_flush_int(p, p->key_buf, p->ptr_top, p->ptr_i); + + p->is_sorting = 1; + + p->thread_ptr_top = p->ptr_top; + p->thread_ptr_i = p->ptr_i; + p->thread_key_buf = p->key_buf; + + tmp = p->key_buf; + p->key_buf = p->alt_buf; + p->alt_buf = tmp; + + pthread_cond_signal(&p->work_available); + + if (is_final) + { + while (p->is_sorting) + pthread_cond_wait(&p->cond_sorting, &p->mutex); + } + pthread_mutex_unlock(&p->mutex); #endif + } + else + key_block_flush_int(p, p->key_buf, p->ptr_top, p->ptr_i); p->ptr_i = 0; p->key_buf_used = 0; } @@ -390,6 +416,7 @@ int key_block_get_no_files(zebra_key_block_t p) /* * Local variables: * c-basic-offset: 4 + * c-file-style: "Stroustrup" * indent-tabs-mode: nil * End: * vim: shiftwidth=4 tabstop=8 expandtab