X-Git-Url: http://git.indexdata.com/?p=idzebra-moved-to-github.git;a=blobdiff_plain;f=index%2Fkey_block.c;h=c5b0641482c7c21120bd4d7abdb336d39924e3da;hp=318fbb0a142ef236350ec66a2bec9e383d2ca374;hb=e9b6a86cc5ac30d6c6331c46ef3edaa08b99a2a7;hpb=9cb67fae9997acfd1399515584c6ab5222f3eaf1 diff --git a/index/key_block.c b/index/key_block.c index 318fbb0..c5b0641 100644 --- a/index/key_block.c +++ b/index/key_block.c @@ -1,8 +1,5 @@ -/* $Id: key_block.c,v 1.2 2006-11-21 14:54:12 adam Exp $ - Copyright (C) 1995-2006 - Index Data ApS - -This file is part of the Zebra server. +/* This file is part of the Zebra server. + Copyright (C) Index Data Zebra is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free @@ -20,8 +17,12 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ +#if HAVE_CONFIG_H +#include +#endif #include #include +#include #include #include @@ -29,9 +30,9 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include #endif -#include -#include "index.h" #include "key_block.h" +#include +#include struct zebra_key_block { char **key_buf; @@ -40,8 +41,9 @@ struct zebra_key_block { size_t key_buf_used; int key_file_no; char *key_tmp_dir; -#if YAZ_POSIX_THREADS + int use_threads; char **alt_buf; +#if YAZ_POSIX_THREADS char **thread_key_buf; size_t thread_ptr_top; size_t thread_ptr_i; @@ -63,8 +65,6 @@ struct encode_info { char buf[ENCODE_BUFLEN]; }; -static int log_level = 0; - #define USE_SHELLSORT 0 #if USE_SHELLSORT @@ -75,11 +75,11 @@ static void shellsort(void *ar, int r, size_t s, char v[100]; int h, i, j, k; static const int incs[16] = { 1391376, 463792, 198768, 86961, 33936, - 13776, 4592, 1968, 861, 336, + 13776, 4592, 1968, 861, 336, 112, 48, 21, 7, 3, 1 }; for ( k = 0; k < 16; k++) for (h = incs[k], i = h; i < r; i++) - { + { memcpy (v, a+s*i, s); j = i; while (j > h && (*cmp)(a + s*(j-h), v) > 0) @@ -88,7 +88,7 @@ static void shellsort(void *ar, int r, size_t s, j -= h; } memcpy (a+s*j, v, s); - } + } } #endif @@ -99,15 +99,21 @@ static void encode_key_init(struct encode_info *i) i->decode_handle = iscz1_start(); } -static void encode_key_write (char *k, struct encode_info *i, FILE *outf) +static void encode_key_write(const char *k, struct encode_info *i, FILE *outf) { struct it_key key; char *bp = i->buf, *bp0; const char *src = (char *) &key; + size_t klen = strlen(k); + + if (fwrite (k, klen+1, 1, outf) != 1) + { + yaz_log (YLOG_FATAL|YLOG_ERRNO, "fwrite"); + zebra_exit("encode_key_write"); + } + + k = k + klen+1; - /* copy term to output buf */ - while ((*bp++ = *k++)) - ; /* and copy & align key so we can mangle */ memcpy (&key, k+1, sizeof(struct it_key)); /* *k is insert/delete */ @@ -144,7 +150,7 @@ static void encode_key_write (char *k, struct encode_info *i, FILE *outf) } static void encode_key_flush (struct encode_info *i, FILE *outf) -{ +{ iscz1_stop(i->encode_handle); iscz1_stop(i->decode_handle); } @@ -159,47 +165,60 @@ static void *thread_func(void *vp) while (1) { pthread_mutex_lock(&p->mutex); - + while (!p->is_sorting && !p->exit_flag) pthread_cond_wait(&p->work_available, &p->mutex); if (p->exit_flag) break; - + pthread_mutex_unlock(&p->mutex); - - key_block_flush_int(p, p->thread_key_buf, + + key_block_flush_int(p, p->thread_key_buf, p->thread_ptr_top, p->thread_ptr_i); - + pthread_mutex_lock(&p->mutex); p->is_sorting = 0; pthread_cond_signal(&p->cond_sorting); - pthread_mutex_unlock(&p->mutex); + pthread_mutex_unlock(&p->mutex); } pthread_mutex_unlock(&p->mutex); return 0; } #endif -zebra_key_block_t key_block_create(int mem, const char *key_tmp_dir) +zebra_key_block_t key_block_create(int mem, const char *key_tmp_dir, + int use_threads) { zebra_key_block_t p = xmalloc(sizeof(*p)); +#if YAZ_POSIX_THREADS + /* we'll be making two memory areas so cut in half */ + if (use_threads) + mem = mem / 2; +#endif p->key_buf = (char**) xmalloc (mem); p->ptr_top = mem/sizeof(char*); p->ptr_i = 0; p->key_buf_used = 0; p->key_tmp_dir = xstrdup(key_tmp_dir); p->key_file_no = 0; + p->alt_buf = 0; + p->use_threads = 0; + if (use_threads) + { #if YAZ_POSIX_THREADS - p->alt_buf = (char**) xmalloc (mem); - p->is_sorting = 0; - p->exit_flag = 0; - pthread_mutex_init(&p->mutex, 0); - pthread_cond_init(&p->work_available, 0); - pthread_cond_init(&p->cond_sorting, 0); - pthread_create(&p->thread_id, 0, thread_func, p); + p->use_threads = use_threads; + p->is_sorting = 0; + p->exit_flag = 0; + pthread_mutex_init(&p->mutex, 0); + pthread_cond_init(&p->work_available, 0); + pthread_cond_init(&p->cond_sorting, 0); + pthread_create(&p->thread_id, 0, thread_func, p); + p->alt_buf = (char**) xmalloc (mem); #endif + } + yaz_log(YLOG_DEBUG, "key_block_create t=%d", p->use_threads); return p; } @@ -208,24 +227,27 @@ void key_block_destroy(zebra_key_block_t *pp) zebra_key_block_t p = *pp; if (p) { + if (p->use_threads) + { #if YAZ_POSIX_THREADS - pthread_mutex_lock(&p->mutex); + pthread_mutex_lock(&p->mutex); - while (p->is_sorting) - pthread_cond_wait(&p->cond_sorting, &p->mutex); + while (p->is_sorting) + pthread_cond_wait(&p->cond_sorting, &p->mutex); - p->exit_flag = 1; + p->exit_flag = 1; - pthread_cond_broadcast(&p->work_available); + pthread_cond_broadcast(&p->work_available); - pthread_mutex_unlock(&p->mutex); - pthread_join(p->thread_id, 0); - pthread_cond_destroy(&p->work_available); - pthread_cond_destroy(&p->cond_sorting); - pthread_mutex_destroy(&p->mutex); + pthread_mutex_unlock(&p->mutex); + pthread_join(p->thread_id, 0); + pthread_cond_destroy(&p->work_available); + pthread_cond_destroy(&p->cond_sorting); + pthread_mutex_destroy(&p->mutex); - xfree(p->alt_buf); #endif + xfree(p->alt_buf); + } xfree(p->key_buf); xfree(p->key_tmp_dir); xfree(p); @@ -233,7 +255,7 @@ void key_block_destroy(zebra_key_block_t *pp) } } -void key_block_write(zebra_key_block_t p, SYSNO sysno, struct it_key *key_in, +void key_block_write(zebra_key_block_t p, zint sysno, struct it_key *key_in, int cmd, const char *str_buf, size_t str_len, zint staticrank, int static_rank_enable) { @@ -247,27 +269,30 @@ void key_block_write(zebra_key_block_t p, SYSNO sysno, struct it_key *key_in, assert(p->ptr_i > 0); (p->key_buf)[p->ptr_top - p->ptr_i] = (char*)p->key_buf + p->key_buf_used; - + /* key_in->mem[0] ord/ch */ /* key_in->mem[1] filter specified record ID */ - + /* encode the ordinal value (field/use/attribute) .. */ ch = CAST_ZINT_TO_INT(key_in->mem[0]); p->key_buf_used += key_SU_encode(ch, (char*)p->key_buf + p->key_buf_used); - + /* copy the 0-terminated stuff from str to output */ memcpy((char*)p->key_buf + p->key_buf_used, str_buf, str_len); p->key_buf_used += str_len; ((char*)p->key_buf)[(p->key_buf_used)++] = '\0'; - + /* the delete/insert indicator */ ((char*)p->key_buf)[(p->key_buf_used)++] = cmd; - + if (static_rank_enable) + { + assert(staticrank >= 0); key_out.mem[j++] = staticrank; - + } + if (key_in->mem[1]) /* filter specified record ID */ key_out.mem[j++] = key_in->mem[1]; else @@ -275,7 +300,7 @@ void key_block_write(zebra_key_block_t p, SYSNO sysno, struct it_key *key_in, for (i = 2; i < key_in->len; i++) key_out.mem[j++] = key_in->mem[i]; key_out.len = j; - + memcpy((char*)p->key_buf + p->key_buf_used, &key_out, sizeof(key_out)); (p->key_buf_used) += sizeof(key_out); @@ -283,18 +308,20 @@ void key_block_write(zebra_key_block_t p, SYSNO sysno, struct it_key *key_in, void key_block_flush_int(zebra_key_block_t p, - char **key_buf, size_t ptr_top, size_t ptr_i) + char **key_buf, size_t ptr_top, size_t ptr_i) { FILE *outf; char out_fname[200]; char *prevcp, *cp; struct encode_info encode_info; + if (ptr_i == 0) + return ; + (p->key_file_no)++; - yaz_log(YLOG_LOG, "sorting section %d", (p->key_file_no)); - yaz_log(log_level, " sort_buff at %p n=%d", - key_buf + ptr_top - ptr_i,ptr_i); + yaz_log(YLOG_DEBUG, "sorting section %d", (p->key_file_no)); + assert(ptr_i > 0); #if USE_SHELLSORT shellsort(key_buf + ptr_top - ptr_i, ptr_i, @@ -310,12 +337,12 @@ void key_block_flush_int(zebra_key_block_t p, yaz_log (YLOG_FATAL|YLOG_ERRNO, "fopen %s", out_fname); zebra_exit("key_block_flush"); } - yaz_log(YLOG_LOG, "writing section %d", p->key_file_no); + yaz_log(YLOG_DEBUG, "writing section %d", p->key_file_no); prevcp = cp = (key_buf)[ptr_top - ptr_i]; - + encode_key_init (&encode_info); encode_key_write (cp, &encode_info, outf); - + while (--ptr_i > 0) { cp = (key_buf)[ptr_top - ptr_i]; @@ -335,47 +362,46 @@ void key_block_flush_int(zebra_key_block_t p, yaz_log (YLOG_FATAL|YLOG_ERRNO, "fclose %s", out_fname); zebra_exit("key_block_flush"); } - yaz_log(YLOG_LOG, "finished section %d", p->key_file_no); + yaz_log(YLOG_DEBUG, "finished section %d", p->key_file_no); } void key_block_flush(zebra_key_block_t p, int is_final) - /* optimizing: if final=1, and no files written yet */ - /* push the keys directly to merge, sidestepping the */ - /* temp file altogether. Speeds small updates */ { -#if YAZ_POSIX_THREADS - char **tmp; -#endif if (!p) return; + if (p->use_threads) + { #if YAZ_POSIX_THREADS - pthread_mutex_lock(&p->mutex); + char **tmp; - while (p->is_sorting) - pthread_cond_wait(&p->cond_sorting, &p->mutex); + pthread_mutex_lock(&p->mutex); - p->is_sorting = 1; + while (p->is_sorting) + pthread_cond_wait(&p->cond_sorting, &p->mutex); - p->thread_ptr_top = p->ptr_top; - p->thread_ptr_i = p->ptr_i; - p->thread_key_buf = p->key_buf; + p->is_sorting = 1; - tmp = p->key_buf; - p->key_buf = p->alt_buf; - p->alt_buf = tmp; + p->thread_ptr_top = p->ptr_top; + p->thread_ptr_i = p->ptr_i; + p->thread_key_buf = p->key_buf; - pthread_cond_signal(&p->work_available); + tmp = p->key_buf; + p->key_buf = p->alt_buf; + p->alt_buf = tmp; - if (is_final) - { - while (p->is_sorting) - pthread_cond_wait(&p->cond_sorting, &p->mutex); - } - pthread_mutex_unlock(&p->mutex); -#else - key_block_flush_int(p, p->key_buf, p->ptr_top, p->ptr_i); + pthread_cond_signal(&p->work_available); + + if (is_final) + { + while (p->is_sorting) + pthread_cond_wait(&p->cond_sorting, &p->mutex); + } + pthread_mutex_unlock(&p->mutex); #endif + } + else + key_block_flush_int(p, p->key_buf, p->ptr_top, p->ptr_i); p->ptr_i = 0; p->key_buf_used = 0; } @@ -390,6 +416,7 @@ int key_block_get_no_files(zebra_key_block_t p) /* * Local variables: * c-basic-offset: 4 + * c-file-style: "Stroustrup" * indent-tabs-mode: nil * End: * vim: shiftwidth=4 tabstop=8 expandtab