-/* $Id: key_block.c,v 1.2 2006-11-21 14:54:12 adam Exp $
- Copyright (C) 1995-2006
- Index Data ApS
-
-This file is part of the Zebra server.
+/* This file is part of the Zebra server.
+ Copyright (C) Index Data
Zebra is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
*/
+#if HAVE_CONFIG_H
+#include <config.h>
+#endif
#include <stdio.h>
#include <stdlib.h>
+#include <string.h>
#include <assert.h>
#include <ctype.h>
#include <pthread.h>
#endif
-#include <yaz/nmem.h>
-#include "index.h"
#include "key_block.h"
+#include <yaz/nmem.h>
+#include <yaz/xmalloc.h>
struct zebra_key_block {
char **key_buf;
size_t key_buf_used;
int key_file_no;
char *key_tmp_dir;
-#if YAZ_POSIX_THREADS
+ int use_threads;
char **alt_buf;
+#if YAZ_POSIX_THREADS
char **thread_key_buf;
size_t thread_ptr_top;
size_t thread_ptr_i;
char buf[ENCODE_BUFLEN];
};
-static int log_level = 0;
-
#define USE_SHELLSORT 0
#if USE_SHELLSORT
char v[100];
int h, i, j, k;
static const int incs[16] = { 1391376, 463792, 198768, 86961, 33936,
- 13776, 4592, 1968, 861, 336,
+ 13776, 4592, 1968, 861, 336,
112, 48, 21, 7, 3, 1 };
for ( k = 0; k < 16; k++)
for (h = incs[k], i = h; i < r; i++)
- {
+ {
memcpy (v, a+s*i, s);
j = i;
while (j > h && (*cmp)(a + s*(j-h), v) > 0)
j -= h;
}
memcpy (a+s*j, v, s);
- }
+ }
}
#endif
i->decode_handle = iscz1_start();
}
-static void encode_key_write (char *k, struct encode_info *i, FILE *outf)
+static void encode_key_write(const char *k, struct encode_info *i, FILE *outf)
{
struct it_key key;
char *bp = i->buf, *bp0;
const char *src = (char *) &key;
+ size_t klen = strlen(k);
+
+ if (fwrite (k, klen+1, 1, outf) != 1)
+ {
+ yaz_log (YLOG_FATAL|YLOG_ERRNO, "fwrite");
+ zebra_exit("encode_key_write");
+ }
+
+ k = k + klen+1;
- /* copy term to output buf */
- while ((*bp++ = *k++))
- ;
/* and copy & align key so we can mangle */
memcpy (&key, k+1, sizeof(struct it_key)); /* *k is insert/delete */
}
static void encode_key_flush (struct encode_info *i, FILE *outf)
-{
+{
iscz1_stop(i->encode_handle);
iscz1_stop(i->decode_handle);
}
while (1)
{
pthread_mutex_lock(&p->mutex);
-
+
while (!p->is_sorting && !p->exit_flag)
pthread_cond_wait(&p->work_available, &p->mutex);
if (p->exit_flag)
break;
-
+
pthread_mutex_unlock(&p->mutex);
-
- key_block_flush_int(p, p->thread_key_buf,
+
+ key_block_flush_int(p, p->thread_key_buf,
p->thread_ptr_top, p->thread_ptr_i);
-
+
pthread_mutex_lock(&p->mutex);
p->is_sorting = 0;
pthread_cond_signal(&p->cond_sorting);
- pthread_mutex_unlock(&p->mutex);
+ pthread_mutex_unlock(&p->mutex);
}
pthread_mutex_unlock(&p->mutex);
return 0;
}
#endif
-zebra_key_block_t key_block_create(int mem, const char *key_tmp_dir)
+zebra_key_block_t key_block_create(int mem, const char *key_tmp_dir,
+ int use_threads)
{
zebra_key_block_t p = xmalloc(sizeof(*p));
+#if YAZ_POSIX_THREADS
+ /* we'll be making two memory areas so cut in half */
+ if (use_threads)
+ mem = mem / 2;
+#endif
p->key_buf = (char**) xmalloc (mem);
p->ptr_top = mem/sizeof(char*);
p->ptr_i = 0;
p->key_buf_used = 0;
p->key_tmp_dir = xstrdup(key_tmp_dir);
p->key_file_no = 0;
+ p->alt_buf = 0;
+ p->use_threads = 0;
+ if (use_threads)
+ {
#if YAZ_POSIX_THREADS
- p->alt_buf = (char**) xmalloc (mem);
- p->is_sorting = 0;
- p->exit_flag = 0;
- pthread_mutex_init(&p->mutex, 0);
- pthread_cond_init(&p->work_available, 0);
- pthread_cond_init(&p->cond_sorting, 0);
- pthread_create(&p->thread_id, 0, thread_func, p);
+ p->use_threads = use_threads;
+ p->is_sorting = 0;
+ p->exit_flag = 0;
+ pthread_mutex_init(&p->mutex, 0);
+ pthread_cond_init(&p->work_available, 0);
+ pthread_cond_init(&p->cond_sorting, 0);
+ pthread_create(&p->thread_id, 0, thread_func, p);
+ p->alt_buf = (char**) xmalloc (mem);
#endif
+ }
+ yaz_log(YLOG_DEBUG, "key_block_create t=%d", p->use_threads);
return p;
}
zebra_key_block_t p = *pp;
if (p)
{
+ if (p->use_threads)
+ {
#if YAZ_POSIX_THREADS
- pthread_mutex_lock(&p->mutex);
+ pthread_mutex_lock(&p->mutex);
- while (p->is_sorting)
- pthread_cond_wait(&p->cond_sorting, &p->mutex);
+ while (p->is_sorting)
+ pthread_cond_wait(&p->cond_sorting, &p->mutex);
- p->exit_flag = 1;
+ p->exit_flag = 1;
- pthread_cond_broadcast(&p->work_available);
+ pthread_cond_broadcast(&p->work_available);
- pthread_mutex_unlock(&p->mutex);
- pthread_join(p->thread_id, 0);
- pthread_cond_destroy(&p->work_available);
- pthread_cond_destroy(&p->cond_sorting);
- pthread_mutex_destroy(&p->mutex);
+ pthread_mutex_unlock(&p->mutex);
+ pthread_join(p->thread_id, 0);
+ pthread_cond_destroy(&p->work_available);
+ pthread_cond_destroy(&p->cond_sorting);
+ pthread_mutex_destroy(&p->mutex);
- xfree(p->alt_buf);
#endif
+ xfree(p->alt_buf);
+ }
xfree(p->key_buf);
xfree(p->key_tmp_dir);
xfree(p);
}
}
-void key_block_write(zebra_key_block_t p, SYSNO sysno, struct it_key *key_in,
+void key_block_write(zebra_key_block_t p, zint sysno, struct it_key *key_in,
int cmd, const char *str_buf, size_t str_len,
zint staticrank, int static_rank_enable)
{
assert(p->ptr_i > 0);
(p->key_buf)[p->ptr_top - p->ptr_i] =
(char*)p->key_buf + p->key_buf_used;
-
+
/* key_in->mem[0] ord/ch */
/* key_in->mem[1] filter specified record ID */
-
+
/* encode the ordinal value (field/use/attribute) .. */
ch = CAST_ZINT_TO_INT(key_in->mem[0]);
p->key_buf_used +=
key_SU_encode(ch, (char*)p->key_buf +
p->key_buf_used);
-
+
/* copy the 0-terminated stuff from str to output */
memcpy((char*)p->key_buf + p->key_buf_used, str_buf, str_len);
p->key_buf_used += str_len;
((char*)p->key_buf)[(p->key_buf_used)++] = '\0';
-
+
/* the delete/insert indicator */
((char*)p->key_buf)[(p->key_buf_used)++] = cmd;
-
+
if (static_rank_enable)
+ {
+ assert(staticrank >= 0);
key_out.mem[j++] = staticrank;
-
+ }
+
if (key_in->mem[1]) /* filter specified record ID */
key_out.mem[j++] = key_in->mem[1];
else
for (i = 2; i < key_in->len; i++)
key_out.mem[j++] = key_in->mem[i];
key_out.len = j;
-
+
memcpy((char*)p->key_buf + p->key_buf_used,
&key_out, sizeof(key_out));
(p->key_buf_used) += sizeof(key_out);
void key_block_flush_int(zebra_key_block_t p,
- char **key_buf, size_t ptr_top, size_t ptr_i)
+ char **key_buf, size_t ptr_top, size_t ptr_i)
{
FILE *outf;
char out_fname[200];
char *prevcp, *cp;
struct encode_info encode_info;
+ if (ptr_i == 0)
+ return ;
+
(p->key_file_no)++;
- yaz_log(YLOG_LOG, "sorting section %d", (p->key_file_no));
- yaz_log(log_level, " sort_buff at %p n=%d",
- key_buf + ptr_top - ptr_i,ptr_i);
+ yaz_log(YLOG_DEBUG, "sorting section %d", (p->key_file_no));
+ assert(ptr_i > 0);
#if USE_SHELLSORT
shellsort(key_buf + ptr_top - ptr_i, ptr_i,
yaz_log (YLOG_FATAL|YLOG_ERRNO, "fopen %s", out_fname);
zebra_exit("key_block_flush");
}
- yaz_log(YLOG_LOG, "writing section %d", p->key_file_no);
+ yaz_log(YLOG_DEBUG, "writing section %d", p->key_file_no);
prevcp = cp = (key_buf)[ptr_top - ptr_i];
-
+
encode_key_init (&encode_info);
encode_key_write (cp, &encode_info, outf);
-
+
while (--ptr_i > 0)
{
cp = (key_buf)[ptr_top - ptr_i];
yaz_log (YLOG_FATAL|YLOG_ERRNO, "fclose %s", out_fname);
zebra_exit("key_block_flush");
}
- yaz_log(YLOG_LOG, "finished section %d", p->key_file_no);
+ yaz_log(YLOG_DEBUG, "finished section %d", p->key_file_no);
}
void key_block_flush(zebra_key_block_t p, int is_final)
- /* optimizing: if final=1, and no files written yet */
- /* push the keys directly to merge, sidestepping the */
- /* temp file altogether. Speeds small updates */
{
-#if YAZ_POSIX_THREADS
- char **tmp;
-#endif
if (!p)
return;
+ if (p->use_threads)
+ {
#if YAZ_POSIX_THREADS
- pthread_mutex_lock(&p->mutex);
+ char **tmp;
- while (p->is_sorting)
- pthread_cond_wait(&p->cond_sorting, &p->mutex);
+ pthread_mutex_lock(&p->mutex);
- p->is_sorting = 1;
+ while (p->is_sorting)
+ pthread_cond_wait(&p->cond_sorting, &p->mutex);
- p->thread_ptr_top = p->ptr_top;
- p->thread_ptr_i = p->ptr_i;
- p->thread_key_buf = p->key_buf;
+ p->is_sorting = 1;
- tmp = p->key_buf;
- p->key_buf = p->alt_buf;
- p->alt_buf = tmp;
+ p->thread_ptr_top = p->ptr_top;
+ p->thread_ptr_i = p->ptr_i;
+ p->thread_key_buf = p->key_buf;
- pthread_cond_signal(&p->work_available);
+ tmp = p->key_buf;
+ p->key_buf = p->alt_buf;
+ p->alt_buf = tmp;
- if (is_final)
- {
- while (p->is_sorting)
- pthread_cond_wait(&p->cond_sorting, &p->mutex);
- }
- pthread_mutex_unlock(&p->mutex);
-#else
- key_block_flush_int(p, p->key_buf, p->ptr_top, p->ptr_i);
+ pthread_cond_signal(&p->work_available);
+
+ if (is_final)
+ {
+ while (p->is_sorting)
+ pthread_cond_wait(&p->cond_sorting, &p->mutex);
+ }
+ pthread_mutex_unlock(&p->mutex);
#endif
+ }
+ else
+ key_block_flush_int(p, p->key_buf, p->ptr_top, p->ptr_i);
p->ptr_i = 0;
p->key_buf_used = 0;
}
/*
* Local variables:
* c-basic-offset: 4
+ * c-file-style: "Stroustrup"
* indent-tabs-mode: nil
* End:
* vim: shiftwidth=4 tabstop=8 expandtab
projects / idzebra-moved-to-github.git / blobdiff