MULTILINE_CPP_IS_BRIEF = NO
-# If the DETAILS_AT_TOP tag is set to YES then Doxygen
-# will output the detailed description near the top, like JavaDoc.
-# If set to NO, the detailed description appears after the member
-# documentation.
-
-DETAILS_AT_TOP = NO
-
# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented
# member inherits the documentation from any documented member that it
# re-implements.
+--- 2.0.40 2009/07/13
+
+Fixed problem in result set handling and segment handling.
+
+--- 2.0.39 2009/07/09
+
+Improved speed of record fetching by using a different block sizes
+for recd{A,B}.
+
+Added zlib-compression of recd-storage (record data). This is enabled
+by using recordCompression: zlib in zebra.cfg.
+
+Dictionary delete more agressive (dict_del_string).
+
--- 2.0.38 2009/06/19
Improved estimated hits for AND operation (bug #2907)
dnl Copyright (C) 1994-2009 Index Data
dnl
AC_PREREQ(2.60)
-AC_INIT([idzebra],[2.0.38],[zebra-help@indexdata.dk])
+AC_INIT([idzebra],[2.0.40],[zebra-help@indexdata.dk])
AC_CONFIG_SRCDIR(configure.ac)
AC_CONFIG_AUX_DIR(config)
AM_INIT_AUTOMAKE([1.9])
fi
fi
dnl
+dnl ------- zlib
+AC_CHECK_LIB([z],[compress2])
+if test "$ac_cv_lib_z_compress2" = "yes"; then
+ AC_CHECK_HEADERS([zlib.h])
+fi
dnl ------ -lm
AC_CHECK_LIB(m,sqrt)
dnl ------ -ldl
+idzebra (2.0.40-1indexdata) unstable; urgency=low
+
+ * Upstream.
+
+ -- Adam Dickmeiss <adam@indexdata.dk> Mon, 13 Jul 2009 14:17:31 +0200
+
+idzebra (2.0.39-1indexdata) unstable; urgency=low
+
+ * Upstream.
+
+ -- Adam Dickmeiss <adam@indexdata.dk> Thu, 09 Jul 2009 10:32:04 +0200
+
idzebra (2.0.38-1indexdata) unstable; urgency=low
* Upstream.
Priority: optional
Maintainer: Adam Dickmeiss <adam@indexdata.dk>
Standards-Version: 3.6.0
-Build-Depends: debhelper (>= 4.0.0), libyaz3-dev (>= 3.0.30), libexpat1-dev, tcl8.4-dev, libbz2-dev, libxslt1-dev
+Build-Depends: debhelper (>= 4.0.0), libyaz3-dev (>= 3.0.30), libexpat1-dev, tcl8.4-dev, libbz2-dev, libxslt1-dev, zlib1g-dev
Package: idzebra-2.0
Section: text
#include "dict-p.h"
-int dict_close (Dict dict)
+int dict_close(Dict dict)
{
if (!dict)
return 0;
if (dict->rw)
{
void *head_buf;
- dict_bf_readp (dict->dbf, 0, &head_buf);
- memcpy (head_buf, &dict->head, sizeof(dict->head));
- dict_bf_touch (dict->dbf, 0);
+ dict_bf_readp(dict->dbf, 0, &head_buf);
+ memcpy(head_buf, &dict->head, sizeof(dict->head));
+ dict_bf_touch(dict->dbf, 0);
}
- dict_bf_close (dict->dbf);
- xfree (dict);
+ dict_bf_close(dict->dbf);
+ xfree(dict);
return 0;
}
#include "dict-p.h"
-int dict_bf_close (Dict_BFile dbf)
+int dict_bf_close(Dict_BFile dbf)
{
- dict_bf_flush_blocks (dbf, -1);
+ dict_bf_flush_blocks(dbf, -1);
- xfree (dbf->all_blocks);
- xfree (dbf->all_data);
- xfree (dbf->hash_array);
+ xfree(dbf->all_blocks);
+ xfree(dbf->all_data);
+ xfree(dbf->hash_array);
bf_close(dbf->bf);
- xfree (dbf);
+ xfree(dbf);
return 0;
}
/*
from_indxp = (short*) ((char*) from_p+DICT_bsize(from_p));
to_indxp = (short*) ((char*) to_p+DICT_bsize(to_p));
to_info = (char*) to_p + DICT_infoffset;
- for (i = DICT_nodir (from_p); --i >= 0; )
+ for (i = DICT_nodir(from_p); --i >= 0; )
{
if (*--from_indxp > 0) /* tail string here! */
{
from_info = (char*) from_p + *from_indxp;
*--to_indxp = to_info - to_p;
slen = (dict_strlen((Dict_char*) from_info)+1)*sizeof(Dict_char);
- memcpy (to_info, from_info, slen);
+ memcpy(to_info, from_info, slen);
from_info += slen;
to_info += slen;
}
*--to_indxp = -(to_info - to_p);
from_info = (char*) from_p - *from_indxp;
- memcpy (&subptr, from_info, sizeof(subptr));
+ memcpy(&subptr, from_info, sizeof(subptr));
subptr = map[subptr];
from_info += sizeof(Dict_ptr);
- memcpy (&subchar, from_info, sizeof(subchar));
+ memcpy(&subchar, from_info, sizeof(subchar));
from_info += sizeof(Dict_char);
- memcpy (to_info, &subptr, sizeof(Dict_ptr));
+ memcpy(to_info, &subptr, sizeof(Dict_ptr));
to_info += sizeof(Dict_ptr);
- memcpy (to_info, &subchar, sizeof(Dict_char));
+ memcpy(to_info, &subchar, sizeof(Dict_char));
to_info += sizeof(Dict_char);
}
- assert (to_info < (char*) to_indxp);
+ assert(to_info < (char*) to_indxp);
slen = *from_info+1;
- memcpy (to_info, from_info, slen);
+ memcpy(to_info, from_info, slen);
to_info += slen;
++no;
}
DICT_nodir(to_p) = no;
}
-int dict_copy_compact (BFiles bfs, const char *from_name, const char *to_name)
+int dict_copy_compact(BFiles bfs, const char *from_name, const char *to_name)
{
int no_dir = 0;
Dict dict_from, dict_to;
int *map, i;
- dict_from = dict_open (bfs, from_name, 0, 0, 0, 4096);
+ dict_from = dict_open(bfs, from_name, 0, 0, 0, 4096);
if (!dict_from)
return -1;
- map = (int *) xmalloc ((dict_from->head.last+1) * sizeof(*map));
- for (i = 0; i <= (int) (dict_from->head.last); i++)
+ map = (int *) xmalloc((dict_from->head.last+1) * sizeof(*map));
+ for (i = 0; i <= (int)(dict_from->head.last); i++)
map[i] = -1;
- dict_to = dict_open (bfs, to_name, 0, 1, 1, 4096);
+ dict_to = dict_open(bfs, to_name, 0, 1, 1, 4096);
if (!dict_to)
return -1;
map[0] = 0;
void *buf;
int size;
#if 0
- yaz_log (YLOG_LOG, "map[%d] = %d", i, map[i]);
+ yaz_log(YLOG_LOG, "map[%d] = %d", i, map[i]);
#endif
- dict_bf_readp (dict_from->dbf, i, &buf);
+ dict_bf_readp(dict_from->dbf, i, &buf);
size = ((DICT_size(buf)+sizeof(short)-1)/sizeof(short) +
DICT_nodir(buf))*sizeof(short);
map[i+1] = map[i] + size;
no_dir += DICT_nodir(buf);
}
#if 0
- yaz_log (YLOG_LOG, "map[%d] = %d", i, map[i]);
- yaz_log (YLOG_LOG, "nodir = %d", no_dir);
+ yaz_log(YLOG_LOG, "map[%d] = %d", i, map[i]);
+ yaz_log(YLOG_LOG, "nodir = %d", no_dir);
#endif
dict_to->head.root = map[1];
dict_to->head.last = map[i];
for (i = 1; i< (int) (dict_from->head.last); i++)
{
void *old_p, *new_p;
- dict_bf_readp (dict_from->dbf, i, &old_p);
+ dict_bf_readp(dict_from->dbf, i, &old_p);
- yaz_log (YLOG_LOG, "dict_bf_newp no=%d size=%d", map[i],
+ yaz_log(YLOG_LOG, "dict_bf_newp no=%d size=%d", map[i],
map[i+1] - map[i]);
- dict_bf_newp (dict_to->dbf, map[i], &new_p, map[i+1] - map[i]);
+ dict_bf_newp(dict_to->dbf, map[i], &new_p, map[i+1] - map[i]);
DICT_type(new_p) = 0;
DICT_backptr(new_p) = map[i-1];
dict_copy_page(dict_from, (char*) new_p, (char*) old_p, map);
}
- dict_close (dict_from);
- dict_close (dict_to);
+ dict_close(dict_from);
+ dict_close(dict_to);
return 0;
}
/*
#include "dict-p.h"
-static void dict_del_subtree (Dict dict, Dict_ptr ptr,
- void *client,
- int (*f)(const char *, void *))
+static void dict_del_subtree(Dict dict, Dict_ptr ptr,
+ void *client,
+ int (*f)(const char *, void *))
{
void *p = 0;
short *indxp;
if (!ptr)
return;
- dict_bf_readp (dict->dbf, ptr, &p);
+ dict_bf_readp(dict->dbf, ptr, &p);
indxp = (short*) ((char*) p+DICT_bsize(p)-sizeof(short));
hi = DICT_nodir(p)-1;
for (i = 0; i <= hi; i++)
/* unsigned char length of information */
/* char * information */
char *info = (char*)p - indxp[-i];
- memcpy (&subptr, info, sizeof(Dict_ptr));
+ memcpy(&subptr, info, sizeof(Dict_ptr));
if (info[sizeof(Dict_ptr)+sizeof(Dict_char)])
{
}
if (subptr)
{
- dict_del_subtree (dict, subptr, client, f);
+ dict_del_subtree(dict, subptr, client, f);
/* page may be gone. reread it .. */
- dict_bf_readp (dict->dbf, ptr, &p);
+ dict_bf_readp(dict->dbf, ptr, &p);
indxp = (short*) ((char*) p+DICT_bsize(p)-sizeof(short));
}
}
}
DICT_backptr(p) = dict->head.freelist;
dict->head.freelist = ptr;
- dict_bf_touch (dict->dbf, ptr);
+ dict_bf_touch(dict->dbf, ptr);
}
-static int dict_del_string (Dict dict, const Dict_char *str, Dict_ptr ptr,
- int sub_flag, void *client,
- int (*f)(const char *, void *))
+static int dict_del_string(Dict dict, const Dict_char *str, Dict_ptr ptr,
+ int sub_flag, void *client,
+ int (*f)(const char *, void *))
{
int mid, lo, hi;
int cmp;
void *p;
short *indxp;
char *info;
+ int r = 0;
+ Dict_ptr subptr = 0;
if (!ptr)
return 0;
- dict_bf_readp (dict->dbf, ptr, &p);
+ dict_bf_readp(dict->dbf, ptr, &p);
mid = lo = 0;
hi = DICT_nodir(p)-1;
indxp = (short*) ((char*) p+DICT_bsize(p)-sizeof(short));
if (sub_flag)
{
/* determine if prefix match */
- if (!dict_strncmp (str, (Dict_char*) info, dict_strlen(str)))
+ if (!dict_strncmp(str, (Dict_char*) info, dict_strlen(str)))
{
if (f)
(*f)(info + (dict_strlen((Dict_char*) info)+1)
}
DICT_type(p) = 1;
(DICT_nodir(p))--;
- dict_bf_touch (dict->dbf, ptr);
+ dict_bf_touch(dict->dbf, ptr);
--hi;
mid = lo = 0;
+ r = 1; /* signal deleted */
/* start again (may not be the most efficient way to go)*/
continue;
}
}
DICT_type(p) = 1;
(DICT_nodir(p))--;
- dict_bf_touch (dict->dbf, ptr);
- return 1;
+ dict_bf_touch(dict->dbf, ptr);
+ r = 1;
+ break;
}
}
}
else
{
Dict_char dc;
- Dict_ptr subptr;
/* Dict_ptr subptr */
/* Dict_char sub char */
/* unsigned char length of information */
/* char * information */
info = (char*)p - indxp[-mid];
- memcpy (&dc, info+sizeof(Dict_ptr), sizeof(Dict_char));
+ memcpy(&dc, info+sizeof(Dict_ptr), sizeof(Dict_char));
cmp = dc- *str;
if (!cmp)
{
- memcpy (&subptr, info, sizeof(Dict_ptr));
+ memcpy(&subptr, info, sizeof(Dict_ptr));
if (*++str == DICT_EOS)
{
- if (sub_flag && subptr)
- {
- Dict null_ptr = 0;
- memcpy (info, &null_ptr, sizeof(Dict_ptr));
- }
if (info[sizeof(Dict_ptr)+sizeof(Dict_char)])
{
+ /* entry does exist. Wipe it out */
info[sizeof(Dict_ptr)+sizeof(Dict_char)] = 0;
DICT_type(p) = 1;
- dict_bf_touch (dict->dbf, ptr);
+ dict_bf_touch(dict->dbf, ptr);
if (f)
(*f)(info+sizeof(Dict_ptr)+sizeof(Dict_char),
client);
- if (sub_flag && subptr)
- dict_del_subtree (dict, subptr, client, f);
- return 1;
+ r = 1;
}
- if (sub_flag && subptr)
+ if (sub_flag)
{
+ /* must delete all suffixes (subtrees) as well */
+ hi = DICT_nodir(p)-1;
+ while (mid < hi)
+ {
+ indxp[-mid] = indxp[-mid-1];
+ mid++;
+ }
+ (DICT_nodir(p))--;
DICT_type(p) = 1;
- dict_bf_touch (dict->dbf, ptr);
- dict_del_subtree (dict, subptr, client, f);
+ dict_bf_touch(dict->dbf, ptr);
}
- return 0;
}
else
{
- if (subptr == 0)
- return 0;
- ptr = subptr;
- dict_bf_readp (dict->dbf, ptr, &p);
- mid = lo = 0;
- hi = DICT_nodir(p)-1;
- indxp = (short*) ((char*) p+DICT_bsize(p)-sizeof(short));
- continue;
+ /* subptr may be 0 */
+ r = dict_del_string(dict, str, subptr, sub_flag, client, f);
+
+ /* recover */
+ dict_bf_readp(dict->dbf, ptr, &p);
+ indxp = (short*)
+ ((char*) p+DICT_bsize(p)-sizeof(short));
+ info = (char*)p - indxp[-mid];
+
+ if (r == 2)
+ { /* subptr page is empty and already removed */
+ hi = DICT_nodir(p)-1;
+ while (mid < hi)
+ {
+ indxp[-mid] = indxp[-mid-1];
+ mid++;
+ }
+ (DICT_nodir(p))--;
+ dict_bf_touch(dict->dbf, ptr);
+ r = 1;
+ }
+ subptr = 0; /* prevent dict_del_subtree (below) */
}
+ break;
}
}
if (cmp < 0)
else
hi = mid-1;
}
- return 0;
+ if (DICT_nodir(p) == 0 && ptr != dict->head.root)
+ {
+ DICT_backptr(p) = dict->head.freelist;
+ dict->head.freelist = ptr;
+ dict_bf_touch(dict->dbf, ptr);
+ r = 2;
+ }
+ if (subptr && sub_flag)
+ dict_del_subtree(dict, subptr, client, f);
+
+ return r;
}
-int dict_delete (Dict dict, const char *p)
+int dict_delete(Dict dict, const char *p)
{
- return dict_del_string (dict, (const Dict_char*) p, dict->head.root, 0,
- 0, 0);
+ return dict_del_string(dict, (const Dict_char*) p, dict->head.root, 0,
+ 0, 0);
}
-int dict_delete_subtree (Dict dict, const char *p, void *client,
- int (*f)(const char *info, void *client))
+int dict_delete_subtree(Dict dict, const char *p, void *client,
+ int (*f)(const char *info, void *client))
{
- return dict_del_string (dict, (const Dict_char*) p, dict->head.root, 1,
- client, f);
+ return dict_del_string(dict, (const Dict_char*) p, dict->head.root, 1,
+ client, f);
}
/*
* Local variables:
char *prog;
-int main (int argc, char **argv)
+int main(int argc, char **argv)
{
char *arg;
int ret;
inputfile = arg;
else
{
- yaz_log (YLOG_FATAL, "too many files specified\n");
- exit (1);
+ yaz_log(YLOG_FATAL, "too many files specified\n");
+ exit(1);
}
}
else if (ret == 'v')
{
- yaz_log_init (yaz_log_mask_str(arg), prog, NULL);
+ yaz_log_init(yaz_log_mask_str(arg), prog, NULL);
}
else if (ret == 'h')
{
- fprintf (stderr, "usage:\n"
- " %s [-8] [-h] [-v n] [file]\n", prog);
- exit (1);
+ fprintf(stderr, "usage:\n"
+ " %s [-8] [-h] [-v n] [file]\n", prog);
+ exit(1);
}
else if (ret == '8')
use8 = 1;
else
{
- yaz_log (YLOG_FATAL, "Unknown option '-%s'", arg);
- exit (1);
+ yaz_log(YLOG_FATAL, "Unknown option '-%s'", arg);
+ exit(1);
}
}
if (inputfile)
{
- ipf = fopen (inputfile, "r");
+ ipf = fopen(inputfile, "r");
if (!ipf)
{
- yaz_log (YLOG_FATAL|YLOG_ERRNO, "cannot open '%s'", inputfile);
- exit (1);
+ yaz_log(YLOG_FATAL|YLOG_ERRNO, "cannot open '%s'", inputfile);
+ exit(1);
}
}
- while (fgets (ipf_buf, 1023, ipf))
+ while (fgets(ipf_buf, 1023, ipf))
{
char *ipf_ptr = ipf_buf;
for (;*ipf_ptr && *ipf_ptr != '\n';ipf_ptr++)
i++;
if (ipf_ptr[i])
ipf_ptr[i++] = '\0';
- printf ("%s\n", ipf_ptr);
+ printf("%s\n", ipf_ptr);
ipf_ptr += (i-1);
}
}
#include "dict-p.h"
-static void common_init (Dict_BFile bf, int block_size, int cache)
+static void common_init(Dict_BFile bf, int block_size, int cache)
{
int i;
bf->hits = bf->misses = 0;
/* Allocate all blocks in one chunk. */
- bf->all_data = xmalloc (block_size * cache);
+ bf->all_data = xmalloc(block_size * cache);
/* Allocate and initialize hash array (as empty) */
bf->hash_array = (struct Dict_file_block **)
/* Allocate all block descriptors in one chunk */
bf->all_blocks = (struct Dict_file_block *)
- xmalloc (sizeof(*bf->all_blocks) * cache);
+ xmalloc(sizeof(*bf->all_blocks) * cache);
/* Initialize the free list */
bf->free_list = bf->all_blocks;
}
-Dict_BFile dict_bf_open (BFiles bfs, const char *name, int block_size,
- int cache, int rw)
+Dict_BFile dict_bf_open(BFiles bfs, const char *name, int block_size,
+ int cache, int rw)
{
Dict_BFile dbf;
-
- dbf = (Dict_BFile) xmalloc (sizeof(*dbf));
- dbf->bf = bf_open (bfs, name, block_size, rw);
+
+ dbf = (Dict_BFile) xmalloc(sizeof(*dbf));
+ dbf->bf = bf_open(bfs, name, block_size, rw);
if (!dbf->bf)
{
xfree(dbf);
return 0;
}
- common_init (dbf, block_size, cache);
+ common_init(dbf, block_size, cache);
return dbf;
}
-void dict_bf_compact (Dict_BFile dbf)
+void dict_bf_compact(Dict_BFile dbf)
{
dbf->compact_flag = 1;
}
/* This file is part of the Zebra server.
Copyright (C) 1994-2009 Index Data
-Zebra is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free
-Software Foundation; either version 2, or (at your option) any later
-version.
+ Zebra is free software; you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free
+ Software Foundation; either version 2, or (at your option) any later
+ version.
-Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
-WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-for more details.
+ Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ for more details.
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "dict-p.h"
-void dict_pr_lru (Dict_BFile bf)
+void dict_pr_lru(Dict_BFile bf)
{
struct Dict_file_block *p;
for (p=bf->lru_back; p; p = p->lru_next)
{
- printf (" %d", p->no);
+ printf(" %d", p->no);
}
- printf ("\n");
- fflush (stdout);
+ printf("\n");
+ fflush(stdout);
}
-static struct Dict_file_block *find_block (Dict_BFile bf, int no)
+static struct Dict_file_block *find_block(Dict_BFile bf, int no)
{
struct Dict_file_block *p;
return p;
}
-static void release_block (Dict_BFile bf, struct Dict_file_block *p)
+static void release_block(Dict_BFile bf, struct Dict_file_block *p)
{
- assert (p);
+ assert(p);
/* remove from lru queue */
if (p->lru_prev)
bf->free_list = p;
}
-void dict_bf_flush_blocks (Dict_BFile bf, int no_to_flush)
+void dict_bf_flush_blocks(Dict_BFile bf, int no_to_flush)
{
struct Dict_file_block *p;
int i;
if (p->dirty)
{
if (!bf->compact_flag)
- bf_write (bf->bf, p->no, 0, 0, p->data);
+ bf_write(bf->bf, p->no, 0, 0, p->data);
else
{
int effective_block = p->no / bf->block_size;
if (remain >= p->nbytes)
{
- bf_write (bf->bf, effective_block, effective_offset,
- p->nbytes, p->data);
+ bf_write(bf->bf, effective_block, effective_offset,
+ p->nbytes, p->data);
#if 0
- yaz_log (YLOG_LOG, "bf_write no=%d offset=%d size=%d",
- effective_block, effective_offset,
- p->nbytes);
+ yaz_log(YLOG_LOG, "bf_write no=%d offset=%d size=%d",
+ effective_block, effective_offset,
+ p->nbytes);
#endif
}
else
{
#if 0
- yaz_log (YLOG_LOG, "bf_write1 no=%d offset=%d size=%d",
- effective_block, effective_offset,
- remain);
+ yaz_log(YLOG_LOG, "bf_write1 no=%d offset=%d size=%d",
+ effective_block, effective_offset,
+ remain);
#endif
- bf_write (bf->bf, effective_block, effective_offset,
- remain, p->data);
+ bf_write(bf->bf, effective_block, effective_offset,
+ remain, p->data);
#if 0
- yaz_log (YLOG_LOG, "bf_write2 no=%d offset=%d size=%d",
- effective_block+1, 0, p->nbytes - remain);
+ yaz_log(YLOG_LOG, "bf_write2 no=%d offset=%d size=%d",
+ effective_block+1, 0, p->nbytes - remain);
#endif
- bf_write (bf->bf, effective_block+1, 0,
- p->nbytes - remain, (char*)p->data + remain);
+ bf_write(bf->bf, effective_block+1, 0,
+ p->nbytes - remain, (char*)p->data + remain);
}
}
}
- release_block (bf, p);
+ release_block(bf, p);
}
}
-static struct Dict_file_block *alloc_block (Dict_BFile bf, int no)
+static struct Dict_file_block *alloc_block(Dict_BFile bf, int no)
{
struct Dict_file_block *p, **pp;
if (!bf->free_list)
- dict_bf_flush_blocks (bf, 1);
- assert (bf->free_list);
+ dict_bf_flush_blocks(bf, 1);
+ assert(bf->free_list);
p = bf->free_list;
bf->free_list = p->h_next;
p->dirty = 0;
return p;
}
-static void move_to_front (Dict_BFile bf, struct Dict_file_block *p)
+static void move_to_front(Dict_BFile bf, struct Dict_file_block *p)
{
/* Already at front? */
if (!p->lru_next)
bf->lru_front = p;
}
-int dict_bf_readp (Dict_BFile bf, int no, void **bufp)
+int dict_bf_readp(Dict_BFile bf, int no, void **bufp)
{
struct Dict_file_block *p;
int i;
- if ((p = find_block (bf, no)))
+ if ((p = find_block(bf, no)))
{
*bufp = p->data;
- move_to_front (bf, p);
+ move_to_front(bf, p);
bf->hits++;
return 1;
}
bf->misses++;
- p = alloc_block (bf, no);
+ p = alloc_block(bf, no);
if (!bf->compact_flag)
- i = bf_read (bf->bf, no, 0, 0, p->data);
+ i = bf_read(bf->bf, no, 0, 0, p->data);
else
{
int effective_block = no / bf->block_size;
int effective_offset = no - effective_block * bf->block_size;
- i = bf_read (bf->bf, effective_block, effective_offset,
- bf->block_size - effective_offset, p->data);
+ i = bf_read(bf->bf, effective_block, effective_offset,
+ bf->block_size - effective_offset, p->data);
if (i > 0 && effective_offset > 0)
- i = bf_read (bf->bf, effective_block+1, 0, effective_offset,
- (char*) p->data + bf->block_size - effective_offset);
+ i = bf_read(bf->bf, effective_block+1, 0, effective_offset,
+ (char*) p->data + bf->block_size - effective_offset);
i = 1;
}
if (i > 0)
*bufp = p->data;
return i;
}
- release_block (bf, p);
+ release_block(bf, p);
*bufp = NULL;
return i;
}
-int dict_bf_newp (Dict_BFile dbf, int no, void **bufp, int nbytes)
+int dict_bf_newp(Dict_BFile dbf, int no, void **bufp, int nbytes)
{
struct Dict_file_block *p;
- if (!(p = find_block (dbf, no)))
- p = alloc_block (dbf, no);
+ if (!(p = find_block(dbf, no)))
+ p = alloc_block(dbf, no);
else
- move_to_front (dbf, p);
+ move_to_front(dbf, p);
*bufp = p->data;
- memset (p->data, 0, dbf->block_size);
+ memset(p->data, 0, dbf->block_size);
p->dirty = 1;
p->nbytes = nbytes;
#if 0
- printf ("bf_newp of %d:", no);
- dict_pr_lru (dbf);
+ printf("bf_newp of %d:", no);
+ dict_pr_lru(dbf);
#endif
return 1;
}
-int dict_bf_touch (Dict_BFile dbf, int no)
+int dict_bf_touch(Dict_BFile dbf, int no)
{
struct Dict_file_block *p;
- if ((p = find_block (dbf, no)))
+ if ((p = find_block(dbf, no)))
{
p->dirty = 1;
return 0;
#define CHECK 0
-static int dict_ins (Dict dict, const Dict_char *str,
- Dict_ptr back_ptr, int userlen, void *userinfo);
-static void clean_page (Dict dict, Dict_ptr ptr, void *p, Dict_char *out,
- Dict_ptr subptr, char *userinfo);
+static int dict_ins(Dict dict, const Dict_char *str,
+ Dict_ptr back_ptr, int userlen, void *userinfo);
+static void clean_page(Dict dict, Dict_ptr ptr, void *p, Dict_char *out,
+ Dict_ptr subptr, char *userinfo);
-static Dict_ptr new_page (Dict dict, Dict_ptr back_ptr, void **pp)
+static Dict_ptr new_page(Dict dict, Dict_ptr back_ptr, void **pp)
{
void *p;
Dict_ptr ptr = dict->head.last;
if (!dict->head.freelist)
{
- dict_bf_newp (dict->dbf, dict->head.last, &p, dict->head.page_size);
+ dict_bf_newp(dict->dbf, dict->head.last, &p, dict->head.page_size);
(dict->head.last)++;
}
else
{
ptr = dict->head.freelist;
- dict_bf_readp (dict->dbf, ptr, &p);
+ dict_bf_readp(dict->dbf, ptr, &p);
dict->head.freelist = DICT_backptr(p);
}
- assert (p);
+ assert(p);
DICT_type(p) = 0;
DICT_backptr(p) = back_ptr;
DICT_nodir(p) = 0;
return ptr;
}
-static int split_page (Dict dict, Dict_ptr ptr, void *p)
+static int split_page(Dict dict, Dict_ptr ptr, void *p)
{
void *subp;
char *info_here;
dict->no_split++;
/* determine splitting char... */
indxp = (short*) ((char*) p+DICT_bsize(p)-sizeof(short));
- for (i = DICT_nodir (p); --i >= 0; --indxp)
+ for (i = DICT_nodir(p); --i >= 0; --indxp)
{
if (*indxp > 0) /* tail string here! */
{
Dict_char dc;
- memcpy (&dc, (char*) p + *indxp, sizeof(dc));
+ memcpy(&dc, (char*) p + *indxp, sizeof(dc));
if (best_no < 0)
{ /* first entry met */
best_char = prev_char = dc;
assert(best_no >= 0); /* we didn't find any tail string entry at all! */
j = best_indxp - (short*) p;
- subptr = new_page (dict, ptr, &subp);
+ subptr = new_page(dict, ptr, &subp);
/* scan entries to see if there is a string with */
/* length 1. info_here indicates if such entry exist */
info_here = NULL;
info = (char*) p + ((short*) p)[j];
/* entry start */
- memcpy (&dc, info, sizeof(dc));
- assert (dc == best_char);
+ memcpy(&dc, info, sizeof(dc));
+ assert(dc == best_char);
slen = 1+dict_strlen((Dict_char*) info);
- assert (slen > 1);
+ assert(slen > 1);
if (slen == 2)
{
- assert (!info_here);
+ assert(!info_here);
info_here = info+slen*sizeof(Dict_char);
}
else
{
info1 = info+slen*sizeof(Dict_char); /* info start */
- dict_ins (dict, (Dict_char*) (info+sizeof(Dict_char)),
- subptr, *info1, info1+1);
- dict_bf_readp (dict->dbf, ptr, &p);
+ dict_ins(dict, (Dict_char*) (info+sizeof(Dict_char)),
+ subptr, *info1, info1+1);
+ dict_bf_readp(dict->dbf, ptr, &p);
}
}
/* now clean the page ... */
- clean_page (dict, ptr, p, &best_char, subptr, info_here);
+ clean_page(dict, ptr, p, &best_char, subptr, info_here);
return 0;
}
-static void clean_page (Dict dict, Dict_ptr ptr, void *p, Dict_char *out,
- Dict_ptr subptr, char *userinfo)
+static void clean_page(Dict dict, Dict_ptr ptr, void *p, Dict_char *out,
+ Dict_ptr subptr, char *userinfo)
{
- char *np = (char *) xmalloc (dict->head.page_size);
+ char *np = (char *) xmalloc(dict->head.page_size);
int i, slen, no = 0;
short *indxp1, *indxp2;
char *info1, *info2;
indxp1 = (short*) ((char*) p+DICT_bsize(p)-sizeof(short));
indxp2 = (short*) ((char*) np+DICT_bsize(np));
info2 = (char*) np + DICT_infoffset;
- for (i = DICT_nodir (p); --i >= 0; --indxp1)
+ for (i = DICT_nodir(p); --i >= 0; --indxp1)
{
if (*indxp1 > 0) /* tail string here! */
{
/* char * information */
info1 = (char*) p + *indxp1;
- if (out && memcmp (out, info1, sizeof(Dict_char)) == 0)
+ if (out && memcmp(out, info1, sizeof(Dict_char)) == 0)
{
if (subptr == 0)
continue;
*--indxp2 = -(info2 - np);
- memcpy (info2, &subptr, sizeof(Dict_ptr));
+ memcpy(info2, &subptr, sizeof(Dict_ptr));
info2 += sizeof(Dict_ptr);
- memcpy (info2, out, sizeof(Dict_char));
+ memcpy(info2, out, sizeof(Dict_char));
info2 += sizeof(Dict_char);
if (userinfo)
{
- memcpy (info2, userinfo, *userinfo+1);
+ memcpy(info2, userinfo, *userinfo+1);
info2 += *userinfo + 1;
}
else
}
*--indxp2 = info2 - np;
slen = (dict_strlen((Dict_char*) info1)+1)*sizeof(Dict_char);
- memcpy (info2, info1, slen);
+ memcpy(info2, info1, slen);
info1 += slen;
info2 += slen;
}
/* unsigned char length of information */
/* char * information */
- assert (*indxp1 < 0);
+ assert(*indxp1 < 0);
*--indxp2 = -(info2 - np);
info1 = (char*) p - *indxp1;
- memcpy (info2, info1, sizeof(Dict_ptr)+sizeof(Dict_char));
+ memcpy(info2, info1, sizeof(Dict_ptr)+sizeof(Dict_char));
info1 += sizeof(Dict_ptr)+sizeof(Dict_char);
info2 += sizeof(Dict_ptr)+sizeof(Dict_char);
}
slen = *info1+1;
- memcpy (info2, info1, slen);
+ memcpy(info2, info1, slen);
info2 += slen;
++no;
}
#if 1
- memcpy ((char*)p+DICT_infoffset,
- (char*)np+DICT_infoffset,
- info2 - ((char*)np+DICT_infoffset));
- memcpy ((char*)p + ((char*)indxp2 - (char*)np),
- indxp2,
- ((char*) np+DICT_bsize(p)) - (char*)indxp2);
+ memcpy((char*)p+DICT_infoffset,
+ (char*)np+DICT_infoffset,
+ info2 - ((char*)np+DICT_infoffset));
+ memcpy((char*)p + ((char*)indxp2 - (char*)np),
+ indxp2,
+ ((char*) np+DICT_bsize(p)) - (char*)indxp2);
#else
- memcpy ((char*)p+DICT_infoffset, (char*)np+DICT_infoffset,
- DICT_pagesize(dict)-DICT_infoffset);
+ memcpy((char*)p+DICT_infoffset, (char*)np+DICT_infoffset,
+ DICT_pagesize(dict)-DICT_infoffset);
#endif
DICT_size(p) = info2 - np;
DICT_type(p) = 0;
DICT_nodir(p) = no;
- xfree (np);
- dict_bf_touch (dict->dbf, ptr);
+ xfree(np);
+ dict_bf_touch(dict->dbf, ptr);
}
/* return 1 if before but change of info */
/* return 2 if same as before */
-static int dict_ins (Dict dict, const Dict_char *str,
- Dict_ptr ptr, int userlen, void *userinfo)
+static int dict_ins(Dict dict, const Dict_char *str,
+ Dict_ptr ptr, int userlen, void *userinfo)
{
int hi, lo, mid, slen, cmp = 1;
short *indxp;
char *info;
void *p;
- dict_bf_readp (dict->dbf, ptr, &p);
+ dict_bf_readp(dict->dbf, ptr, &p);
- assert (p);
- assert (ptr);
+ assert(p);
+ assert(ptr);
mid = lo = 0;
hi = DICT_nodir(p)-1;
if (*info == userlen)
{
/* change of userinfo ? */
- if (memcmp (info+1, userinfo, userlen))
+ if (memcmp(info+1, userinfo, userlen))
{
- dict_bf_touch (dict->dbf, ptr);
- memcpy (info+1, userinfo, userlen);
+ dict_bf_touch(dict->dbf, ptr);
+ memcpy(info+1, userinfo, userlen);
return 1;
}
/* same userinfo */
/* room for new userinfo */
DICT_type(p) = 1;
*info = userlen;
- dict_bf_touch (dict->dbf, ptr);
- memcpy (info+1, userinfo, userlen);
+ dict_bf_touch(dict->dbf, ptr);
+ memcpy(info+1, userinfo, userlen);
return 1;
}
break;
/* unsigned char length of information */
/* char * information */
info = (char*)p - indxp[-mid];
- memcpy (&dc, info+sizeof(Dict_ptr), sizeof(Dict_char));
+ memcpy(&dc, info+sizeof(Dict_ptr), sizeof(Dict_char));
cmp = dc- *str;
if (!cmp)
{
- memcpy (&subptr, info, sizeof(Dict_ptr));
+ memcpy(&subptr, info, sizeof(Dict_ptr));
if (*++str == DICT_EOS)
{
/* finish of string. Store userinfo here... */
int xlen = info[sizeof(Dict_ptr)+sizeof(Dict_char)];
if (xlen == userlen)
{
- if (memcmp (info+sizeof(Dict_ptr)+sizeof(Dict_char)+1,
- userinfo, userlen))
+ if (memcmp(info+sizeof(Dict_ptr)+sizeof(Dict_char)+1,
+ userinfo, userlen))
{
- dict_bf_touch (dict->dbf, ptr);
- memcpy (info+sizeof(Dict_ptr)+sizeof(Dict_char)+1,
- userinfo, userlen);
+ dict_bf_touch(dict->dbf, ptr);
+ memcpy(info+sizeof(Dict_ptr)+sizeof(Dict_char)+1,
+ userinfo, userlen);
return 1;
}
return 2;
{
DICT_type(p) = 1;
info[sizeof(Dict_ptr)+sizeof(Dict_char)] = userlen;
- memcpy (info+sizeof(Dict_ptr)+sizeof(Dict_char)+1,
- userinfo, userlen);
- dict_bf_touch (dict->dbf, ptr);
+ memcpy(info+sizeof(Dict_ptr)+sizeof(Dict_char)+1,
+ userinfo, userlen);
+ dict_bf_touch(dict->dbf, ptr);
return 1;
}
/* xlen < userlen, expanding needed ... */
/* not enough room - split needed ... */
if (DICT_type(p) == 1)
{
- clean_page (dict, ptr, p, NULL, 0, NULL);
- return dict_ins (dict, str-1, ptr,
- userlen, userinfo);
+ clean_page(dict, ptr, p, NULL, 0, NULL);
+ return dict_ins(dict, str-1, ptr,
+ userlen, userinfo);
}
- if (split_page (dict, ptr, p))
+ if (split_page(dict, ptr, p))
{
- yaz_log (YLOG_FATAL, "Unable to split page %d\n", ptr);
+ yaz_log(YLOG_FATAL, "Unable to split page %d\n", ptr);
assert(0);
}
- return dict_ins (dict, str-1, ptr, userlen, userinfo);
+ return dict_ins(dict, str-1, ptr, userlen, userinfo);
}
else
{ /* enough room - no split needed ... */
info = (char*)p + DICT_size(p);
- memcpy (info, &subptr, sizeof(subptr));
- memcpy (info+sizeof(Dict_ptr), &dc, sizeof(Dict_char));
+ memcpy(info, &subptr, sizeof(subptr));
+ memcpy(info+sizeof(Dict_ptr), &dc, sizeof(Dict_char));
info[sizeof(Dict_char)+sizeof(Dict_ptr)] = userlen;
- memcpy (info+sizeof(Dict_char)+sizeof(Dict_ptr)+1,
- userinfo, userlen);
+ memcpy(info+sizeof(Dict_char)+sizeof(Dict_ptr)+1,
+ userinfo, userlen);
indxp[-mid] = -DICT_size(p);
DICT_size(p) += sizeof(Dict_char)+sizeof(Dict_ptr)
+1+userlen;
DICT_type(p) = 1;
- dict_bf_touch (dict->dbf, ptr);
+ dict_bf_touch(dict->dbf, ptr);
}
if (xlen)
return 1;
{
if (subptr == 0)
{
- subptr = new_page (dict, ptr, NULL);
- memcpy (info, &subptr, sizeof(subptr));
- dict_bf_touch (dict->dbf, ptr);
+ subptr = new_page(dict, ptr, NULL);
+ memcpy(info, &subptr, sizeof(subptr));
+ dict_bf_touch(dict->dbf, ptr);
}
- return dict_ins (dict, str, subptr, userlen, userinfo);
+ return dict_ins(dict, str, subptr, userlen, userinfo);
}
}
}
{
if (DICT_type(p))
{
- clean_page (dict, ptr, p, NULL, 0, NULL);
- return dict_ins (dict, str, ptr, userlen, userinfo);
+ clean_page(dict, ptr, p, NULL, 0, NULL);
+ return dict_ins(dict, str, ptr, userlen, userinfo);
}
- split_page (dict, ptr, p);
- return dict_ins (dict, str, ptr, userlen, userinfo);
+ split_page(dict, ptr, p);
+ return dict_ins(dict, str, ptr, userlen, userinfo);
}
if (cmp)
{
else
DICT_type(p) = 1;
info = (char*)p + DICT_size(p);
- memcpy (info, str, slen);
+ memcpy(info, str, slen);
info += slen;
*info++ = userlen;
- memcpy (info, userinfo, userlen);
+ memcpy(info, userinfo, userlen);
info += userlen;
*indxp = DICT_size(p);
DICT_size(p) = info- (char*) p;
- dict_bf_touch (dict->dbf, ptr);
+ dict_bf_touch(dict->dbf, ptr);
if (cmp)
return 0;
return 1;
}
-int dict_insert (Dict dict, const char *str, int userlen, void *userinfo)
+int dict_insert(Dict dict, const char *str, int userlen, void *userinfo)
{
if (!dict->rw)
return -1;
if (!dict->head.root)
{
void *p;
- dict->head.root = new_page (dict, 0, &p);
+ dict->head.root = new_page(dict, 0, &p);
if (!dict->head.root)
return -1;
}
- return dict_ins (dict, (const Dict_char *) str, dict->head.root,
- userlen, userinfo);
+ return dict_ins(dict, (const Dict_char *) str, dict->head.root,
+ userlen, userinfo);
}
/*
/* This file is part of the Zebra server.
Copyright (C) 1994-2009 Index Data
-Zebra is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free
-Software Foundation; either version 2, or (at your option) any later
-version.
+ Zebra is free software; you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free
+ Software Foundation; either version 2, or (at your option) any later
+ version.
-Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
-WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-for more details.
+ Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ for more details.
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
*/
typedef struct {
- int n; /* no of MatchWord needed */
- int range; /* max no. of errors */
- int fact; /* (range+1)*n */
- MatchWord *match_mask; /* match_mask */
+ int n; /* no of MatchWord needed */
+ int range; /* max no. of errors */
+ int fact; /* (range+1)*n */
+ MatchWord *match_mask; /* match_mask */
} MatchContext;
#define INLINE
-static INLINE void set_bit (MatchContext *mc, MatchWord *m, int ch, int state)
+static INLINE void set_bit(MatchContext *mc, MatchWord *m, int ch, int state)
{
int off = state & (WORD_BITS-1);
int wno = state / WORD_BITS;
m[mc->n * ch + wno] |= 1<<off;
}
-static INLINE MatchWord get_bit (MatchContext *mc, MatchWord *m, int ch,
- int state)
+static INLINE MatchWord get_bit(MatchContext *mc, MatchWord *m, int ch,
+ int state)
{
int off = state & (WORD_BITS-1);
int wno = state / WORD_BITS;
return m[mc->n * ch + wno] & (1<<off);
}
-static MatchContext *mk_MatchContext (struct DFA *dfa, int range)
+static MatchContext *mk_MatchContext(struct DFA *dfa, int range)
{
- MatchContext *mc = (MatchContext *) xmalloc (sizeof(*mc));
+ MatchContext *mc = (MatchContext *) xmalloc(sizeof(*mc));
int s;
mc->n = (dfa->no_states+WORD_BITS) / WORD_BITS;
mc->range = range;
mc->fact = (range+1)*mc->n;
- mc->match_mask = (MatchWord *) xcalloc (mc->n, sizeof(*mc->match_mask));
+ mc->match_mask = (MatchWord *) xcalloc(mc->n, sizeof(*mc->match_mask));
for (s = 0; s<dfa->no_states; s++)
if (dfa->states[s]->rule_no)
- set_bit (mc, mc->match_mask, 0, s);
+ set_bit(mc, mc->match_mask, 0, s);
return mc;
}
-static void rm_MatchContext (MatchContext **mc)
+static void rm_MatchContext(MatchContext **mc)
{
- xfree ((*mc)->match_mask);
- xfree (*mc);
+ xfree((*mc)->match_mask);
+ xfree(*mc);
*mc = NULL;
}
-static void mask_shift (MatchContext *mc, MatchWord *Rdst, MatchWord *Rsrc,
- struct DFA *dfa, int ch)
+static void mask_shift(MatchContext *mc, MatchWord *Rdst, MatchWord *Rsrc,
+ struct DFA *dfa, int ch)
{
int j, s = 0;
MatchWord *Rsrc_p = Rsrc, mask;
while (--i >= 0)
if (ch >= state->trans[i].ch[0] &&
ch <= state->trans[i].ch[1])
- set_bit (mc, Rdst, 0, state->trans[i].to);
+ set_bit(mc, Rdst, 0, state->trans[i].to);
}
if (mask & 2)
{
while (--i >= 0)
if (ch >= state->trans[i].ch[0] &&
ch <= state->trans[i].ch[1])
- set_bit (mc, Rdst, 0, state->trans[i].to);
+ set_bit(mc, Rdst, 0, state->trans[i].to);
}
if (mask & 4)
{
while (--i >= 0)
if (ch >= state->trans[i].ch[0] &&
ch <= state->trans[i].ch[1])
- set_bit (mc, Rdst, 0, state->trans[i].to);
+ set_bit(mc, Rdst, 0, state->trans[i].to);
}
if (mask & 8)
{
while (--i >= 0)
if (ch >= state->trans[i].ch[0] &&
ch <= state->trans[i].ch[1])
- set_bit (mc, Rdst, 0, state->trans[i].to);
+ set_bit(mc, Rdst, 0, state->trans[i].to);
}
}
s += 4;
}
}
-static void shift (MatchContext *mc, MatchWord *Rdst, MatchWord *Rsrc,
- struct DFA *dfa)
+static void shift(MatchContext *mc, MatchWord *Rdst, MatchWord *Rsrc,
+ struct DFA *dfa)
{
int j, s = 0;
MatchWord *Rsrc_p = Rsrc, mask;
struct DFA_state *state = dfa->states[s];
int i = state->tran_no;
while (--i >= 0)
- set_bit (mc, Rdst, 0, state->trans[i].to);
+ set_bit(mc, Rdst, 0, state->trans[i].to);
}
if (mask & 2)
{
struct DFA_state *state = dfa->states[s+1];
int i = state->tran_no;
while (--i >= 0)
- set_bit (mc, Rdst, 0, state->trans[i].to);
+ set_bit(mc, Rdst, 0, state->trans[i].to);
}
if (mask & 4)
{
struct DFA_state *state = dfa->states[s+2];
int i = state->tran_no;
while (--i >= 0)
- set_bit (mc, Rdst, 0, state->trans[i].to);
+ set_bit(mc, Rdst, 0, state->trans[i].to);
}
if (mask & 8)
{
struct DFA_state *state = dfa->states[s+3];
int i = state->tran_no;
while (--i >= 0)
- set_bit (mc, Rdst, 0, state->trans[i].to);
+ set_bit(mc, Rdst, 0, state->trans[i].to);
}
}
s += 4;
}
}
-static void or (MatchContext *mc, MatchWord *Rdst,
- MatchWord *Rsrc1, MatchWord *Rsrc2)
+static void or(MatchContext *mc, MatchWord *Rdst,
+ MatchWord *Rsrc1, MatchWord *Rsrc2)
{
int i;
for (i = 0; i<mc->n; i++)
Rdst[i] = Rsrc1[i] | Rsrc2[i];
}
-static INLINE int move (MatchContext *mc, MatchWord *Rj1, MatchWord *Rj,
- Dict_char ch, struct DFA *dfa, MatchWord *Rtmp,
- int range)
+static INLINE int move(MatchContext *mc, MatchWord *Rj1, MatchWord *Rj,
+ Dict_char ch, struct DFA *dfa, MatchWord *Rtmp,
+ int range)
{
int d;
MatchWord *Rtmp_2 = Rtmp + mc->n;
- mask_shift (mc, Rj1, Rj, dfa, ch);
+ mask_shift(mc, Rj1, Rj, dfa, ch);
for (d = 1; d <= mc->range; d++)
{
- or (mc, Rtmp, Rj, Rj1); /* 2,3 */
+ or(mc, Rtmp, Rj, Rj1); /* 2,3 */
- shift (mc, Rtmp_2, Rtmp, dfa);
+ shift(mc, Rtmp_2, Rtmp, dfa);
- mask_shift (mc, Rtmp, Rj+mc->n, dfa, ch); /* 1 */
+ mask_shift(mc, Rtmp, Rj+mc->n, dfa, ch); /* 1 */
- or (mc, Rtmp, Rtmp_2, Rtmp); /* 1,2,3*/
+ or(mc, Rtmp, Rtmp_2, Rtmp); /* 1,2,3*/
Rj1 += mc->n;
- or (mc, Rj1, Rtmp, Rj); /* 1,2,3,4 */
+ or(mc, Rj1, Rtmp, Rj); /* 1,2,3,4 */
Rj += mc->n;
}
short *indxp;
char *info;
- dict_bf_readp (dict->dbf, ptr, &p);
+ dict_bf_readp(dict->dbf, ptr, &p);
lo = 0;
hi = DICT_nodir(p)-1;
indxp = (short*) ((char*) p+DICT_bsize(p)-sizeof(short));
if (was_match)
{
int ret = userfunc((char*) prefix,
- info+(j+1)*sizeof(Dict_char), client);
+ info+(j+1)*sizeof(Dict_char), client);
if (ret)
return ret;
}
range = mc->range;
else
range = 0;
- move (mc, Rj1, Rj0, ch, dfa, Rj_tmp, range);
+ move(mc, Rj1, Rj0, ch, dfa, Rj_tmp, range);
for (d = mc->n; --d >= 0; )
if (Rj1[range*mc->n + d])
break;
/* unsigned char length of information */
/* char * information */
info = (char*)p - indxp[-lo];
- memcpy (&ch, info+sizeof(Dict_ptr), sizeof(Dict_char));
+ memcpy(&ch, info+sizeof(Dict_ptr), sizeof(Dict_char));
prefix[pos] = ch;
if (pos > *max_pos)
range = mc->range;
else
range = 0;
- move (mc, Rj1, Rj, ch, dfa, Rj_tmp, range);
+ move(mc, Rj1, Rj, ch, dfa, Rj_tmp, range);
for (d = mc->n; --d >= 0; )
if (Rj1[range*mc->n + d])
break;
break;
}
}
- memcpy (&subptr, info, sizeof(Dict_ptr));
+ memcpy(&subptr, info, sizeof(Dict_ptr));
if (subptr)
{
int ret = grep(dict, subptr, mc, Rj1, pos+1,
if (ret)
return ret;
- dict_bf_readp (dict->dbf, ptr, &p);
+ dict_bf_readp(dict->dbf, ptr, &p);
indxp = (short*) ((char*) p+DICT_bsize(p)-sizeof(short));
}
}
for (i = 0; pattern[i]; i++)
{
yaz_log(YLOG_DEBUG, " %2d %3d %c", i, pattern[i],
- (pattern[i] > ' ' && pattern[i] < 127) ? pattern[i] : '?');
+ (pattern[i] > ' ' && pattern[i] < 127) ? pattern[i] : '?');
}
- dfa_set_cmap (dfa, dict->grep_cmap_data, dict->grep_cmap);
+ dfa_set_cmap(dfa, dict->grep_cmap_data, dict->grep_cmap);
- i = dfa_parse (dfa, &this_pattern);
+ i = dfa_parse(dfa, &this_pattern);
if (i || *this_pattern)
{
yaz_log(YLOG_WARN, "dfa_parse fail=%d", i);
- dfa_delete (&dfa);
+ dfa_delete(&dfa);
return -1;
}
- dfa_mkstate (dfa);
+ dfa_mkstate(dfa);
- mc = mk_MatchContext (dfa, range);
+ mc = mk_MatchContext(dfa, range);
Rj = (MatchWord *) xcalloc((MAX_LENGTH+1) * mc->n, sizeof(*Rj));
for (d = 1; d<=mc->range; d++)
{
int s;
- memcpy (Rj + mc->n * d, Rj + mc->n * (d-1), mc->n * sizeof(*Rj));
+ memcpy(Rj + mc->n * d, Rj + mc->n * (d-1), mc->n * sizeof(*Rj));
for (s = 0; s < dfa->no_states; s++)
{
- if (get_bit (mc, Rj, d-1, s))
+ if (get_bit(mc, Rj, d-1, s))
{
struct DFA_state *state = dfa->states[s];
int i = state->tran_no;
while (--i >= 0)
- set_bit (mc, Rj, d, state->trans[i].to);
+ set_bit(mc, Rj, d, state->trans[i].to);
}
}
}
yaz_log(YLOG_DEBUG, "max_pos = %d", *max_pos);
dfa_delete(&dfa);
xfree(Rj);
- rm_MatchContext (&mc);
+ rm_MatchContext(&mc);
return ret;
}
-void dict_grep_cmap (Dict dict, void *vp,
- const char **(*cmap)(void *vp, const char **from, int len))
+void dict_grep_cmap(Dict dict, void *vp,
+ const char **(*cmap)(void *vp, const char **from, int len))
{
dict->grep_cmap = cmap;
dict->grep_cmap_data = vp;
#include "dict-p.h"
-static char *dict_look (Dict dict, const Dict_char *str, Dict_ptr ptr)
+static char *dict_look(Dict dict, const Dict_char *str, Dict_ptr ptr)
{
int mid, lo, hi;
int cmp;
short *indxp;
char *info;
- dict_bf_readp (dict->dbf, ptr, &p);
+ dict_bf_readp(dict->dbf, ptr, &p);
mid = lo = 0;
hi = DICT_nodir(p)-1;
indxp = (short*) ((char*) p+DICT_bsize(p)-sizeof(short));
/* unsigned char length of information */
/* char * information */
info = (char*)p - indxp[-mid];
- memcpy (&dc, info+sizeof(Dict_ptr), sizeof(Dict_char));
+ memcpy(&dc, info+sizeof(Dict_ptr), sizeof(Dict_char));
cmp = dc- *str;
if (!cmp)
{
- memcpy (&subptr, info, sizeof(Dict_ptr));
+ memcpy(&subptr, info, sizeof(Dict_ptr));
if (*++str == DICT_EOS)
{
if (info[sizeof(Dict_ptr)+sizeof(Dict_char)])
if (subptr == 0)
return NULL;
ptr = subptr;
- dict_bf_readp (dict->dbf, ptr, &p);
+ dict_bf_readp(dict->dbf, ptr, &p);
mid = lo = 0;
hi = DICT_nodir(p)-1;
indxp = (short*) ((char*) p+DICT_bsize(p)-sizeof(short));
return NULL;
}
-char *dict_lookup (Dict dict, const char *p)
+char *dict_lookup(Dict dict, const char *p)
{
dict->no_lookup++;
if (!dict->head.root)
return NULL;
- return dict_look (dict, (const Dict_char *) p, dict->head.root);
+ return dict_look(dict, (const Dict_char *) p, dict->head.root);
}
/*
* Local variables:
char *info;
MatchWord match_mask = 1<<(mi->m-1);
- dict_bf_readp (dict->dbf, ptr, &p);
+ dict_bf_readp(dict->dbf, ptr, &p);
lo = 0;
hi = DICT_nodir(p)-1;
indxp = (short*) ((char*) p+DICT_bsize(p)-sizeof(short));
/* unsigned char length of information */
/* char * information */
info = (char*)p - indxp[-lo];
- memcpy (&ch, info+sizeof(Dict_ptr), sizeof(Dict_char));
+ memcpy(&ch, info+sizeof(Dict_ptr), sizeof(Dict_char));
prefix[pos] = ch;
sc = mi->s[ch & 255];
MatchInfo *mi;
mi = (MatchInfo *) xmalloc(sizeof(*mi));
- mi->m = dict_strlen (pattern);
+ mi->m = dict_strlen(pattern);
mi->s = s = (MatchWord *) xmalloc(sizeof(*s)*256); /* 256 !!! */
for (i = 0; i < 256; i++)
s[i] = 0;
/* This file is part of the Zebra server.
Copyright (C) 1994-2009 Index Data
-Zebra is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free
-Software Foundation; either version 2, or (at your option) any later
-version.
+ Zebra is free software; you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free
+ Software Foundation; either version 2, or (at your option) any later
+ version.
-Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
-WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-for more details.
+ Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ for more details.
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
void *head_buf;
int compact_flag = dict->head.compact_flag;
- memset (dict->head.magic_str, 0, sizeof(dict->head.magic_str));
- strcpy (dict->head.magic_str, DICT_MAGIC);
+ memset(dict->head.magic_str, 0, sizeof(dict->head.magic_str));
+ strcpy(dict->head.magic_str, DICT_MAGIC);
dict->head.last = 1;
dict->head.root = 0;
dict->head.freelist = 0;
/* create header with information (page 0) */
if (dict->rw)
- dict_bf_newp (dict->dbf, 0, &head_buf, page_size);
+ dict_bf_newp(dict->dbf, 0, &head_buf, page_size);
}
-Dict dict_open (BFiles bfs, const char *name, int cache, int rw,
- int compact_flag, int page_size)
+Dict dict_open(BFiles bfs, const char *name, int cache, int rw,
+ int compact_flag, int page_size)
{
Dict dict;
void *head_buf;
- dict = (Dict) xmalloc (sizeof(*dict));
+ dict = (Dict) xmalloc(sizeof(*dict));
if (cache < 5)
cache = 5;
page_size = DICT_DEFAULT_PAGESIZE;
if (page_size < 2048)
{
- yaz_log (YLOG_WARN, "Page size for dict %s %d<2048. Set to 2048",
- name, page_size);
+ yaz_log(YLOG_WARN, "Page size for dict %s %d<2048. Set to 2048",
+ name, page_size);
page_size = 2048;
}
- dict->dbf = dict_bf_open (bfs, name, page_size, cache, rw);
+ dict->dbf = dict_bf_open(bfs, name, page_size, cache, rw);
dict->rw = rw;
dict->no_split = 0;
dict->no_insert = 0;
if(!dict->dbf)
{
- yaz_log (YLOG_WARN, "Cannot open `%s'", name);
- xfree (dict);
+ yaz_log(YLOG_WARN, "Cannot open `%s'", name);
+ xfree(dict);
return NULL;
}
- if (dict_bf_readp (dict->dbf, 0, &head_buf) <= 0)
+ if (dict_bf_readp(dict->dbf, 0, &head_buf) <= 0)
{
dict->head.page_size = page_size;
dict->head.compact_flag = compact_flag;
}
else /* header was there, check magic and page size */
{
- memcpy (&dict->head, head_buf, sizeof(dict->head));
- if (strcmp (dict->head.magic_str, DICT_MAGIC))
+ memcpy(&dict->head, head_buf, sizeof(dict->head));
+ if (strcmp(dict->head.magic_str, DICT_MAGIC))
{
- yaz_log (YLOG_WARN, "Bad magic of `%s'", name);
+ yaz_log(YLOG_WARN, "Bad magic of `%s'", name);
dict_bf_close(dict->dbf);
xfree(dict);
return 0;
}
if (dict->head.page_size != page_size)
{
- yaz_log (YLOG_WARN, "Page size for existing dict %s is %d. Current is %d",
- name, dict->head.page_size, page_size);
+ yaz_log(YLOG_WARN, "Page size for existing dict %s is %d. Current is %d",
+ name, dict->head.page_size, page_size);
}
}
if (dict->head.compact_flag)
return dict;
}
-int dict_strcmp (const Dict_char *s1, const Dict_char *s2)
+int dict_strcmp(const Dict_char *s1, const Dict_char *s2)
{
- return strcmp ((const char *) s1, (const char *) s2);
+ return strcmp((const char *) s1, (const char *) s2);
}
-int dict_strncmp (const Dict_char *s1, const Dict_char *s2, size_t n)
+int dict_strncmp(const Dict_char *s1, const Dict_char *s2, size_t n)
{
- return strncmp ((const char *) s1, (const char *) s2, n);
+ return strncmp((const char *) s1, (const char *) s2, n);
}
-int dict_strlen (const Dict_char *s)
+int dict_strlen(const Dict_char *s)
{
return strlen((const char *) s);
}
/* This file is part of the Zebra server.
Copyright (C) 1994-2009 Index Data
-Zebra is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free
-Software Foundation; either version 2, or (at your option) any later
-version.
+ Zebra is free software; you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free
+ Software Foundation; either version 2, or (at your option) any later
+ version.
-Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
-WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-for more details.
+ Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ for more details.
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
short *indxp;
char *info;
- dict_bf_readp (dict->dbf, ptr, &p);
+ dict_bf_readp(dict->dbf, ptr, &p);
hi = DICT_nodir(p)-1;
if (start != -1)
lo = start;
/* char * information */
info = (char*)p - indxp[-lo];
- memcpy (&dc, info+sizeof(Dict_ptr), sizeof(Dict_char));
+ memcpy(&dc, info+sizeof(Dict_ptr), sizeof(Dict_char));
str[pos] = dc;
- memcpy (&subptr, info, sizeof(Dict_ptr));
+ memcpy(&subptr, info, sizeof(Dict_ptr));
if (dir>0 && info[sizeof(Dict_ptr)+sizeof(Dict_char)])
{
- str[pos+1] = DICT_EOS;
- if ((*userfunc)((char*) str,
- info+sizeof(Dict_ptr)+sizeof(Dict_char),
- *count * dir, client))
- {
- *count = 0;
- }
- else
- --(*count);
+ str[pos+1] = DICT_EOS;
+ if ((*userfunc)((char*) str,
+ info+sizeof(Dict_ptr)+sizeof(Dict_char),
+ *count * dir, client))
+ {
+ *count = 0;
+ }
+ else
+ --(*count);
}
if (*count>0 && subptr)
{
- scan_direction (dict, subptr, pos+1, str, -1, count,
- client, userfunc, dir);
- dict_bf_readp (dict->dbf, ptr, &p);
+ scan_direction(dict, subptr, pos+1, str, -1, count,
+ client, userfunc, dir);
+ dict_bf_readp(dict->dbf, ptr, &p);
indxp = (short*) ((char*) p+DICT_bsize(p)-sizeof(short));
}
if (*count>0 && dir<0 && info[sizeof(Dict_ptr)+sizeof(Dict_char)])
{
- str[pos+1] = DICT_EOS;
- if ((*userfunc)((char*) str,
- info+sizeof(Dict_ptr)+sizeof(Dict_char),
- *count * dir, client))
- {
- *count = 0;
- }
- else
- --(*count);
+ str[pos+1] = DICT_EOS;
+ if ((*userfunc)((char*) str,
+ info+sizeof(Dict_ptr)+sizeof(Dict_char),
+ *count * dir, client))
+ {
+ *count = 0;
+ }
+ else
+ --(*count);
}
}
lo += dir;
short *indxp;
char *info;
- dict_bf_readp (dict->dbf, ptr, &p);
+ dict_bf_readp(dict->dbf, ptr, &p);
if (!p)
return;
mid = lo = 0;
/* unsigned char length of information */
/* char * information */
info = (char*)p + indxp[-mid];
- cmp = dict_strcmp ((Dict_char*) info, str + pos);
+ cmp = dict_strcmp((Dict_char*) info, str + pos);
if (!cmp)
{
if (*after)
/* unsigned char length of information */
/* char * information */
info = (char*)p - indxp[-mid];
- memcpy (&dc, info+sizeof(Dict_ptr), sizeof(Dict_char));
+ memcpy(&dc, info+sizeof(Dict_ptr), sizeof(Dict_char));
cmp = dc - str[pos];
if (!cmp)
{
- memcpy (&subptr, info, sizeof(Dict_ptr));
+ memcpy(&subptr, info, sizeof(Dict_ptr));
if (str[pos+1] == DICT_EOS)
{
if (info[sizeof(Dict_ptr)+sizeof(Dict_char)])
-Subproject commit 3194c01cfd794ce4553ffd568e0cdc3640dc3f05
+Subproject commit 5d3fcb2cd2ef1cf9c5c5b40ef5c5353913b56b4f
# Flags for the non-authenticated user. w=write (allows ES Updates)
perm.anonymous: rw
+# Record compression (none, bzip2 or zlib)
+recordcompression: none
+
#rank: zvrank
recordId: (bib-1,title)
BuildRoot: %{_tmppath}/idzebra-%{version}-root
Packager: Adam Dickmeiss <adam@indexdata.dk>
URL: http://www.indexdata.dk/zebra/
-BuildRequires: libyaz3-devel expat-devel bzip2-devel tcl
+BuildRequires: libyaz3-devel expat-devel bzip2-devel tcl zlib-devel
Summary: High-performance, structured text indexing and retrival engine.
%description
*/
Records rec_open(BFiles bfs, int rw, int compression_method);
+/** \brief check whether a compression method is supported
+ \param compression_method (REC_COMPRESS_..)
+ \retval 0 if method is unsupported
+ \retval 1 if method is supported
+*/
+int rec_check_compression_method(int compression_method);
+
char *rec_strdup(const char *s, size_t *len);
void rec_prstat(Records p, int verbose);
zint rec_sysno_to_int(zint sysno);
-/** \brief compression types */
+
+/** \brief No compression ("none") */
#define REC_COMPRESS_NONE 0
+/** \brief BZIP2 compression (slow and requires big chunks) */
#define REC_COMPRESS_BZIP2 1
+/** \brief zlib compression (faster and works off small chunks) */
+#define REC_COMPRESS_ZLIB 2
+
enum {
recInfo_fileType,
*/
/*
- * Format of first block
+ * Format of first block (assumes a 512 block size)
* next (8 bytes)
* ref_count (2 bytes)
* block (500 bytes)
#if HAVE_BZLIB_H
#include <bzlib.h>
#endif
+#if HAVE_ZLIB_H
+#include <zlib.h>
+#endif
#define REC_BLOCK_TYPES 2
#define REC_HEAD_MAGIC "recindex"
return ZEBRA_OK;
}
+int rec_check_compression_method(int compression_method)
+{
+ switch(compression_method)
+ {
+ case REC_COMPRESS_ZLIB:
+#if HAVE_ZLIB_H
+ return 1;
+#else
+ return 0;
+#endif
+ case REC_COMPRESS_BZIP2:
+#if HAVE_BZLIB_H
+ return 1;
+#else
+ return 0;
+#endif
+ case REC_COMPRESS_NONE:
+ return 1;
+ }
+ return 0;
+}
+
Records rec_open(BFiles bfs, int rw, int compression_method)
{
Records p;
memset(&p->head, '\0', sizeof(p->head));
p->compression_method = compression_method;
p->rw = rw;
- p->tmp_size = 1024;
+ p->tmp_size = 4096;
p->tmp_buf = (char *) xmalloc(p->tmp_size);
p->compression_chunk_size = 0;
+ if (compression_method == REC_COMPRESS_BZIP2)
+ p->compression_chunk_size = 90000;
p->recindex = recindex_open(bfs, rw, 0 /* 1=isamb for recindex */);
r = recindex_read_head(p->recindex, p->tmp_buf);
switch (r)
p->head.block_last[i] = 1;
p->head.block_used[i] = 0;
}
- p->head.block_size[0] = 128;
+ p->head.block_size[0] = 256;
p->head.block_move[0] = 0;
for (i = 1; i<REC_BLOCK_TYPES; i++)
{
- p->head.block_size[i] = p->head.block_size[i-1] * 4;
- p->head.block_move[i] = p->head.block_size[i] * 24;
+ p->head.block_size[i] = p->head.block_size[i-1] * 8;
+ p->head.block_move[i] = p->head.block_size[i] * 2;
}
if (rw)
{
recindex_get_fname(p->recindex), version, REC_VERSION);
ret = ZEBRA_FAIL;
}
- p->compression_chunk_size = 90000; /* good for BZIP2 */
break;
}
for (i = 0; i<REC_BLOCK_TYPES; i++)
}
}
+static ZEBRA_RES rec_flush_shared(Records p, short ref_count, zint *sysnos,
+ char *out_buf, int out_offset)
+{
+ ZEBRA_RES ret = ZEBRA_OK;
+ if (ref_count)
+ {
+ int i;
+ unsigned int csize = 0; /* indicate compression "not performed yet" */
+ char compression_method = p->compression_method;
+ switch (compression_method)
+ {
+ case REC_COMPRESS_ZLIB:
+#if HAVE_ZLIB_H
+ csize = out_offset + (out_offset >> 6) + 620;
+ while (1)
+ {
+ int r;
+ uLongf destLen = csize;
+ rec_tmp_expand(p, csize);
+ r = compress((Bytef *) p->tmp_buf+sizeof(zint)+sizeof(short)+
+ sizeof(char),
+ &destLen, (const Bytef *) out_buf, out_offset);
+ csize = destLen;
+ if (r == Z_OK)
+ {
+ break;
+ }
+ if (r != Z_MEM_ERROR)
+ {
+ yaz_log(YLOG_WARN, "compress error: %d", r);
+ csize = 0;
+ break;
+ }
+ csize = csize * 2;
+ }
+#endif
+ break;
+ case REC_COMPRESS_BZIP2:
+#if HAVE_BZLIB_H
+ csize = out_offset + (out_offset >> 6) + 620;
+ rec_tmp_expand(p, csize);
+#ifdef BZ_CONFIG_ERROR
+ i = BZ2_bzBuffToBuffCompress
+#else
+ i = bzBuffToBuffCompress
+#endif
+ (p->tmp_buf+sizeof(zint)+sizeof(short)+
+ sizeof(char),
+ &csize, out_buf, out_offset, 1, 0, 30);
+ if (i != BZ_OK)
+ {
+ yaz_log(YLOG_WARN, "bzBuffToBuffCompress error code=%d", i);
+ csize = 0;
+ }
+#endif
+ break;
+ case REC_COMPRESS_NONE:
+ break;
+ }
+ if (!csize)
+ {
+ /* either no compression or compression not supported ... */
+ csize = out_offset;
+ rec_tmp_expand(p, csize);
+ memcpy(p->tmp_buf + sizeof(zint) + sizeof(short) + sizeof(char),
+ out_buf, out_offset);
+ csize = out_offset;
+ compression_method = REC_COMPRESS_NONE;
+ }
+ memcpy(p->tmp_buf + sizeof(zint), &ref_count, sizeof(ref_count));
+ memcpy(p->tmp_buf + sizeof(zint)+sizeof(short),
+ &compression_method, sizeof(compression_method));
+
+ /* -------- compression */
+ if (rec_write_tmp_buf(p, csize + sizeof(short) + sizeof(char), sysnos)
+ != ZEBRA_OK)
+ ret = ZEBRA_FAIL;
+ }
+ return ret;
+}
+
static ZEBRA_RES rec_write_multiple(Records p, int saveCount)
{
int i;
short ref_count = 0;
- char compression_method;
Record last_rec = 0;
int out_size = 1000;
int out_offset = 0;
}
*sysnop = -1;
- if (ref_count)
- {
- unsigned int csize = 0; /* indicate compression "not performed yet" */
- compression_method = p->compression_method;
- switch (compression_method)
- {
- case REC_COMPRESS_BZIP2:
-#if HAVE_BZLIB_H
- csize = out_offset + (out_offset >> 6) + 620;
- rec_tmp_expand(p, csize);
-#ifdef BZ_CONFIG_ERROR
- i = BZ2_bzBuffToBuffCompress
-#else
- i = bzBuffToBuffCompress
-#endif
- (p->tmp_buf+sizeof(zint)+sizeof(short)+
- sizeof(char),
- &csize, out_buf, out_offset, 1, 0, 30);
- if (i != BZ_OK)
- {
- yaz_log(YLOG_WARN, "bzBuffToBuffCompress error code=%d", i);
- csize = 0;
- }
- yaz_log(YLOG_LOG, "compress %4d %5d %5d", ref_count, out_offset,
- csize);
-#endif
- break;
- case REC_COMPRESS_NONE:
- break;
- }
- if (!csize)
- {
- /* either no compression or compression not supported ... */
- csize = out_offset;
- rec_tmp_expand(p, csize);
- memcpy(p->tmp_buf + sizeof(zint) + sizeof(short) + sizeof(char),
- out_buf, out_offset);
- csize = out_offset;
- compression_method = REC_COMPRESS_NONE;
- }
- memcpy(p->tmp_buf + sizeof(zint), &ref_count, sizeof(ref_count));
- memcpy(p->tmp_buf + sizeof(zint)+sizeof(short),
- &compression_method, sizeof(compression_method));
-
- /* -------- compression */
- if (rec_write_tmp_buf(p, csize + sizeof(short) + sizeof(char), sysnos)
- != ZEBRA_OK)
- ret = ZEBRA_FAIL;
- }
+ rec_flush_shared(p, ref_count, sysnos, out_buf, out_offset);
xfree(out_buf);
xfree(sysnos);
return ret;
char *nptr, *cptr;
char *in_buf = 0;
char *bz_buf = 0;
-#if HAVE_BZLIB_H
- unsigned int bz_size;
-#endif
char compression_method;
assert(sysno > 0);
in_size = entry.size - sizeof(short) - sizeof(char);
switch (compression_method)
{
+ case REC_COMPRESS_ZLIB:
+#if HAVE_ZLIB_H
+ if (1)
+ {
+ unsigned int bz_size = entry.size * 20 + 100;
+ while (1)
+ {
+ uLongf destLen = bz_size;
+ bz_buf = (char *) xmalloc(bz_size);
+ i = uncompress((Bytef *) bz_buf, &destLen,
+ (const Bytef *) in_buf, in_size);
+ if (i == Z_OK)
+ {
+ bz_size = destLen;
+ break;
+ }
+ yaz_log(YLOG_LOG, "failed");
+ xfree(bz_buf);
+ bz_size *= 2;
+ }
+ in_buf = bz_buf;
+ in_size = bz_size;
+ }
+#else
+ yaz_log(YLOG_FATAL, "cannot decompress record(s) in ZLIB format");
+ return 0;
+#endif
+ break;
case REC_COMPRESS_BZIP2:
#if HAVE_BZLIB_H
- bz_size = entry.size * 20 + 100;
- while (1)
- {
- bz_buf = (char *) xmalloc(bz_size);
+ if (1)
+ {
+ unsigned int bz_size = entry.size * 20 + 100;
+ while (1)
+ {
+ bz_buf = (char *) xmalloc(bz_size);
#ifdef BZ_CONFIG_ERROR
- i = BZ2_bzBuffToBuffDecompress
+ i = BZ2_bzBuffToBuffDecompress
#else
- i = bzBuffToBuffDecompress
+ i = bzBuffToBuffDecompress
#endif
- (bz_buf, &bz_size, in_buf, in_size, 0, 0);
- yaz_log(YLOG_LOG, "decompress %5d %5d", in_size, bz_size);
- if (i == BZ_OK)
- break;
- yaz_log(YLOG_LOG, "failed");
- xfree(bz_buf);
- bz_size *= 2;
- }
- in_buf = bz_buf;
- in_size = bz_size;
+ (bz_buf, &bz_size, in_buf, in_size, 0, 0);
+ if (i == BZ_OK)
+ break;
+ yaz_log(YLOG_LOG, "failed");
+ xfree(bz_buf);
+ bz_size *= 2;
+ }
+ in_buf = bz_buf;
+ in_size = bz_size;
+ }
#else
yaz_log(YLOG_FATAL, "cannot decompress record(s) in BZIP2 format");
return 0;
{
struct zebra_register *reg;
int record_compression = REC_COMPRESS_NONE;
- const char *recordCompression = 0;
+ const char *compression_str = 0;
const char *profilePath;
int sort_type = ZEBRA_SORT_TYPE_FLAT;
ZEBRA_RES ret = ZEBRA_OK;
zebraRankInstall(reg, rank_similarity_class);
zebraRankInstall(reg, rank_static_class);
- recordCompression = res_get_def(res, "recordCompression", "none");
- if (!strcmp(recordCompression, "none"))
+ compression_str = res_get_def(res, "recordCompression", "none");
+ if (!strcmp(compression_str, "none"))
record_compression = REC_COMPRESS_NONE;
- if (!strcmp(recordCompression, "bzip2"))
+ else if (!strcmp(compression_str, "bzip2"))
record_compression = REC_COMPRESS_BZIP2;
+ else if (!strcmp(compression_str, "zlib"))
+ record_compression = REC_COMPRESS_ZLIB;
+ else
+ {
+ yaz_log(YLOG_FATAL, "invalid recordCompression: %s", compression_str);
+ ret = ZEBRA_FAIL;
+ }
+
+ if (!rec_check_compression_method(record_compression))
+ {
+ yaz_log(YLOG_FATAL, "unsupported recordCompression: %s",
+ compression_str);
+ ret = ZEBRA_FAIL;
+ }
{
const char *index_fname = res_get_def(res, "index", "default.idx");
TERMID termid;
char buf[100];
- while(rfd->counted_items <= rs->hits_limit
+ while (rfd->counted_items <= rs->hits_limit
&& rset_default_read(rfd, buf, &termid))
;
" hits_limit=" ZINT_FORMAT,
rs, rs->hits_count, rs->hits_limit);
rs->hits_approx = 0;
- if (rs->hits_count > rs->hits_limit)
+ if (rs->hits_count > rs->hits_limit && rs->hits_limit > 0)
{
double cur, tot;
zint est;
rs->hits_approx = 1;
}
}
- yaz_log(log_level, "rset_close p=%p count=" ZINT_FORMAT, rs,
+ yaz_log(log_level, "rset_close(%s) p=%p count=" ZINT_FORMAT,
+ rs->control->desc, rs,
rs->hits_count);
}
rset_close_int(rs, rfd);
rset = (RSET) nmem_malloc(nmem, sizeof(*rset));
yaz_log(log_level, "rs_create(%s) rs=%p (nm=%p)", sel->desc, rset, nmem);
- yaz_log(log_level, " ref_id=%s limit=" ZINT_FORMAT,
- (term && term->ref_id ? term->ref_id : "null"),
- rset->hits_limit);
+ yaz_log(log_level, " ref_id=%s",
+ (term && term->ref_id ? term->ref_id : "null"));
rset->nmem = nmem;
rset->control = sel;
rset->refcount = 1;
while ((more = rset_read(rfd, buf, term)) > 0)
{
- if ((rfd->rset->keycontrol->cmp)(untilbuf, buf) <= 1)
+ if ((rfd->rset->keycontrol->cmp)(untilbuf, buf) < rset->scope)
break;
}
if (log_level)
- yaz_log (log_level, "rset_default_forward exiting m=%d c=%d",
- more, rset->scope);
+ yaz_log(log_level, "rset_default_forward exiting rfd=%p scope=%d m=%d c=%d",
+ rfd, rset->scope, more, rset->scope);
return more;
}
projects / idzebra-moved-to-github.git / commitdiff