X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Frecindex.c;h=793898b794e3ff4f5abfafd74a1db69a46f14724;hb=174ad2c7bbf2b7312ac080de2fd85d0509a55404;hp=bc9a1b5596b06408004b56e5379e8e3fcbc42359;hpb=189d7a12a2d0bac0bf87b53fa658c972fffc238c;p=idzebra-moved-to-github.git

diff --git a/index/recindex.c b/index/recindex.c
index bc9a1b5..793898b 100644
--- a/index/recindex.c
+++ b/index/recindex.c
@@ -1,20 +1,36 @@
-/*
- * Copyright (C) 1994-2002, Index Data
- * All rights reserved.
- *
- * $Id: recindex.c,v 1.33 2002-07-15 11:50:01 adam Exp $
- */
+/* $Id: recindex.c,v 1.41 2005-01-15 19:38:26 adam Exp $
+   Copyright (C) 1995-2005
+   Index Data ApS
+
+This file is part of the Zebra server.
+
+Zebra is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 2, or (at your option) any later
+version.
 
+Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with Zebra; see the file LICENSE.zebra.  If not, write to the
+Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
+02111-1307, USA.
+*/
+
+#define RIDX_CHUNK 128
 
 /*
  *  Format of first block
- *      next       (4 bytes)
- *      ref_count  (4 bytes)
- *      block      (504 bytes)
+ *      next       (8 bytes)
+ *      ref_count  (2 bytes)
+ *      block      (500 bytes)
  *
  *  Format of subsequent blocks 
- *      next  (4 bytes)
- *      block (508 bytes)
+ *      next  (8 bytes)
+ *      block (502 bytes)
  *
  *  Format of each record
  *      sysno
@@ -40,7 +56,7 @@ static void rec_write_head (Records p)
     r = bf_write (p->index_BFile, 0, 0, sizeof(p->head), &p->head);    
     if (r)
     {
-        logf (LOG_FATAL|LOG_ERRNO, "write head of %s", p->index_fname);
+        yaz_log (YLOG_FATAL|YLOG_ERRNO, "write head of %s", p->index_fname);
         exit (1);
     }
 }
@@ -51,39 +67,56 @@ static void rec_tmp_expand (Records p, int size)
         p->tmp_size < p->head.block_size[REC_BLOCK_TYPES-1]*2)
     {
         xfree (p->tmp_buf);
-        p->tmp_size = size + p->head.block_size[REC_BLOCK_TYPES-1]*2 + 2048;
+        p->tmp_size = size + (int)
+	        	(p->head.block_size[REC_BLOCK_TYPES-1])*2 + 2048;
         p->tmp_buf = (char *) xmalloc (p->tmp_size);
     }
 }
 
-static int read_indx (Records p, int sysno, void *buf, int itemsize, 
+static int read_indx (Records p, SYSNO sysno, void *buf, int itemsize, 
                       int ignoreError)
 {
     int r;
-    int pos = (sysno-1)*itemsize;
+    zint pos = (sysno-1)*itemsize;
+    int off = (int) (pos%RIDX_CHUNK);
+    int sz1 = RIDX_CHUNK - off;    /* sz1 is size of buffer to read.. */
 
-    r = bf_read (p->index_BFile, 1+pos/128, pos%128, itemsize, buf);
+    if (sz1 > itemsize)
+	sz1 = itemsize;  /* no more than itemsize bytes */
+
+    r = bf_read (p->index_BFile, 1+pos/RIDX_CHUNK, off, sz1, buf);
+    if (r == 1 && sz1 < itemsize) /* boundary? - must read second part */
+	r = bf_read (p->index_BFile, 2+pos/RIDX_CHUNK, 0, itemsize - sz1,
+			(char*) buf + sz1);
     if (r != 1 && !ignoreError)
     {
-        logf (LOG_FATAL|LOG_ERRNO, "read in %s at pos %ld",
+        yaz_log (YLOG_FATAL|YLOG_ERRNO, "read in %s at pos %ld",
               p->index_fname, (long) pos);
         exit (1);
     }
     return r;
 }
 
-static void write_indx (Records p, int sysno, void *buf, int itemsize)
+static void write_indx (Records p, SYSNO sysno, void *buf, int itemsize)
 {
-    int pos = (sysno-1)*itemsize;
+    zint pos = (sysno-1)*itemsize;
+    int off = (int) (pos%RIDX_CHUNK);
+    int sz1 = RIDX_CHUNK - off;    /* sz1 is size of buffer to read.. */
+
+    if (sz1 > itemsize)
+	sz1 = itemsize;  /* no more than itemsize bytes */
 
-    bf_write (p->index_BFile, 1+pos/128, pos%128, itemsize, buf);
+    bf_write(p->index_BFile, 1+pos/RIDX_CHUNK, off, sz1, buf);
+    if (sz1 < itemsize)   /* boundary? must write second part */
+	bf_write(p->index_BFile, 2+pos/RIDX_CHUNK, 0, itemsize - sz1,
+		(char*) buf + sz1);
 }
 
-static void rec_release_blocks (Records p, int sysno)
+static void rec_release_blocks (Records p, SYSNO sysno)
 {
     struct record_index_entry entry;
-    int freeblock;
-    char block_and_ref[sizeof(short) + sizeof(int)];
+    zint freeblock;
+    char block_and_ref[sizeof(zint) + sizeof(short)];
     int dst_type;
     int first = 1;
 
@@ -92,29 +125,30 @@ static void rec_release_blocks (Records p, int sysno)
 
     freeblock = entry.next;
     assert (freeblock > 0);
-    dst_type = freeblock & 7;
+    dst_type = (int) (freeblock & 7);
     assert (dst_type < REC_BLOCK_TYPES);
     freeblock = freeblock / 8;
     while (freeblock)
     {
         if (bf_read (p->data_BFile[dst_type], freeblock, 0,
-		     sizeof(block_and_ref), block_and_ref) != 1)
+		     first ? sizeof(block_and_ref) : sizeof(zint),
+		     block_and_ref) != 1)
         {
-            logf (LOG_FATAL|LOG_ERRNO, "read in rec_del_single");
+            yaz_log (YLOG_FATAL|YLOG_ERRNO, "read in rec_del_single");
             exit (1);
         }
 	if (first)
 	{
 	    short ref;
-	    memcpy (&ref, block_and_ref + sizeof(int), sizeof(ref));
+	    memcpy (&ref, block_and_ref + sizeof(freeblock), sizeof(ref));
 	    --ref;
-	    memcpy (block_and_ref + sizeof(int), &ref, sizeof(ref));
+	    memcpy (block_and_ref + sizeof(freeblock), &ref, sizeof(ref));
 	    if (ref)
 	    {
 		if (bf_write (p->data_BFile[dst_type], freeblock, 0,
 			      sizeof(block_and_ref), block_and_ref))
 		{
-		    logf (LOG_FATAL|LOG_ERRNO, "write in rec_del_single");
+		    yaz_log (YLOG_FATAL|YLOG_ERRNO, "write in rec_del_single");
 		    exit (1);
 		}
 		return;
@@ -125,11 +159,11 @@ static void rec_release_blocks (Records p, int sysno)
         if (bf_write (p->data_BFile[dst_type], freeblock, 0, sizeof(freeblock),
                       &p->head.block_free[dst_type]))
         {
-            logf (LOG_FATAL|LOG_ERRNO, "write in rec_del_single");
+            yaz_log (YLOG_FATAL|YLOG_ERRNO, "write in rec_del_single");
             exit (1);
         }
         p->head.block_free[dst_type] = freeblock;
-        memcpy (&freeblock, block_and_ref, sizeof(int));
+        memcpy (&freeblock, block_and_ref, sizeof(freeblock));
 
         p->head.block_used[dst_type]--;
     }
@@ -148,12 +182,12 @@ static void rec_delete_single (Records p, Record rec)
     write_indx (p, rec->sysno, &entry, sizeof(entry));
 }
 
-static void rec_write_tmp_buf (Records p, int size, int *sysnos)
+static void rec_write_tmp_buf (Records p, int size, SYSNO *sysnos)
 {
     struct record_index_entry entry;
     int no_written = 0;
     char *cptr = p->tmp_buf;
-    int block_prev = -1, block_free;
+    zint block_prev = -1, block_free;
     int dst_type = 0;
     int i;
 
@@ -169,7 +203,7 @@ static void rec_write_tmp_buf (Records p, int size, int *sysnos)
                          block_free, 0, sizeof(*p->head.block_free),
                          &p->head.block_free[dst_type]) != 1)
             {
-                logf (LOG_FATAL|LOG_ERRNO, "read in %s at free block %d",
+                yaz_log (YLOG_FATAL|YLOG_ERRNO, "read in %s at free block " ZINT_FORMAT,
                       p->data_fname[dst_type], block_free);
 		exit (1);
             }
@@ -189,19 +223,19 @@ static void rec_write_tmp_buf (Records p, int size, int *sysnos)
         }
         else
         {
-            memcpy (cptr, &block_free, sizeof(int));
+            memcpy (cptr, &block_free, sizeof(block_free));
             bf_write (p->data_BFile[dst_type], block_prev, 0, 0, cptr);
             cptr = p->tmp_buf + no_written;
         }
         block_prev = block_free;
-        no_written += p->head.block_size[dst_type] - sizeof(int);
+        no_written += (int)(p->head.block_size[dst_type]) - sizeof(zint);
         p->head.block_used[dst_type]++;
     }
     assert (block_prev != -1);
     block_free = 0;
-    memcpy (cptr, &block_free, sizeof(int));
+    memcpy (cptr, &block_free, sizeof(block_free));
     bf_write (p->data_BFile[dst_type], block_prev, 0,
-              sizeof(int) + (p->tmp_buf+size) - cptr, cptr);
+              sizeof(block_free) + (p->tmp_buf+size) - cptr, cptr);
 }
 
 Records rec_open (BFiles bfs, int rw, int compression_method)
@@ -216,10 +250,10 @@ Records rec_open (BFiles bfs, int rw, int compression_method)
     p->tmp_size = 1024;
     p->tmp_buf = (char *) xmalloc (p->tmp_size);
     p->index_fname = "reci";
-    p->index_BFile = bf_open (bfs, p->index_fname, 128, rw);
+    p->index_BFile = bf_open (bfs, p->index_fname, RIDX_CHUNK, rw);
     if (p->index_BFile == NULL)
     {
-        logf (LOG_FATAL|LOG_ERRNO, "open %s", p->index_fname);
+        yaz_log (YLOG_FATAL|YLOG_ERRNO, "open %s", p->index_fname);
         exit (1);
     }
     r = bf_read (p->index_BFile, 0, 0, 0, p->tmp_buf);
@@ -252,13 +286,13 @@ Records rec_open (BFiles bfs, int rw, int compression_method)
         memcpy (&p->head, p->tmp_buf, sizeof(p->head));
         if (memcmp (p->head.magic, REC_HEAD_MAGIC, sizeof(p->head.magic)))
         {
-            logf (LOG_FATAL, "file %s has bad format", p->index_fname);
+            yaz_log (YLOG_FATAL, "file %s has bad format", p->index_fname);
             exit (1);
         }
 	version = atoi (p->head.version);
 	if (version != REC_VERSION)
 	{
-	    logf (LOG_FATAL, "file %s is version %d, but version"
+	    yaz_log (YLOG_FATAL, "file %s is version %d, but version"
 		  " %d is required", p->index_fname, version, REC_VERSION);
 	    exit (1);
 	}
@@ -275,10 +309,10 @@ Records rec_open (BFiles bfs, int rw, int compression_method)
     for (i = 0; i<REC_BLOCK_TYPES; i++)
     {
         if (!(p->data_BFile[i] = bf_open (bfs, p->data_fname[i],
-                                          p->head.block_size[i],
+                                          (int) (p->head.block_size[i]),
                                           rw)))
         {
-            logf (LOG_FATAL|LOG_ERRNO, "bf_open %s", p->data_fname[i]);
+            yaz_log (YLOG_FATAL|YLOG_ERRNO, "bf_open %s", p->data_fname[i]);
             exit (1);
         }
     }
@@ -320,6 +354,36 @@ static void rec_decode_unsigned(unsigned *np, unsigned char *buf, int *len)
     *np = n;
 }
 
+static void rec_encode_zint (zint n, unsigned char *buf, int *len)
+{
+    (*len) = 0;
+    while (n > 127)
+    {
+	buf[*len] = (unsigned) (128 + (n & 127));
+	n = n >> 7;
+	(*len)++;
+    }
+    buf[*len] = (unsigned) n;
+    (*len)++;
+}
+
+static void rec_decode_zint(zint *np, unsigned char *buf, int *len)
+{
+    zint  n = 0;
+    zint w = 1;
+    (*len) = 0;
+
+    while (buf[*len] > 127)
+    {
+	n += w*(buf[*len] & 127);
+	w = w << 7;
+	(*len)++;
+    }
+    n += w * buf[*len];
+    (*len)++;
+    *np = n;
+}
+
 static void rec_cache_flush_block1 (Records p, Record rec, Record last_rec,
 				    char **out_buf, int *out_size,
 				    int *out_offset)
@@ -341,7 +405,7 @@ static void rec_cache_flush_block1 (Records p, Record rec, Record last_rec,
 	}
 	if (i == 0)
 	{
-	    rec_encode_unsigned (rec->sysno, *out_buf + *out_offset, &len);
+	    rec_encode_zint (rec->sysno, *out_buf + *out_offset, &len);
 	    (*out_offset) += len;
 	}
 	if (rec->size[i] == 0)
@@ -374,8 +438,8 @@ static void rec_write_multiple (Records p, int saveCount)
     int out_size = 1000;
     int out_offset = 0;
     char *out_buf = (char *) xmalloc (out_size);
-    int *sysnos = (int *) xmalloc (sizeof(*sysnos) * (p->cache_cur + 1));
-    int *sysnop = sysnos;
+    SYSNO *sysnos = (SYSNO *) xmalloc (sizeof(*sysnos) * (p->cache_cur + 1));
+    SYSNO *sysnop = sysnos;
 
     for (i = 0; i<p->cache_cur - saveCount; i++)
     {
@@ -424,15 +488,15 @@ static void rec_write_multiple (Records p, int saveCount)
 #else
 	    i = bzBuffToBuffCompress 
 #endif
-			 	     (p->tmp_buf+sizeof(int)+sizeof(short)+
+			 	     (p->tmp_buf+sizeof(zint)+sizeof(short)+
 				      sizeof(char),
 				      &csize, out_buf, out_offset, 1, 0, 30);
 	    if (i != BZ_OK)
 	    {
-		logf (LOG_WARN, "bzBuffToBuffCompress error code=%d", i);
+		yaz_log (YLOG_WARN, "bzBuffToBuffCompress error code=%d", i);
 		csize = 0;
 	    }
-	    logf (LOG_LOG, "compress %4d %5d %5d", ref_count, out_offset,
+	    yaz_log (YLOG_LOG, "compress %4d %5d %5d", ref_count, out_offset,
 		  csize);
 #endif
 	    break;
@@ -444,13 +508,13 @@ static void rec_write_multiple (Records p, int saveCount)
 	    /* either no compression or compression not supported ... */
 	    csize = out_offset;
 	    rec_tmp_expand (p, csize);
-	    memcpy (p->tmp_buf + sizeof(int) + sizeof(short) + sizeof(char),
+	    memcpy (p->tmp_buf + sizeof(zint) + sizeof(short) + sizeof(char),
 		    out_buf, out_offset);
 	    csize = out_offset;
 	    compression_method = REC_COMPRESS_NONE;
 	}
-	memcpy (p->tmp_buf + sizeof(int), &ref_count, sizeof(ref_count));
-	memcpy (p->tmp_buf + sizeof(int)+sizeof(short),
+	memcpy (p->tmp_buf + sizeof(zint), &ref_count, sizeof(ref_count));
+	memcpy (p->tmp_buf + sizeof(zint)+sizeof(short),
 		&compression_method, sizeof(compression_method));
 		
 	/* -------- compression */
@@ -481,7 +545,7 @@ static void rec_cache_flush (Records p, int saveCount)
     p->cache_cur = saveCount;
 }
 
-static Record *rec_cache_lookup (Records p, int sysno,
+static Record *rec_cache_lookup (Records p, SYSNO sysno,
                                  enum recordCacheFlag flag)
 {
     int i;
@@ -552,12 +616,13 @@ void rec_close (Records *pp)
     *pp = NULL;
 }
 
-static Record rec_get_int (Records p, int sysno)
+static Record rec_get_int (Records p, SYSNO sysno)
 {
     int i, in_size, r;
     Record rec, *recp;
     struct record_index_entry entry;
-    int freeblock, dst_type;
+    zint freeblock;
+    int dst_type;
     char *nptr, *cptr;
     char *in_buf = 0;
     char *bz_buf = 0;
@@ -578,7 +643,7 @@ static Record rec_get_int (Records p, int sysno)
     if (!entry.size)
         return NULL;       /* record is deleted */
 
-    dst_type = entry.next & 7;
+    dst_type = (int) (entry.next & 7);
     assert (dst_type < REC_BLOCK_TYPES);
     freeblock = entry.next / 8;
 
@@ -594,7 +659,7 @@ static Record rec_get_int (Records p, int sysno)
 
     while (freeblock)
     {
-        int tmp;
+        zint tmp;
 
         cptr += p->head.block_size[dst_type] - sizeof(freeblock);
         
@@ -608,9 +673,9 @@ static Record rec_get_int (Records p, int sysno)
 
     rec = (Record) xmalloc (sizeof(*rec));
     rec->sysno = sysno;
-    memcpy (&compression_method, p->tmp_buf + sizeof(int) + sizeof(short),
+    memcpy (&compression_method, p->tmp_buf + sizeof(zint) + sizeof(short),
 	    sizeof(compression_method));
-    in_buf = p->tmp_buf + sizeof(int) + sizeof(short) + sizeof(char);
+    in_buf = p->tmp_buf + sizeof(zint) + sizeof(short) + sizeof(char);
     in_size = entry.size - sizeof(short) - sizeof(char);
     switch (compression_method)
     {
@@ -626,17 +691,17 @@ static Record rec_get_int (Records p, int sysno)
 	    i = bzBuffToBuffDecompress
 #endif
                  (bz_buf, &bz_size, in_buf, in_size, 0, 0);
-	    logf (LOG_LOG, "decompress %5d %5d", in_size, bz_size);
+	    yaz_log (YLOG_LOG, "decompress %5d %5d", in_size, bz_size);
 	    if (i == BZ_OK)
 		break;
-	    logf (LOG_LOG, "failed");
+	    yaz_log (YLOG_LOG, "failed");
 	    xfree (bz_buf);
             bz_size *= 2;
 	}
 	in_buf = bz_buf;
 	in_size = bz_size;
 #else
-	logf (LOG_FATAL, "cannot decompress record(s) in BZIP2 format");
+	yaz_log (YLOG_FATAL, "cannot decompress record(s) in BZIP2 format");
 	exit (1);
 #endif
 	break;
@@ -649,9 +714,9 @@ static Record rec_get_int (Records p, int sysno)
     nptr = in_buf;                /* skip ref count */
     while (nptr < in_buf + in_size)
     {
-	int this_sysno;
+	zint this_sysno;
 	int len;
-	rec_decode_unsigned (&this_sysno, nptr, &len);
+	rec_decode_zint (&this_sysno, nptr, &len);
 	nptr += len;
 
 	for (i = 0; i < REC_NO_INFO; i++)
@@ -695,7 +760,7 @@ static Record rec_get_int (Records p, int sysno)
     return rec;
 }
 
-Record rec_get (Records p, int sysno)
+Record rec_get (Records p, SYSNO sysno)
 {
     Record rec;
     zebra_mutex_lock (&p->mutex);
@@ -707,7 +772,8 @@ Record rec_get (Records p, int sysno)
 
 static Record rec_new_int (Records p)
 {
-    int sysno, i;
+    int i;
+    SYSNO sysno;
     Record rec;
 
     assert (p);