X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=isam%2Fisam.c;h=018edd1f7873e3b19d391adeaa54a93beeaf0206;hb=49f1c8c877affdeb9e9221047c48beed3f475afb;hp=4a160a88309b3f99e00fa7746b3d8e0fb21cc301;hpb=fc5b1ebd4b23c1c55514a973f35ee98f2207597e;p=idzebra-moved-to-github.git

diff --git a/isam/isam.c b/isam/isam.c
index 4a160a8..018edd1 100644
--- a/isam/isam.c
+++ b/isam/isam.c
@@ -1,10 +1,35 @@
 /*
- * Copyright (C) 1994, Index Data I/S 
+ * Copyright (C) 1994-1999, Index Data
  * All rights reserved.
  * Sebastian Hammer, Adam Dickmeiss
  *
  * $Log: isam.c,v $
- * Revision 1.18  1996-02-06 10:19:56  quinn
+ * Revision 1.26  1999-05-26 07:49:14  adam
+ * C++ compilation.
+ *
+ * Revision 1.25  1999/02/02 14:51:16  adam
+ * Updated WIN32 code specific sections. Changed header.
+ *
+ * Revision 1.24  1997/10/27 14:25:39  adam
+ * Fixed memory leaks.
+ *
+ * Revision 1.23  1997/09/17 12:19:20  adam
+ * Zebra version corresponds to YAZ version 1.4.
+ * Changed Zebra server so that it doesn't depend on global common_resource.
+ *
+ * Revision 1.22  1996/10/29 13:56:53  adam
+ * Include of zebrautl.h instead of alexutil.h.
+ *
+ * Revision 1.21  1996/03/29 14:11:47  quinn
+ * Change to is_merge
+ *
+ * Revision 1.20  1996/03/19  13:14:57  quinn
+ * Moved an xfree()
+ *
+ * Revision 1.19  1996/02/10  12:20:56  quinn
+ * *** empty log message ***
+ *
+ * Revision 1.18  1996/02/06  10:19:56  quinn
  * Attempt at fixing bug. Not all blocks were read before they were unlinked
  * prior to a remap operation.
  *
@@ -64,17 +89,32 @@
 #include <string.h>
 #include <ctype.h>
 
-#include <alexutil.h>
+#include <zebrautl.h>
 #include <bfile.h>
 #include <isam.h>
-#include <common.h>
 #include "isutil.h"
 #include "rootblk.h"
 #include "keyops.h"
 
-static int (*extcmp)(const void *p1, const void *p2);
 static ispt_struct *ispt_freelist = 0;
 
+static struct
+{
+    int total_merge_operations;
+    int total_items;
+    int dub_items_removed;
+    int new_items;
+    int failed_deletes;
+    int skipped_inserts;
+    int delete_insert_noop;
+    int delete_replace;
+    int deletes;
+    int remaps;
+    int block_jumps;
+    int tab_deletes;
+    int new_tables;
+} statistics;
+
 static ISPT ispt_alloc()
 {
     ISPT p;
@@ -85,7 +125,7 @@ static ISPT ispt_alloc()
     	ispt_freelist = ispt_freelist->next;
     }
     else
-    	p = xmalloc(sizeof(ispt_struct));
+    	p = (ISPT) xmalloc(sizeof(ispt_struct));
     return p;
 }
 
@@ -121,29 +161,48 @@ static int splitargs(const char *s, char *bf[], int max)
  * Open isam file.
  * Process resources.
  */
-ISAM is_open(const char *name, int (*cmp)(const void *p1, const void *p2),
-    int writeflag, int keysize)
+ISAM is_open(BFiles bfs, const char *name,
+	     int (*cmp)(const void *p1, const void *p2),
+	     int writeflag, int keysize, Res res)
 {
-    ISAM new;
+    ISAM inew;
     char *nm, *r, *pp[IS_MAX_BLOCKTYPES+1], m[2];
     int num, size, rs, tmp, i;
     is_type_header th;
 
     logf (LOG_DEBUG, "is_open(%s, %s)", name, writeflag ? "RW" : "RDONLY");
-    new = xmalloc(sizeof(*new));
-    new->writeflag = writeflag;
+    if (writeflag)
+    {
+	statistics.total_merge_operations = 0;
+	statistics.total_items = 0;
+	statistics.dub_items_removed = 0;
+	statistics.new_items = 0;
+	statistics.failed_deletes = 0;
+	statistics.skipped_inserts = 0;
+	statistics.delete_insert_noop = 0;
+	statistics.delete_replace = 0;
+	statistics.deletes = 0;
+	statistics.remaps = 0;
+	statistics.new_tables = 0;
+	statistics.block_jumps = 0;
+	statistics.tab_deletes = 0;
+    }
+
+    inew = (ISAM) xmalloc(sizeof(*inew));
+    inew->writeflag = writeflag;
     for (i = 0; i < IS_MAX_BLOCKTYPES; i++)
-	new->types[i].index = 0;                        /* dummy */
+	inew->types[i].index = 0;                        /* dummy */
 
     /* determine number and size of blocktypes */
-    if (!(r = res_get_def(common_resource, nm = strconcat(name, ".",
-	"blocktypes", 0), "64 512 4K 32K")) ||
+    if (!(r = res_get_def(res,
+			  nm = strconcat(name, ".",
+					 "blocktypes", 0), "64 512 4K 32K")) ||
 	!(num = splitargs(r, pp, IS_MAX_BLOCKTYPES)))
     {
     	logf (LOG_FATAL, "Failed to locate resource %s", nm);
     	return 0;
     }
-    new->num_types = num;
+    inew->num_types = num;
     for (i = 0; i < num; i++)
     {
     	if ((rs = sscanf(pp[i], "%d%1[bBkKmM]", &size, m)) < 1)
@@ -156,58 +215,57 @@ ISAM is_open(const char *name, int (*cmp)(const void *p1, const void *p2),
 	switch (*m)
 	{
 		case 'b': case 'B':
-		    new->types[i].blocksize = size; break;
+		    inew->types[i].blocksize = size; break;
 		case 'k': case 'K':
-		    new->types[i].blocksize = size * 1024; break;
+		    inew->types[i].blocksize = size * 1024; break;
 		case 'm': case 'M':
-		    new->types[i].blocksize = size * 1048576; break;
+		    inew->types[i].blocksize = size * 1048576; break;
 		default:
 		    logf (LOG_FATAL, "Illegal size suffix: %c", *m);
 		    return 0;
 	}
-	new->types[i].dbuf = xmalloc(new->types[i].blocksize);
+	inew->types[i].dbuf = (char *) xmalloc(inew->types[i].blocksize);
 	m[0] = 'A' + i;
 	m[1] = '\0';
-	if (!(new->types[i].bf = bf_open(strconcat(name, m, 0), 
-	    new->types[i].blocksize, writeflag)))
+	if (!(inew->types[i].bf = bf_open(bfs, strconcat(name, m, 0), 
+	    inew->types[i].blocksize, writeflag)))
 	{
 	    logf (LOG_FATAL, "bf_open failed");
 	    return 0;
 	}
-	if ((rs = is_rb_read(&new->types[i], &th)) > 0)
+	if ((rs = is_rb_read(&inew->types[i], &th)) > 0)
 	{
-	    if (th.blocksize != new->types[i].blocksize)
+	    if (th.blocksize != inew->types[i].blocksize)
 	    {
 	    	logf (LOG_FATAL, "File blocksize mismatch in %s", name);
 	    	exit(1);
 	    }
-	    new->types[i].freelist = th.freelist;
-	    new->types[i].top = th.top;
+	    inew->types[i].freelist = th.freelist;
+	    inew->types[i].top = th.top;
 	}
 	else if (writeflag) /* write dummy superblock to determine top */
 	{
-	    if ((rs = is_rb_write(&new->types[i], &th)) <=0)  /* dummy */
+	    if ((rs = is_rb_write(&inew->types[i], &th)) <=0)  /* dummy */
 	    {
 	    	logf (LOG_FATAL, "Failed to write initial superblock.");
 	    	exit(1);
 	    }
-	    new->types[i].freelist = -1;
-	    new->types[i].top = rs;
+	    inew->types[i].freelist = -1;
+	    inew->types[i].top = rs;
 	}
 	/* ELSE: this is an empty file opened in read-only mode. */
     }
     if (keysize > 0)
-        new->keysize = keysize;
+        inew->keysize = keysize;
     else
     {
-        if (!(r = res_get_def(common_resource, nm = strconcat(name, ".",
-                                                              "keysize",
-                                                              0), "4")))
+        if (!(r = res_get_def(res, nm = strconcat(name, ".",
+						  "keysize", 0), "4")))
         {
             logf (LOG_FATAL, "Failed to locate resource %s", nm);
             return 0;
         }
-        if ((new->keysize = atoi(r)) <= 0)
+        if ((inew->keysize = atoi(r)) <= 0)
         {
             logf (LOG_FATAL, "Must specify positive keysize.");
             return 0;
@@ -215,23 +273,24 @@ ISAM is_open(const char *name, int (*cmp)(const void *p1, const void *p2),
     }
 
     /* determine repack percent */
-    if (!(r = res_get_def(common_resource, nm = strconcat(name, ".", "repack",
-    	0), IS_DEF_REPACK_PERCENT)))
+    if (!(r = res_get_def(res, nm = strconcat(name, ".", "repack",
+					      0), IS_DEF_REPACK_PERCENT)))
     {
     	logf (LOG_FATAL, "Failed to locate resource %s", nm);
     	return 0;
     }
-    new->repack = atoi(r);
+    inew->repack = atoi(r);
 
     /* determine max keys/blocksize */
-    if (!(r = res_get_def(common_resource, nm = strconcat(name, ".",
-	"maxkeys", 0), "50 640 10000")) || !(num = splitargs(r, pp,
-	IS_MAX_BLOCKTYPES)))
+    if (!(r = res_get_def(res,
+			  nm = strconcat(name, ".",
+					 "maxkeys", 0), "50 640 10000")) ||
+	!(num = splitargs(r, pp, IS_MAX_BLOCKTYPES)))
     {
     	logf (LOG_FATAL, "Failed to locate resource %s", nm);
     	return 0;
     }
-    if (num < new->num_types -1)
+    if (num < inew->num_types -1)
     {
     	logf (LOG_FATAL, "Not enough elements in %s", nm);
     	return 0;
@@ -243,23 +302,23 @@ ISAM is_open(const char *name, int (*cmp)(const void *p1, const void *p2),
 	    logf (LOG_FATAL, "Error in resource %s: %s", r, pp[i]);
 	    return 0;
 	}
-	new->types[i].max_keys = tmp;
+	inew->types[i].max_keys = tmp;
     }
 
     /* determine max keys/block */
-    for (i = 0; i < new->num_types; i++)
+    for (i = 0; i < inew->num_types; i++)
     {
-    	if (!new->types[i].index)
+    	if (!inew->types[i].index)
     	{
-	    new->types[i].max_keys_block = (new->types[i].blocksize - 2 *
-		sizeof(int)) / new->keysize;
-	    new->types[i].max_keys_block0 = (new->types[i].blocksize - 3 *
-		sizeof(int)) / new->keysize;
+	    inew->types[i].max_keys_block = (inew->types[i].blocksize - 2 *
+		sizeof(int)) / inew->keysize;
+	    inew->types[i].max_keys_block0 = (inew->types[i].blocksize - 3 *
+		sizeof(int)) / inew->keysize;
 	}
 	else
-	    new->types[i].max_keys_block = new->types[i].max_keys_block0 /
-		new->keysize;
-	if (new->types[i].max_keys_block0 < 1)
+	    inew->types[i].max_keys_block = inew->types[i].max_keys_block0 /
+		inew->keysize;
+	if (inew->types[i].max_keys_block0 < 1)
 	{
 	    logf (LOG_FATAL, "Blocksize too small in %s", name);
 	    exit(1);
@@ -267,14 +326,15 @@ ISAM is_open(const char *name, int (*cmp)(const void *p1, const void *p2),
     }
 
     /* determine nice fill rates */
-    if (!(r = res_get_def(common_resource, nm = strconcat(name, ".",
-	"nicefill", 0), "90 90 90 95")) || !(num = splitargs(r, pp,
-	IS_MAX_BLOCKTYPES)))
+    if (!(r = res_get_def(res,
+			  nm = strconcat(name, ".",
+					 "nicefill", 0), "90 90 90 95")) ||
+	!(num = splitargs(r, pp, IS_MAX_BLOCKTYPES)))
     {
     	logf (LOG_FATAL, "Failed to locate resource %s", nm);
     	return 0;
     }
-    if (num < new->num_types)
+    if (num < inew->num_types)
     {
     	logf (LOG_FATAL, "Not enough elements in %s", nm);
     	return 0;
@@ -286,14 +346,14 @@ ISAM is_open(const char *name, int (*cmp)(const void *p1, const void *p2),
 	    logf (LOG_FATAL, "Error in resource %s: %s", r, pp[i]);
 	    return 0;
 	}
-	new->types[i].nice_keys_block = (new->types[i].max_keys_block0 * tmp) /
+	inew->types[i].nice_keys_block = (inew->types[i].max_keys_block0 * tmp) /
 	    100;
-	if (new->types[i].nice_keys_block < 1)
-		new->types[i].nice_keys_block = 1;
+	if (inew->types[i].nice_keys_block < 1)
+		inew->types[i].nice_keys_block = 1;
     }
 
-    new->cmp = cmp ? cmp : is_default_cmp;
-    return new;
+    inew->cmp = cmp ? cmp : is_default_cmp;
+    return inew;
 }
 
 /*
@@ -324,6 +384,31 @@ int is_close(ISAM is)
 	    bf_close(is->types[i].bf);
 	}
     }
+    for (i = 0; i < is->num_types; i++)
+	xfree (is->types[i].dbuf);
+
+    if (is->writeflag)
+    {
+	logf(LOG_LOG, "ISAM statistics:");
+	logf(LOG_LOG, "total_merge_operations      %d",
+	    statistics.total_merge_operations);
+	logf(LOG_LOG, "total_items                 %d", statistics.total_items);
+	logf(LOG_LOG, "dub_items_removed           %d",
+	    statistics.dub_items_removed);
+	logf(LOG_LOG, "new_items                   %d", statistics.new_items);
+	logf(LOG_LOG, "failed_deletes              %d",
+	    statistics.failed_deletes);
+	logf(LOG_LOG, "skipped_inserts             %d",
+	    statistics.skipped_inserts);
+	logf(LOG_LOG, "delete_insert_noop          %d",
+	    statistics.delete_insert_noop);
+	logf(LOG_LOG, "delete_replace              %d",
+	    statistics.delete_replace);
+	logf(LOG_LOG, "delete                      %d", statistics.deletes);
+	logf(LOG_LOG, "remaps                      %d", statistics.remaps);
+	logf(LOG_LOG, "block_jumps                 %d", statistics.block_jumps);
+	logf(LOG_LOG, "tab_deletes                 %d", statistics.tab_deletes);
+    }
     xfree(is);
     return 0;
 }
@@ -337,15 +422,6 @@ static ISAM_P is_address(int type, int pos)
     return r;
 }
 
-int sort_input(const void *p1, const void *p2)
-{
-    int rs;
-
-    if ((rs = (*extcmp)(((char *)p1) + 1, ((char *)p2) + 1)))
-    	return rs;
-    return *((char *)p1) - *((char*)p2);
-}
-
 ISAM_P is_merge(ISAM is, ISAM_P pos, int num, char *data)
 {
     is_mtable tab;
@@ -354,10 +430,11 @@ ISAM_P is_merge(ISAM is, ISAM_P pos, int num, char *data)
     int oldnum, oldtype, i;
     char operation, *record;
 
-    extcmp = is->cmp;
-#if 0
-    qsort(data, num, is_keysize(is) + 1, sort_input);
-#endif
+    statistics.total_merge_operations++;
+    statistics.total_items += num;
+    if (!pos)
+	statistics.new_tables++;
+
     is_m_establish_tab(is, &tab, pos);
     if (pos)
     	if (is_m_read_full(&tab, tab.data) < 0)
@@ -377,6 +454,7 @@ ISAM_P is_merge(ISAM is, ISAM_P pos, int num, char *data)
 	{
 	    data += 1 + is_keysize(is);
 	    num--;
+	    statistics.dub_items_removed++;
 	}
 	if ((res = is_m_seek_record(&tab, record)) > 0)  /* no match */
 	{
@@ -384,15 +462,20 @@ ISAM_P is_merge(ISAM is, ISAM_P pos, int num, char *data)
 	    {
 	        logf (LOG_DEBUG, "XXInserting new record.");
 		is_m_write_record(&tab, record);
+		statistics.new_items++;
 	    }
 	    else
+	    {
 	    	logf (LOG_DEBUG, "XXDeletion failed to find match.");
+		statistics.failed_deletes++;
+	    }
 	}
 	else /* match found */
 	{
 	    if (operation == KEYOP_INSERT)
 	    {
 	        logf (LOG_DEBUG, "XXSkipping insertion - match found.");
+		statistics.skipped_inserts++;
 	    	continue;
 	    }
 	    else if (operation == KEYOP_DELETE)
@@ -406,22 +489,38 @@ ISAM_P is_merge(ISAM is, ISAM_P pos, int num, char *data)
 		        logf (LOG_DEBUG, "XXNoop delete. skipping.");
 		    	data += 1 + is_keysize(is);
 		    	num--;
+			while (num && !memcmp(data, data + is_keysize(tab.is) +
+			    1, is_keysize(tab.is) + 1))
+			{
+			    data += 1 + is_keysize(is);
+			    num--;
+			    statistics.dub_items_removed++;
+			}
+			statistics.delete_insert_noop++;
 		    	continue;
 		    }
 		    /* else check if next key can fit in this position */
-		    is_m_peek_record(&tab, keybuf);
-		    res = (*is->cmp)(data + 1, keybuf);
-		    if (res < 0)
+		    if (is_m_peek_record(&tab, keybuf) &&
+			(*is->cmp)(data + 1, keybuf) < 0)
 		    {
 		        logf (LOG_DEBUG, "XXReplacing record.");
 		    	is_m_replace_record(&tab, data + 1);
 		    	data += 1 + is_keysize(is);
 		    	num--;
+			while (num && !memcmp(data, data + is_keysize(tab.is) +
+			    1, is_keysize(tab.is) + 1))
+			{
+			    data += 1 + is_keysize(is);
+			    num--;
+			    statistics.dub_items_removed++;
+			}
+			statistics.delete_replace++;
 			continue;
 		    }
 		}
 		logf (LOG_DEBUG, "Deleting record.");
 		is_m_delete_record(&tab);
+		statistics.deletes++;
 	    }
 	}
     }
@@ -437,10 +536,15 @@ ISAM_P is_merge(ISAM is, ISAM_P pos, int num, char *data)
 		is_m_read_full(&tab, tab.cur_mblock);
     	is_p_unmap(&tab);
 	tab.pos_type = i;
+	if (pos)
+	    statistics.block_jumps++;
     }
     if (!oldnum || tab.pos_type != oldtype || (abs(oldnum - tab.num_records) *
 	100) / oldnum > tab.is->repack)
+    {
     	is_p_remap(&tab);
+	statistics.remaps++;
+    }
     else
     	is_p_align(&tab);
     if (tab.data)
@@ -449,7 +553,10 @@ ISAM_P is_merge(ISAM is, ISAM_P pos, int num, char *data)
 	pos = is_address(tab.pos_type, tab.data->diskpos);
     }
     else
+    {
     	pos = 0;
+	statistics.tab_deletes++;
+    }
     is_m_release_tab(&tab);
     return pos;
 }