X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=isam%2Fisam.c;h=362932c17c89f504637442568560b0c954d03a76;hb=ac2780bc12c8c2fce4c5e2bfe4c4dbb58616b8e2;hp=42d7acdb51d3ad4fb741fa0b6c652ee3728d4f9c;hpb=b879b04a092d5b00cc866cf16f755e55053d2e89;p=idzebra-moved-to-github.git diff --git a/isam/isam.c b/isam/isam.c index 42d7acd..362932c 100644 --- a/isam/isam.c +++ b/isam/isam.c @@ -1,10 +1,52 @@ /* - * Copyright (C) 1994, Index Data I/S + * Copyright (C) 1994-1997, Index Data I/S * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * * $Log: isam.c,v $ - * Revision 1.11 1995-09-04 12:33:46 adam + * Revision 1.24 1997-10-27 14:25:39 adam + * Fixed memory leaks. + * + * Revision 1.23 1997/09/17 12:19:20 adam + * Zebra version corresponds to YAZ version 1.4. + * Changed Zebra server so that it doesn't depend on global common_resource. + * + * Revision 1.22 1996/10/29 13:56:53 adam + * Include of zebrautl.h instead of alexutil.h. + * + * Revision 1.21 1996/03/29 14:11:47 quinn + * Change to is_merge + * + * Revision 1.20 1996/03/19 13:14:57 quinn + * Moved an xfree() + * + * Revision 1.19 1996/02/10 12:20:56 quinn + * *** empty log message *** + * + * Revision 1.18 1996/02/06 10:19:56 quinn + * Attempt at fixing bug. Not all blocks were read before they were unlinked + * prior to a remap operation. + * + * Revision 1.17 1995/12/06 15:48:44 quinn + * Fixed update-problem. + * + * Revision 1.16 1995/12/06 14:48:26 quinn + * Fixed some strange bugs. + * + * Revision 1.15 1995/12/06 09:59:45 quinn + * Fixed memory-consumption bug in memory.c + * Added more blocksizes to the default ISAM configuration. + * + * Revision 1.14 1995/11/24 17:26:19 quinn + * Mostly about making some ISAM stuff in the config file optional. + * + * Revision 1.13 1995/10/17 18:03:15 adam + * Commented out qsort in is_merge. + * + * Revision 1.12 1995/09/06 16:11:41 adam + * Keysize parameter to is_open (if non-zero). + * + * Revision 1.11 1995/09/04 12:33:46 adam * Various cleanup. YAZ util used instead. * * Revision 1.10 1994/09/28 16:58:32 quinn @@ -41,17 +83,32 @@ #include #include -#include +#include #include #include -#include #include "isutil.h" #include "rootblk.h" #include "keyops.h" -static int (*extcmp)(const void *p1, const void *p2); static ispt_struct *ispt_freelist = 0; +static struct +{ + int total_merge_operations; + int total_items; + int dub_items_removed; + int new_items; + int failed_deletes; + int skipped_inserts; + int delete_insert_noop; + int delete_replace; + int delete; + int remaps; + int block_jumps; + int tab_deletes; + int new_tables; +} statistics; + static ISPT ispt_alloc() { ISPT p; @@ -98,8 +155,9 @@ static int splitargs(const char *s, char *bf[], int max) * Open isam file. * Process resources. */ -ISAM is_open(const char *name, int (*cmp)(const void *p1, const void *p2), - int writeflag) +ISAM is_open(BFiles bfs, const char *name, + int (*cmp)(const void *p1, const void *p2), + int writeflag, int keysize, Res res) { ISAM new; char *nm, *r, *pp[IS_MAX_BLOCKTYPES+1], m[2]; @@ -107,14 +165,33 @@ ISAM is_open(const char *name, int (*cmp)(const void *p1, const void *p2), is_type_header th; logf (LOG_DEBUG, "is_open(%s, %s)", name, writeflag ? "RW" : "RDONLY"); + if (writeflag) + { + statistics.total_merge_operations = 0; + statistics.total_items = 0; + statistics.dub_items_removed = 0; + statistics.new_items = 0; + statistics.failed_deletes = 0; + statistics.skipped_inserts = 0; + statistics.delete_insert_noop = 0; + statistics.delete_replace = 0; + statistics.delete = 0; + statistics.remaps = 0; + statistics.new_tables = 0; + statistics.block_jumps = 0; + statistics.tab_deletes = 0; + } + new = xmalloc(sizeof(*new)); new->writeflag = writeflag; for (i = 0; i < IS_MAX_BLOCKTYPES; i++) new->types[i].index = 0; /* dummy */ /* determine number and size of blocktypes */ - if (!(r = res_get(common_resource, nm = strconcat(name, ".", - "blocktypes", 0))) || !(num = splitargs(r, pp, IS_MAX_BLOCKTYPES))) + if (!(r = res_get_def(res, + nm = strconcat(name, ".", + "blocktypes", 0), "64 512 4K 32K")) || + !(num = splitargs(r, pp, IS_MAX_BLOCKTYPES))) { logf (LOG_FATAL, "Failed to locate resource %s", nm); return 0; @@ -144,7 +221,7 @@ ISAM is_open(const char *name, int (*cmp)(const void *p1, const void *p2), new->types[i].dbuf = xmalloc(new->types[i].blocksize); m[0] = 'A' + i; m[1] = '\0'; - if (!(new->types[i].bf = bf_open(strconcat(name, m, 0), + if (!(new->types[i].bf = bf_open(bfs, strconcat(name, m, 0), new->types[i].blocksize, writeflag))) { logf (LOG_FATAL, "bf_open failed"); @@ -172,21 +249,26 @@ ISAM is_open(const char *name, int (*cmp)(const void *p1, const void *p2), } /* ELSE: this is an empty file opened in read-only mode. */ } - if (!(r = res_get_def(common_resource, nm = strconcat(name, ".", "keysize", - 0), "4"))) - { - logf (LOG_FATAL, "Failed to locate resource %s", nm); - return 0; - } - if ((new->keysize = atoi(r)) <= 0) + if (keysize > 0) + new->keysize = keysize; + else { - logf (LOG_FATAL, "Must specify positive keysize."); - return 0; + if (!(r = res_get_def(res, nm = strconcat(name, ".", + "keysize", 0), "4"))) + { + logf (LOG_FATAL, "Failed to locate resource %s", nm); + return 0; + } + if ((new->keysize = atoi(r)) <= 0) + { + logf (LOG_FATAL, "Must specify positive keysize."); + return 0; + } } /* determine repack percent */ - if (!(r = res_get_def(common_resource, nm = strconcat(name, ".", "repack", - 0), IS_DEF_REPACK_PERCENT))) + if (!(r = res_get_def(res, nm = strconcat(name, ".", "repack", + 0), IS_DEF_REPACK_PERCENT))) { logf (LOG_FATAL, "Failed to locate resource %s", nm); return 0; @@ -194,8 +276,10 @@ ISAM is_open(const char *name, int (*cmp)(const void *p1, const void *p2), new->repack = atoi(r); /* determine max keys/blocksize */ - if (!(r = res_get(common_resource, nm = strconcat(name, ".", - "maxkeys", 0))) || !(num = splitargs(r, pp, IS_MAX_BLOCKTYPES))) + if (!(r = res_get_def(res, + nm = strconcat(name, ".", + "maxkeys", 0), "50 640 10000")) || + !(num = splitargs(r, pp, IS_MAX_BLOCKTYPES))) { logf (LOG_FATAL, "Failed to locate resource %s", nm); return 0; @@ -236,8 +320,10 @@ ISAM is_open(const char *name, int (*cmp)(const void *p1, const void *p2), } /* determine nice fill rates */ - if (!(r = res_get(common_resource, nm = strconcat(name, ".", - "nicefill", 0))) || !(num = splitargs(r, pp, IS_MAX_BLOCKTYPES))) + if (!(r = res_get_def(res, + nm = strconcat(name, ".", + "nicefill", 0), "90 90 90 95")) || + !(num = splitargs(r, pp, IS_MAX_BLOCKTYPES))) { logf (LOG_FATAL, "Failed to locate resource %s", nm); return 0; @@ -292,6 +378,31 @@ int is_close(ISAM is) bf_close(is->types[i].bf); } } + for (i = 0; i < is->num_types; i++) + xfree (is->types[i].dbuf); + + if (is->writeflag) + { + logf(LOG_LOG, "ISAM statistics:"); + logf(LOG_LOG, "total_merge_operations %d", + statistics.total_merge_operations); + logf(LOG_LOG, "total_items %d", statistics.total_items); + logf(LOG_LOG, "dub_items_removed %d", + statistics.dub_items_removed); + logf(LOG_LOG, "new_items %d", statistics.new_items); + logf(LOG_LOG, "failed_deletes %d", + statistics.failed_deletes); + logf(LOG_LOG, "skipped_inserts %d", + statistics.skipped_inserts); + logf(LOG_LOG, "delete_insert_noop %d", + statistics.delete_insert_noop); + logf(LOG_LOG, "delete_replace %d", + statistics.delete_replace); + logf(LOG_LOG, "delete %d", statistics.delete); + logf(LOG_LOG, "remaps %d", statistics.remaps); + logf(LOG_LOG, "block_jumps %d", statistics.block_jumps); + logf(LOG_LOG, "tab_deletes %d", statistics.tab_deletes); + } xfree(is); return 0; } @@ -305,15 +416,6 @@ static ISAM_P is_address(int type, int pos) return r; } -int sort_input(const void *p1, const void *p2) -{ - int rs; - - if ((rs = (*extcmp)(((char *)p1) + 1, ((char *)p2) + 1))) - return rs; - return *((char *)p1) - *((char*)p2); -} - ISAM_P is_merge(ISAM is, ISAM_P pos, int num, char *data) { is_mtable tab; @@ -322,8 +424,11 @@ ISAM_P is_merge(ISAM is, ISAM_P pos, int num, char *data) int oldnum, oldtype, i; char operation, *record; - extcmp = is->cmp; - qsort(data, num, is_keysize(is) + 1, sort_input); + statistics.total_merge_operations++; + statistics.total_items += num; + if (!pos) + statistics.new_tables++; + is_m_establish_tab(is, &tab, pos); if (pos) if (is_m_read_full(&tab, tab.data) < 0) @@ -343,6 +448,7 @@ ISAM_P is_merge(ISAM is, ISAM_P pos, int num, char *data) { data += 1 + is_keysize(is); num--; + statistics.dub_items_removed++; } if ((res = is_m_seek_record(&tab, record)) > 0) /* no match */ { @@ -350,15 +456,20 @@ ISAM_P is_merge(ISAM is, ISAM_P pos, int num, char *data) { logf (LOG_DEBUG, "XXInserting new record."); is_m_write_record(&tab, record); + statistics.new_items++; } else + { logf (LOG_DEBUG, "XXDeletion failed to find match."); + statistics.failed_deletes++; + } } else /* match found */ { if (operation == KEYOP_INSERT) { logf (LOG_DEBUG, "XXSkipping insertion - match found."); + statistics.skipped_inserts++; continue; } else if (operation == KEYOP_DELETE) @@ -372,22 +483,38 @@ ISAM_P is_merge(ISAM is, ISAM_P pos, int num, char *data) logf (LOG_DEBUG, "XXNoop delete. skipping."); data += 1 + is_keysize(is); num--; + while (num && !memcmp(data, data + is_keysize(tab.is) + + 1, is_keysize(tab.is) + 1)) + { + data += 1 + is_keysize(is); + num--; + statistics.dub_items_removed++; + } + statistics.delete_insert_noop++; continue; } /* else check if next key can fit in this position */ - is_m_peek_record(&tab, keybuf); - res = (*is->cmp)(data + 1, keybuf); - if (res < 0) + if (is_m_peek_record(&tab, keybuf) && + (*is->cmp)(data + 1, keybuf) < 0) { logf (LOG_DEBUG, "XXReplacing record."); is_m_replace_record(&tab, data + 1); data += 1 + is_keysize(is); num--; + while (num && !memcmp(data, data + is_keysize(tab.is) + + 1, is_keysize(tab.is) + 1)) + { + data += 1 + is_keysize(is); + num--; + statistics.dub_items_removed++; + } + statistics.delete_replace++; continue; } } logf (LOG_DEBUG, "Deleting record."); is_m_delete_record(&tab); + statistics.delete++; } } } @@ -397,12 +524,21 @@ ISAM_P is_merge(ISAM is, ISAM_P pos, int num, char *data) i++; if (i != tab.pos_type) { + /* read remaining blocks */ + for (; tab.cur_mblock; tab.cur_mblock = tab.cur_mblock->next) + if (tab.cur_mblock->state < IS_MBSTATE_CLEAN) + is_m_read_full(&tab, tab.cur_mblock); is_p_unmap(&tab); tab.pos_type = i; + if (pos) + statistics.block_jumps++; } if (!oldnum || tab.pos_type != oldtype || (abs(oldnum - tab.num_records) * 100) / oldnum > tab.is->repack) + { is_p_remap(&tab); + statistics.remaps++; + } else is_p_align(&tab); if (tab.data) @@ -411,7 +547,10 @@ ISAM_P is_merge(ISAM is, ISAM_P pos, int num, char *data) pos = is_address(tab.pos_type, tab.data->diskpos); } else + { pos = 0; + statistics.tab_deletes++; + } is_m_release_tab(&tab); return pos; } @@ -443,7 +582,7 @@ void is_pt_free(ISPT ip) */ int is_readkey(ISPT ip, void *buf) { - return is_m_read_record(&ip->tab, buf); + return is_m_read_record(&ip->tab, buf, 0); } int is_numkeys(ISPT ip)