X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=isamc%2Fisamd.c;h=8fb7dcaec219e5f243522bc3556fda3ee5421010;hb=e5e4f1de02396b2cfab488f8e5918853e96d6b53;hp=3052990259a3bcf2c2a1da9ee34cb090a862cc4f;hpb=391f50227af9b86500ad1ddee46f3bd9d461aa92;p=idzebra-moved-to-github.git diff --git a/isamc/isamd.c b/isamc/isamd.c index 3052990..8fb7dca 100644 --- a/isamc/isamd.c +++ b/isamc/isamd.c @@ -1,15 +1,26 @@ -/* - * Copyright (c) 1995-1998, Index Data. - * See the file LICENSE for details. - * $Id: isamd.c,v 1.8 1999-08-18 13:28:16 heikki Exp $ - * - * Isamd - isam with diffs - * Programmed by: Heikki Levanto - * - * Todo - * - Statistics are missing and/or completely wrong - * - Lots of code stolen from isamc, not all needed any more - */ +/* $Id: isamd.c,v 1.23 2002-08-02 19:26:56 adam Exp $ + Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002 + Index Data Aps + +This file is part of the Zebra server. + +Zebra is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Zebra is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with Zebra; see the file LICENSE.zebra. If not, write to the +Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA +02111-1307, USA. +*/ + + #include @@ -17,7 +28,7 @@ #include #include -#include +#include #include "../index/index.h" /* isamd uses the internal structure of it_key */ #include "isamd-p.h" @@ -27,34 +38,48 @@ static void init_fc (ISAMD is, int cat); #define ISAMD_FREELIST_CHUNK 1 -#define SMALL_TEST 1 +#define SMALL_TEST 0 ISAMD_M isamd_getmethod (ISAMD_M me) { static struct ISAMD_filecat_s def_cat[] = { #if SMALL_TEST /* blocksz, max. Unused time being */ - { 20, 40 }, - { 32, 0 }, + { 32, 40 }, /* 24 is the smallest unreasonable size! */ + { 64, 0 }, #else - { 24, 1 }, + { 32, 1 }, + { 128, 1 }, + { 256, 1 }, + { 512, 1 }, + { 1024, 1 }, + { 2048, 1 }, + { 4096, 1 }, + { 8192, 0 }, + +#endif +#ifdef SKIPTHIS + + + + { 32, 1 }, + { 128, 1 }, + { 512, 1 }, + { 2048, 1 }, + { 8192, 1 }, + { 32768, 1 }, + {131072, 0 }, + + { 24, 1 }, /* Experimental sizes */ { 32, 1 }, { 64, 1 }, { 128, 1 }, { 256, 1 }, + { 512, 1 }, { 1024, 1 }, { 2048, 0 }, #endif -/* old values from isamc, long time ago... - { 24, 40 }, - { 128, 256 }, - { 512, 1024 }, - { 2048, 4096 }, - { 8192,16384 }, - { 32768, 0 }, -*/ - }; ISAMD_M m = (ISAMD_M) xmalloc (sizeof(*m)); /* never released! */ m->filecat = def_cat; /* ok, only alloc'd once */ @@ -136,14 +161,38 @@ ISAMD isamd_open (BFiles bfs, const char *name, int writeflag, ISAMD_M method) is->files[i].sum_backward = 0; is->files[i].no_next = 0; is->files[i].no_prev = 0; - + is->files[i].no_op_diffonly=0; + is->files[i].no_op_main=0; init_fc (is, i); } + is->last_pos=0; + is->last_cat=0; + is->no_read=0; + is->no_read_main=0; + is->no_write=0; + is->no_op_single=0; + is->no_op_new=0; + is->no_read_keys=0; + is->no_read_eof=0; + is->no_seek_nxt=0; + is->no_seek_sam=0; + is->no_seek_fwd=0; + is->no_seek_prv=0; + is->no_seek_bak=0; + is->no_seek_cat=0; + is->no_fbuilds=0; + is->no_appds=0; + is->no_merges=0; + is->no_non=0; + is->no_singles=0; + return is; } int isamd_block_used (ISAMD is, int type) { + if ( type==-1) /* singleton */ + return 0; if (type < 0 || type >= is->no_files) return -1; return is->files[type].head.lastblock-1; @@ -152,6 +201,8 @@ int isamd_block_used (ISAMD is, int type) int isamd_block_size (ISAMD is, int type) { ISAMD_filecat filecat = is->method->filecat; + if ( type==-1) /* singleton */ + return 0; /* no bytes used */ if (type < 0 || type >= is->no_files) return -1; return filecat[type].bsize; @@ -160,25 +211,27 @@ int isamd_block_size (ISAMD is, int type) int isamd_close (ISAMD is) { int i; + int s; if (is->method->debug>0) { - logf (LOG_LOG, "isamd: next forw mid-f prev backw mid-b"); + logf (LOG_LOG, "isamd statistics"); + logf (LOG_LOG, "f nxt forw mid-f prev backw mid-b"); for (i = 0; ino_files; i++) - logf (LOG_LOG, "isamd:%8d%8d%8.1f%8d%8d%8.1f", + logf (LOG_LOG, "%d%7d%7d%7.1f%7d%7d%7.1f",i, is->files[i].no_next, is->files[i].no_forward, is->files[i].no_forward ? - (double) is->files[i].sum_forward/is->files[i].no_forward - : 0.0, + (double) is->files[i].sum_forward/is->files[i].no_forward + : 0.0, is->files[i].no_prev, is->files[i].no_backward, is->files[i].no_backward ? - (double) is->files[i].sum_backward/is->files[i].no_backward - : 0.0); + (double) is->files[i].sum_backward/is->files[i].no_backward + : 0.0); } if (is->method->debug>0) - logf (LOG_LOG, "isamd: writes reads skipped alloc released remap"); + logf (LOG_LOG, "f writes reads skipped alloc released "); for (i = 0; ino_files; i++) { release_fc (is, i); @@ -187,36 +240,105 @@ int isamd_close (ISAMD is) bf_write (is->files[i].bf, 0, 0, sizeof(ISAMD_head), &is->files[i].head); if (is->method->debug>0) - logf (LOG_LOG, "isamd:%8d%8d%8d%8d%8d%8d", + logf (LOG_LOG, "%d%8d%8d%8d%8d%8d",i, is->files[i].no_writes, is->files[i].no_reads, is->files[i].no_skip_writes, is->files[i].no_allocated, - is->files[i].no_released, - is->files[i].no_remap); + is->files[i].no_released); xfree (is->files[i].fc_list); flush_block (is, i); bf_close (is->files[i].bf); } + + if (is->method->debug>0) + { + logf (LOG_LOG, "f opens main diffonly"); + for (i = 0; ino_files; i++) + { + logf (LOG_LOG, "%d%8d%8d%8d",i, + is->files[i].no_op_main+ + is->files[i].no_op_diffonly, + is->files[i].no_op_main, + is->files[i].no_op_diffonly); + } + logf(LOG_LOG,"open single %8d", is->no_op_single); + logf(LOG_LOG,"open new %8d", is->no_op_new); + + logf(LOG_LOG, "new build %8d", is->no_fbuilds); + logf(LOG_LOG, "append %8d", is->no_appds); + logf(LOG_LOG, " merges %8d", is->no_merges); + logf(LOG_LOG, " singles %8d", is->no_singles); + logf(LOG_LOG, " no-ops %8d", is->no_non); + + logf(LOG_LOG, "read blocks %8d", is->no_read); + logf(LOG_LOG, "read keys: %8d %8.1f k/bl", + is->no_read_keys, + 1.0*(is->no_read_keys+1)/(is->no_read+1) ); + logf(LOG_LOG, "read main-k %8d %8.1f %% of keys", + is->no_read_main, + 100.0*(is->no_read_main+1)/(is->no_read_keys+1) ); + logf(LOG_LOG, "read ends: %8d %8.1f k/e", + is->no_read_eof, + 1.0*(is->no_read_keys+1)/(is->no_read_eof+1) ); + s= is->no_seek_nxt+ is->no_seek_sam+ is->no_seek_fwd + + is->no_seek_prv+ is->no_seek_bak+ is->no_seek_cat; + if (s==0) + s++; + logf(LOG_LOG, "seek same %8d %8.1f%%", + is->no_seek_sam, 100.0*is->no_seek_sam/s ); + logf(LOG_LOG, "seek next %8d %8.1f%%", + is->no_seek_nxt, 100.0*is->no_seek_nxt/s ); + logf(LOG_LOG, "seek prev %8d %8.1f%%", + is->no_seek_prv, 100.0*is->no_seek_prv/s ); + logf(LOG_LOG, "seek forw %8d %8.1f%%", + is->no_seek_fwd, 100.0*is->no_seek_fwd/s ); + logf(LOG_LOG, "seek back %8d %8.1f%%", + is->no_seek_bak, 100.0*is->no_seek_bak/s ); + logf(LOG_LOG, "seek cat %8d %8.1f%%", + is->no_seek_cat, 100.0*is->no_seek_cat/s ); + } xfree (is->files); xfree (is->method); xfree (is); return 0; } +static void isamd_seek_stat(ISAMD is, int cat, int pos) +{ + if (cat != is->last_cat) + is->no_seek_cat++; + else if ( pos == is->last_pos) + is->no_seek_sam++; + else if ( pos == is->last_pos+1) + is->no_seek_nxt++; + else if ( pos == is->last_pos-1) + is->no_seek_prv++; + else if ( pos > is->last_pos) + is->no_seek_fwd++; + else if ( pos < is->last_pos) + is->no_seek_bak++; + is->last_cat = cat; + is->last_pos = pos; +} /* seek_stat */ + int isamd_read_block (ISAMD is, int cat, int pos, char *dst) { + isamd_seek_stat(is,cat,pos); ++(is->files[cat].no_reads); + ++(is->no_read); if (is->method->debug > 6) - logf (LOG_LOG, "isamd: read_block %d %d", cat, pos); + logf (LOG_LOG, "isamd: read_block %d:%d",cat, pos); return bf_read (is->files[cat].bf, pos, 0, 0, dst); } int isamd_write_block (ISAMD is, int cat, int pos, char *src) { + isamd_seek_stat(is,cat,pos); ++(is->files[cat].no_writes); + ++(is->no_write); if (is->method->debug > 6) - logf (LOG_LOG, "isamd: write_block %d %d", cat, pos); + logf (LOG_LOG, "isamd: write_block %d:%d", cat, pos); return bf_write (is->files[cat].bf, pos, 0, 0, src); } @@ -436,26 +558,56 @@ void isamd_pp_close (ISAMD_PP pp) (*is->method->code_stop)(ISAMD_DECODE, pp->decodeClientData); isamd_free_diffs(pp); /* see merge-d.h */ - xfree (pp->buf); - xfree (pp); if (is->method->debug > 5) - logf (LOG_LOG, "isamd_pp_close %p %d=%d:%d sz=%d n=%d=%d:%d", + logf (LOG_LOG, "isamd_pp_close %p %d=%d:%d sz=%d n=%d=%d:%d nk=%d", pp, isamd_addr(pp->pos, pp->cat), pp->cat, pp->pos, pp->size, - pp->next, isamd_type(pp->next), isamd_block(pp->next) ); + pp->next, isamd_type(pp->next), isamd_block(pp->next), + pp->numKeys ); + xfree (pp->buf); + xfree (pp); } +ISAMD_PP isamd_pp_create (ISAMD is, int cat) +/* creates a pp_buff without data in it. pos=0, cat as given */ +{ + ISAMD_PP pp = (ISAMD_PP) xmalloc (sizeof(*pp)); + int sz = is->method->filecat[is->max_cat].bsize; + + pp->numKeys = 0; + pp->buf = (char *) xmalloc (sz); + memset(pp->buf,'\0',sz); /* clear the buffer, for new blocks */ + + pp->next = 0; + pp->size = 0; + pp->offset = 0; + pp->is = is; + pp->diffs=0; + pp->diffbuf=0; + pp->diffinfo=0; + pp->decodeClientData = (*is->method->code_start)(ISAMD_DECODE); + pp->cat = cat; + pp->pos = 0; + is->no_op_new++; + return pp; + +} -ISAMD_PP isamd_pp_open (ISAMD is, ISAMD_P ipos) + +ISAMD_PP isamd_pp_open (ISAMD is, const char *dictbuf, int dictlen) { + ISAMD_P ipos; ISAMD_PP pp = (ISAMD_PP) xmalloc (sizeof(*pp)); char *src; int sz = is->method->filecat[is->max_cat].bsize; /* always allocate for the largest blocks, saves trouble */ - - pp->cat = isamd_type(ipos); - pp->pos = isamd_block(ipos); - + struct it_key singlekey; + char *c_ptr; /* for fake encoding the singlekey */ + char *i_ptr; + int ofs; + int dictnum; + + pp->numKeys = 0; src = pp->buf = (char *) xmalloc (sz); memset(src,'\0',sz); /* clear the buffer, for new blocks */ @@ -463,13 +615,36 @@ ISAMD_PP isamd_pp_open (ISAMD is, ISAMD_P ipos) pp->size = 0; pp->offset = 0; pp->is = is; - pp->decodeClientData = (*is->method->code_start)(ISAMD_DECODE); - pp->numKeys = 0; pp->diffs=0; - pp->diffbuf=0; pp->diffinfo=0; + pp->decodeClientData = (*is->method->code_start)(ISAMD_DECODE); + dictnum=*dictbuf; // numkeys for internals, 0 for externals + + if (0==dictnum) + { + memcpy(&ipos, dictbuf+1, sizeof(ISAMD_P) ); + } + else /* dictionary block, fake a real one */ + { + pp->cat=0; + pp->pos=0; + if (is->method->debug > 5) + logf (LOG_LOG, "isamd_pp_open dict"); + pp->numKeys=(unsigned char) dictbuf[0]; + memcpy(pp->buf+ISAMD_BLOCK_OFFSET_1, dictbuf+1,dictlen-1); + pp->size=pp->offset=dictlen+ISAMD_BLOCK_OFFSET_1-1; + is->no_op_single++; + return pp; + } /* dict block */ + + pp->cat = isamd_type(ipos); + pp->pos = isamd_block(ipos); + + if (0==pp->pos) + is->no_op_new++; + if (pp->pos) { src = pp->buf; @@ -480,18 +655,20 @@ ISAMD_PP isamd_pp_open (ISAMD is, ISAMD_P ipos) src += sizeof(pp->size); memcpy (&pp->numKeys, src, sizeof(pp->numKeys)); src += sizeof(pp->numKeys); - memcpy (&pp->diffs, src, sizeof(pp->diffs)); - src += sizeof(pp->diffs); - assert (pp->next != pp->pos); + assert (pp->next != isamd_addr(pp->pos,pp->cat)); pp->offset = src - pp->buf; assert (pp->offset == ISAMD_BLOCK_OFFSET_1); + assert(pp->size>=ISAMD_BLOCK_OFFSET_1); /*??*/ + if (pp->next) + is->files[pp->cat].no_op_main++; + else + is->files[pp->cat].no_op_diffonly++; } if (is->method->debug > 5) logf (LOG_LOG, "isamd_pp_open %p %d=%d:%d sz=%d n=%d=%d:%d", pp, isamd_addr(pp->pos, pp->cat), pp->cat, pp->pos, pp->size, pp->next, isamd_type(pp->next), isamd_block(pp->next) ); - - + return pp; } @@ -500,22 +677,19 @@ ISAMD_PP isamd_pp_open (ISAMD is, ISAMD_P ipos) void isamd_buildfirstblock(ISAMD_PP pp){ char *dst=pp->buf; assert(pp->buf); - assert(pp->next != pp->pos); + assert(pp->next != isamd_addr(pp->pos,pp->cat)); memcpy(dst, &pp->next, sizeof(pp->next) ); dst += sizeof(pp->next); memcpy(dst, &pp->size,sizeof(pp->size)); dst += sizeof(pp->size); memcpy(dst, &pp->numKeys, sizeof(pp->numKeys)); dst += sizeof(pp->numKeys); - memcpy(dst, &pp->diffs, sizeof(pp->diffs)); - dst += sizeof(pp->diffs); assert (dst - pp->buf == ISAMD_BLOCK_OFFSET_1); if (pp->is->method->debug > 5) - logf (LOG_LOG, "isamd: first: sz=%d p=%d/%d>%d/%d nk=%d d=%d", - pp->size, - pp->cat, pp->pos, - isamd_type(pp->next), isamd_block(pp->next), - pp->numKeys, pp->diffs); + logf (LOG_LOG, "isamd: bldfirst: p=%d=%d:%d n=%d:%d:%d sz=%d nk=%d ", + isamd_addr(pp->pos,pp->cat),pp->cat, pp->pos, + pp->next, isamd_type(pp->next), isamd_block(pp->next), + pp->size, pp->numKeys); } void isamd_buildlaterblock(ISAMD_PP pp){ @@ -539,9 +713,10 @@ void isamd_buildlaterblock(ISAMD_PP pp){ /* returns non-zero if item could be read; 0 otherwise */ int isamd_pp_read (ISAMD_PP pp, void *buf) { + return isamd_read_item (pp, (char **) &buf); - /* note: isamd_read_item is in merge-d.c, because it is so */ - /* convoluted with the merge process */ + /* note: isamd_read_item is in merge-d.c, because it is so */ + /* convoluted with the merge process */ } /* read one main item from file - decode and store it in *dst. @@ -555,6 +730,7 @@ int isamd_read_main_item (ISAMD_PP pp, char **dst) ISAMD is = pp->is; char *src = pp->buf + pp->offset; int newcat; + int oldoffs; if (pp->offset >= pp->size) { @@ -587,7 +763,7 @@ int isamd_read_main_item (ISAMD_PP pp, char **dst) newcat = isamd_type(pp->next); pp->pos = isamd_block(pp->next); pp->cat = isamd_type(pp->next); - + pp->is->no_read_main++; src = pp->buf; /* read block and save 'next' and 'size' entry */ isamd_read_block (is, pp->cat, pp->pos, src); @@ -596,18 +772,24 @@ int isamd_read_main_item (ISAMD_PP pp, char **dst) memcpy (&pp->size, src, sizeof(pp->size)); src += sizeof(pp->size); /* assume block is non-empty */ - assert (src - pp->buf == ISAMD_BLOCK_OFFSET_N); + pp->offset = oldoffs = src - pp->buf; + assert (pp->offset == ISAMD_BLOCK_OFFSET_N); assert (pp->next != isamd_addr(pp->pos,pp->cat)); (*is->method->code_reset)(pp->decodeClientData); + /* finally, read the item */ (*is->method->code_item)(ISAMD_DECODE, pp->decodeClientData, dst, &src); pp->offset = src - pp->buf; - if (is->method->debug > 4) - logf (LOG_LOG, "isamd: read_block size=%d %d %d next=%d", - pp->size, pp->cat, pp->pos, pp->next); + if (is->method->debug > 8) + logf (LOG_LOG, "isamd: read_m: block %d:%d sz=%d ofs=%d-%d next=%d", + pp->cat, pp->pos, pp->size, oldoffs, pp->offset, pp->next); return 2; } + oldoffs=pp->offset; (*is->method->code_item)(ISAMD_DECODE, pp->decodeClientData, dst, &src); pp->offset = src - pp->buf; + if (is->method->debug > 8) + logf (LOG_LOG, "isamd: read_m: got %d:%d sz=%d ofs=%d-%d next=%d", + pp->cat, pp->pos, pp->size, oldoffs, pp->offset, pp->next); return 1; } @@ -631,6 +813,8 @@ static char *hexdump(unsigned char *p, int len, char *buff) { } +#ifdef SKIPTHIS + /* needs different arguments, or something */ void isamd_pp_dump (ISAMD is, ISAMD_P ipos) { ISAMD_PP pp; @@ -639,23 +823,35 @@ void isamd_pp_dump (ISAMD is, ISAMD_P ipos) int i,n; int occur =0; int oldoffs; + int diffmax=1; + int diffidx; char hexbuff[64]; + int olddebug= is->method->debug; + is->method->debug=0; /* no debug logs while reading for dump */ logf(LOG_LOG,"dumping isamd block %d (%d:%d)", (int)ipos, isamd_type(ipos), isamd_block(ipos) ); pp=isamd_pp_open(is,ipos); - logf(LOG_LOG,"numKeys=%d, ofs=%d d=%d", - pp->numKeys, - pp->offset, pp->diffs); - oldoffs= pp->offset; + logf(LOG_LOG,"numKeys=%d, ofs=%d sz=%d", + pp->numKeys, pp->offset, pp->size ); + diffidx=oldoffs= pp->offset; + while ((diffidx < is->method->filecat[pp->cat].bsize) && (diffmax>0)) + { + memcpy(&diffmax,&(pp->buf[diffidx]),sizeof(int)); + logf (LOG_LOG,"diff set at %d-%d: %s", diffidx, diffmax, + hexdump(pp->buf+diffidx,8,0)); + /*! todo: dump the actual diffs as well !!! */ + diffidx=diffmax; + + } /* dump diffs */ while(isamd_pp_read(pp, &key)) { if (oldaddr != isamd_addr(pp->pos,pp->cat) ) { oldaddr = isamd_addr(pp->pos,pp->cat); - logf(LOG_LOG,"block %d (%d:%d) sz=%d nx=%d (%d:%d) ofs=%d", - isamd_addr(pp->pos,pp->cat), - pp->cat, pp->pos, pp->size, + logf(LOG_LOG,"block %d=%d:%d sz=%d nx=%d=%d:%d ofs=%d", + isamd_addr(pp->pos,pp->cat), pp->cat, pp->pos, + pp->size, pp->next, isamd_type(pp->next), isamd_block(pp->next), pp->offset); i=0; @@ -678,11 +874,62 @@ void isamd_pp_dump (ISAMD is, ISAMD_P ipos) } /*!*/ /*TODO: dump diffs too!!! */ isamd_pp_close(pp); + is->method->debug=olddebug; } /* dump */ +#endif + /* * $Log: isamd.c,v $ - * Revision 1.8 1999-08-18 13:28:16 heikki + * Revision 1.23 2002-08-02 19:26:56 adam + * Towards GPL + * + * Revision 1.22 2002/07/12 18:12:21 heikki + * Isam-D now stores small entries directly in the dictionary. + * Needs more tuning and cleaning... + * + * Revision 1.21 2002/07/11 16:16:00 heikki + * Fixed a bug in isamd, failed to store a single key when its bits + * did not fit into a singleton. + * + * Revision 1.20 2002/06/19 10:29:18 adam + * align block sizes for isam sys. Better plot for test + * + * Revision 1.19 1999/11/30 13:48:04 adam + * Improved installation. Updated for inclusion of YAZ header files. + * + * Revision 1.18 1999/10/06 15:18:13 heikki + * + * Improving block sizes again + * + * Revision 1.17 1999/10/06 11:46:36 heikki + * mproved statistics on isam-d + * + * Revision 1.16 1999/10/05 09:57:40 heikki + * Tuning the isam-d (and fixed a small "detail") + * + * Revision 1.15 1999/09/27 14:36:36 heikki + * singletons + * + * Revision 1.14 1999/09/23 18:01:18 heikki + * singleton optimising + * + * Revision 1.13 1999/09/20 15:48:06 heikki + * Small changes + * + * Revision 1.12 1999/09/13 13:28:28 heikki + * isam-d optimizing: merging input data in the same go + * + * Revision 1.11 1999/08/25 18:09:24 heikki + * Starting to optimize + * + * Revision 1.10 1999/08/24 13:17:42 heikki + * Block sizes, comments + * + * Revision 1.9 1999/08/20 12:25:58 heikki + * Statistics in isamd + * + * Revision 1.8 1999/08/18 13:28:16 heikki * Set log levels to decent values * * Revision 1.6 1999/08/17 19:44:25 heikki @@ -699,4 +946,4 @@ void isamd_pp_dump (ISAMD is, ISAMD_P ipos) * Copied from isamh, starting to change things... * * - */ \ No newline at end of file + */