X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=isamb%2Fisamb.c;h=77114239f2788119545f408d864964f48b1636be;hb=1afbdd9135e38942201f0ad0eaae5b6903755aeb;hp=bc38a557bfb0343dd921d7725da5c24c6a292558;hpb=842cbb4cd47fd56905c92429a6ede3a550b618cf;p=idzebra-moved-to-github.git diff --git a/isamb/isamb.c b/isamb/isamb.c index bc38a55..7711423 100644 --- a/isamb/isamb.c +++ b/isamb/isamb.c @@ -1,4 +1,4 @@ -/* $Id: isamb.c,v 1.28 2004-05-30 18:04:49 adam Exp $ +/* $Id: isamb.c,v 1.39 2004-06-02 18:50:37 heikki Exp $ Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003,2004 Index Data Aps @@ -26,6 +26,10 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA #include #include +#ifndef ISAMB_DEBUG +#define ISAMB_DEBUG 0 +#endif + struct ISAMB_head { int first_block; int last_block; @@ -39,6 +43,7 @@ struct ISAMB_head { /* maximum size of encoded buffer */ #define DST_ITEM_MAX 256 +#define ISAMB_MAX_LEVEL 10 /* approx 2*max page + max size of item */ #define DST_BUF_SIZE 16840 @@ -77,6 +82,10 @@ struct ISAMB_s { int cache; /* 0=no cache, 1=use cache, -1=dummy isam (for testing only) */ int log_io; /* log level for bf_read/bf_write calls */ int log_freelist; /* log level for freelist handling */ + int skipped_numbers; /* on a leaf node */ + int returned_numbers; + int skipped_nodes[ISAMB_MAX_LEVEL]; /* [0]=skipped leaves, 1=higher etc */ + int accessed_nodes[ISAMB_MAX_LEVEL]; /* nodes we did not skip */ }; struct ISAMB_block { @@ -97,8 +106,13 @@ struct ISAMB_PP_s { ISAMB isamb; ISAMB_P pos; int level; + int maxlevel; /* total depth */ int total_size; int no_blocks; + int skipped_numbers; /* on a leaf node */ + int returned_numbers; + int skipped_nodes[ISAMB_MAX_LEVEL]; /* [0]=skipped leaves, 1=higher etc */ + int accessed_nodes[ISAMB_MAX_LEVEL]; /* nodes we did not skip */ struct ISAMB_block **block; }; @@ -160,6 +174,10 @@ ISAMB isamb_open (BFiles bfs, const char *name, int writeflag, ISAMC_M *method, isamb->log_io = 0; isamb->log_freelist = 0; isamb->cache = cache; + isamb->skipped_numbers=0; + isamb->returned_numbers=0; + for (i=0;iskipped_nodes[i]= isamb->accessed_nodes[i]=0; assert (cache == 0); isamb->file = xmalloc (sizeof(*isamb->file) * isamb->no_cat); @@ -190,6 +208,9 @@ ISAMB isamb_open (BFiles bfs, const char *name, int writeflag, ISAMC_M *method, assert(isamb->file[i].head.block_size == b_size); b_size = b_size * 4; } +#if ISAMB_DEBUG + logf(LOG_WARN, "isamb debug enabled. Things will be slower than usual"); +#endif return isamb; } @@ -286,6 +307,11 @@ static int get_block (ISAMB b, ISAMC_P pos, char *userbuf, int wr) void isamb_close (ISAMB isamb) { int i; + for (i=0;isamb->accessed_nodes[i];i++) + logf(LOG_DEBUG,"isamb_close level leaf-%d: %d read, %d skipped", + i, isamb->accessed_nodes[i], isamb->skipped_nodes[i]); + logf(LOG_DEBUG,"isamb_close returned %d values, skipped %d", + isamb->skipped_numbers, isamb->returned_numbers); for (i = 0; ino_cat; i++) { flush_blocks (isamb, i); @@ -300,8 +326,7 @@ void isamb_close (ISAMB isamb) xfree (isamb); } - -struct ISAMB_block *open_block (ISAMB b, ISAMC_P pos) +static struct ISAMB_block *open_block (ISAMB b, ISAMC_P pos) { int cat = pos&CAT_MASK; struct ISAMB_block *p; @@ -327,13 +352,14 @@ struct ISAMB_block *open_block (ISAMB b, ISAMC_P pos) p->size = (p->buf[1] + 256 * p->buf[2]) - ISAMB_DATA_OFFSET; if (p->size < 0) { - fprintf (stderr, "pos=%d\n", pos); + yaz_log (LOG_FATAL, "Bad block size %d in pos=%d\n", p->size, pos); } assert (p->size >= 0); p->offset = 0; p->dirty = 0; p->deleted = 0; p->decodeClientData = (*b->method->code_start)(ISAMC_DECODE); + yaz_log (LOG_DEBUG, "isamb_open_block: Opened block %d ofs=%d",pos, p->offset); return p; } @@ -898,14 +924,20 @@ int isamb_merge (ISAMB b, ISAMC_P pos, ISAMC_I *stream) ISAMB_PP isamb_pp_open_x (ISAMB isamb, ISAMB_P pos, int *level) { ISAMB_PP pp = xmalloc (sizeof(*pp)); + int i; pp->isamb = isamb; - pp->block = xmalloc (10 * sizeof(*pp->block)); + pp->block = xmalloc (ISAMB_MAX_LEVEL * sizeof(*pp->block)); pp->pos = pos; pp->level = 0; + pp->maxlevel=0; pp->total_size = 0; pp->no_blocks = 0; + pp->skipped_numbers=0; + pp->returned_numbers=0; + for (i=0;iskipped_nodes[i] = pp->accessed_nodes[i]=0; while (1) { struct ISAMB_block *p = open_block (isamb, pos); @@ -914,15 +946,17 @@ ISAMB_PP isamb_pp_open_x (ISAMB isamb, ISAMB_P pos, int *level) pp->total_size += p->size; pp->no_blocks++; - if (p->leaf) break; + decode_ptr (&src, &pos); p->offset = src - p->bytes; pp->level++; + pp->accessed_nodes[pp->level]++; } pp->block[pp->level+1] = 0; + pp->maxlevel=pp->level; if (level) *level = pp->level; return pp; @@ -938,6 +972,19 @@ void isamb_pp_close_x (ISAMB_PP pp, int *size, int *blocks) int i; if (!pp) return; + logf(LOG_DEBUG,"isamb_pp_close lev=%d returned %d values, skipped %d", + pp->maxlevel, pp->skipped_numbers, pp->returned_numbers); + for (i=pp->maxlevel;i>=0;i--) + if ( pp->skipped_nodes[i] || pp->accessed_nodes[i]) + logf(LOG_DEBUG,"isamb_pp_close level leaf-%d: %d read, %d skipped", i, + pp->accessed_nodes[i], pp->skipped_nodes[i]); + pp->isamb->skipped_numbers += pp->skipped_numbers; + pp->isamb->returned_numbers += pp->returned_numbers; + for (i=pp->maxlevel;i>=0;i--) + { + pp->isamb->accessed_nodes[i] += pp->accessed_nodes[i]; + pp->isamb->skipped_nodes[i] += pp->skipped_nodes[i]; + } if (size) *size = pp->total_size; if (blocks) @@ -960,6 +1007,69 @@ void isamb_pp_close (ISAMB_PP pp) isamb_pp_close_x (pp, 0, 0); } +/* simple recursive dumper .. */ +static void isamb_dump_r (ISAMB b, ISAMB_P pos, void (*pr)(const char *str), + int level) +{ + char buf[1024]; + char prefix_str[1024]; + if (pos) + { + struct ISAMB_block *p = open_block (b, pos); + sprintf(prefix_str, "%*s %d cat=%d size=%d max=%d", level*2, "", + pos, p->cat, p->size, b->file[p->cat].head.block_max); + (*pr)(prefix_str); + sprintf(prefix_str, "%*s %d", level*2, "", pos); + if (p->leaf) + { + while (p->offset < p->size) + { + char *src = p->bytes + p->offset; + char *dst = buf; + (*b->method->code_item)(ISAMC_DECODE, p->decodeClientData, + &dst, &src); + (*b->method->log_item)(LOG_DEBUG, buf, prefix_str); + p->offset = src - (char*) p->bytes; + } + assert(p->offset == p->size); + } + else + { + char *src = p->bytes + p->offset; + int sub; + int item_len; + + decode_ptr (&src, &sub); + p->offset = src - (char*) p->bytes; + + isamb_dump_r(b, sub, pr, level+1); + + while (p->offset < p->size) + { + decode_ptr (&src, &item_len); + (*b->method->log_item)(LOG_DEBUG, src, prefix_str); + src += item_len; + decode_ptr (&src, &sub); + + p->offset = src - (char*) p->bytes; + + isamb_dump_r(b, sub, pr, level+1); + } + } + close_block(b,p); + } +} + +void isamb_dump (ISAMB b, ISAMB_P pos, void (*pr)(const char *str)) +{ + return isamb_dump_r(b, pos, pr, 0); +} + +#if 0 +/* Old isamb_pp_read that Adam wrote, kept as a reference in case we need to + debug the more complex pp_read that also forwards. May be deleted near end + of 2004, if it has not shown to be useful */ + int isamb_pp_read (ISAMB_PP pp, void *buf) { char *dst = buf; @@ -979,7 +1089,7 @@ int isamb_pp_read (ISAMB_PP pp, void *buf) pp->block[pp->level] = 0; (pp->level)--; p = pp->block[pp->level]; - assert (!p->leaf); /* must be int */ + assert (!p->leaf); } src = p->bytes + p->offset; @@ -998,7 +1108,7 @@ int isamb_pp_read (ISAMB_PP pp, void *buf) pp->total_size += p->size; pp->no_blocks++; - if (p->leaf) /* leaf */ + if (p->leaf) { break; } @@ -1014,8 +1124,405 @@ int isamb_pp_read (ISAMB_PP pp, void *buf) (*pp->isamb->method->code_item)(ISAMC_DECODE, p->decodeClientData, &dst, &src); p->offset = src - (char*) p->bytes; + /* key_logdump_txt(LOG_DEBUG,buf, "isamb_pp_read returning 1"); */ + return 1; +} + +#else +int isamb_pp_read (ISAMB_PP pp, void *buf) +{ + return isamb_pp_forward(pp,buf,0); +} +#endif + +#define NEW_FORWARD 0 +#if NEW_FORWARD + +#define ISAMB_DEBUG 1 /* while coding this part */ + +static int isamb_pp_read_on_leaf(ISAMB_PP pp, void *buf) +{ /* reads the next item on the current leaf, returns 0 if end of leaf*/ + struct ISAMB_block *p = pp->block[pp->level]; + char *dst; + char *src; + if (p->offset == p->size) { +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_read_on_leaf returning 0 on node %d",p->pos); +#endif + return 0; /* at end of leaf */ + } + src=p->bytes + p->offset; + dst=buf; + (*pp->isamb->method->code_item) + (ISAMC_DECODE, p->decodeClientData,&dst, &src); + p->offset = src - (char*) p->bytes; +#if ISAMB_DEBUG + (*pp->isamb->method->log_item)(LOG_DEBUG, buf, "read_on_leaf returning 1"); +#endif + return 1; +} /* read_on_leaf */ + + +static int isamb_pp_climb_level(ISAMB_PP pp, int *pos) +{ /* climbs higher in the tree, until finds a level with data left */ + /* returns the node to (consider to) descend to in *pos) */ + struct ISAMB_block *p = pp->block[pp->level]; + char *src; + int item_len; + assert(p->offset <= p->size); + if (pp->level==0) + { +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_climb_level returning 0 at root"); + return 0; +#endif + } + close_block(pp->isamb, pp->block[pp->level]); + pp->block[pp->level]=0; + (pp->level)--; + p=pp->block[pp->level]; + logf(LOG_DEBUG,"isamb_pp_climb_level climbed to node %d ofs=%d", + p->pos, p->offset); + assert (!p->leaf); + assert (p->offset <= p->size); + if (p->offset == p->size ) { + /* we came from the last pointer, climb on */ + if (!isamb_pp_climb_level(pp,pos)) + return 0; + p=pp->block[pp->level]; + } + /* skip the child we just came from */ + assert (p->offset < p->size ); + src=p->bytes + p->offset; + decode_ptr(&src, &item_len); + src += item_len; + decode_ptr(&src, pos); + p->offset=src - (char *)p->bytes; + return 1; +} /* climb_level */ + + +static int isamb_pp_forward_unode(ISAMB_PP pp, int pos, const void *untilbuf) +{ /* scans a upper node until it finds a child <= untilbuf */ + /* pp points to the key value, as always. pos is the child read from */ + /* the buffer */ + /* if all values are too small, returns the last child in the node */ + /* FIXME - this can be detected, and avoided by looking at the */ + /* parent node, but that gets messy. Presumably the cost is */ + /* pretty low anyway */ + struct ISAMB_block *p = pp->block[pp->level]; + char *src=p->bytes + p->offset; + int item_len; + int cmp; + int nxtpos; + assert(!p->leaf); + assert(p->offset <= p->size); + if (p->offset == p->size) + return pos; /* already at the end of it */ + while(p->offset < p->size) { + decode_ptr(&src,&item_len); + cmp=(*pp->isamb->method->compare_item)(untilbuf,src); + src+=item_len; + decode_ptr(&src,&nxtpos); + if (cmp<2) + { + return pos; + } /* found one */ + pos=nxtpos; + p->offset=src-(char*)p->bytes; + } + return pos; /* that's the last one in the line */ + +} /* forward_unode */ + +static void isamb_pp_descend_to_leaf(ISAMB_PP pp, int pos, const void *untilbuf) +{ /* climbs down the tree, from pos, to the leftmost leaf */ + struct ISAMB_block *p = pp->block[pp->level]; + char *src; + assert(!p->leaf); + ++(pp->level); + p=open_block(pp->isamb, pos); + pp->block[pp->level]=p; + ++(pp->accessed_nodes[pp->maxlevel-pp->level]); + ++(pp->no_blocks); +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_descend_to_leaf " + "got lev %d node %d lf=%d", + pp->level, p->pos, p->leaf); +#endif + if (p->leaf) + return; + assert (p->offset==0 ); + src=p->bytes + p->offset; + decode_ptr(&src, &pos); + p->offset=src-(char*)p->bytes; + if (untilbuf) + pos=isamb_pp_forward_unode(pp,pos,untilbuf); + isamb_pp_descend_to_leaf(pp,pos,untilbuf); +} /* descend_to_leaf */ + +static int isamb_pp_find_next_leaf(ISAMB_PP pp) +{ /* finds the next leaf by climbing up and down */ + int pos; + if (!isamb_pp_climb_level(pp,&pos)) + return 0; + isamb_pp_descend_to_leaf(pp, pos,0); return 1; } +static int isamb_pp_forward_on_leaf(ISAMB_PP pp, void *buf, const void *untilbuf) +{ /* forwards on the current leaf, returns 0 if not found */ + int cmp; + int skips=0; + while (1){ + if (!isamb_pp_read_on_leaf(pp,buf)) + return 0; + /* FIXME - this is an extra function call, inline the read? */ + cmp=(*pp->isamb->method->compare_item)(untilbuf,buf); + if (cmp <2){ /* found a good one */ +#if ISAMB_DEBUG + if (skips) + logf(LOG_DEBUG, "isam_pp_fwd_on_leaf skipped %d items",skips); +#endif + pp->returned_numbers++; + return 1; + } + pp->skipped_numbers++; + skips++; + } +} /* forward_on_leaf */ + +static int isamb_pp_climb_desc(ISAMB_PP pp, void *buf, const void *untilbuf) +{ /* climbs up and descends to a leaf where values >= *untilbuf are found */ + int pos; + if (!isamb_pp_climb_level(pp,&pos)) + return 0; + isamb_pp_descend_to_leaf(pp, pos,untilbuf); + return 1; +} /* climb_desc */ + +int isamb_pp_forward (ISAMB_PP pp, void *buf, const void *untilbuf) +{ +#if ISAMB_DEBUG + struct ISAMB_block *p = pp->block[pp->level]; + assert(p->leaf); +#endif + if (untilbuf) { + if (isamb_pp_forward_on_leaf( pp, buf, untilbuf)) + return 1; + if (! isamb_pp_climb_desc( pp, buf, untilbuf)) + return 0; /* could not find a leaf */ + do{ + if (isamb_pp_forward_on_leaf( pp, buf, untilbuf)) + return 1; + }while ( isamb_pp_find_next_leaf(pp)); + return 0; /* could not find at all */ + } + else { /* no untilbuf, a straight read */ + /* FIXME - this should be moved + * directly into the pp_read */ + /* keeping here now, to keep same + * interface as the old fwd */ + if (isamb_pp_read_on_leaf( pp, buf)) + return 1; + if (isamb_pp_find_next_leaf(pp)) + return isamb_pp_read_on_leaf(pp, buf); + else + return 0; + } +} /* isam_pp_forward (new version) */ + +#else + +int isamb_pp_forward (ISAMB_PP pp, void *buf, const void *untilbuf) +{ + /* pseudocode: + * while 1 + * while at end of node + * climb higher. If out, return 0 + * while not on a leaf (and not at its end) + * decode next + * if cmp + * descend to node + * decode next + * if cmp + * return 1 + */ + /* + * The upper nodes consist of a sequence of nodenumbers and keys + * When opening a block, the first node number is read in, and + * offset points to the first key, which is the upper limit of keys + * in the node just read. + */ + char *dst = buf; + char *src; + struct ISAMB_block *p = pp->block[pp->level]; + int cmp; + int item_len; + int pos; + int nxtpos; + int descending=0; /* used to prevent a border condition error */ + if (!p) + return 0; +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_forward starting [%p] p=%d",pp,p->pos); + + (*pp->isamb->method->log_item)(LOG_DEBUG, untilbuf, "until"); + (*pp->isamb->method->log_item)(LOG_DEBUG, buf, "buf"); +#endif + + while (1) + { + while ( (p->offset == p->size) && !descending ) + { /* end of this block - climb higher */ + assert (p->offset <= p->size); +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_forward climbing from l=%d", + pp->level); +#endif + if (pp->level == 0) + { +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_forward returning 0 at root"); +#endif + return 0; /* at end of the root, nothing left */ + } + close_block(pp->isamb, pp->block[pp->level]); + pp->block[pp->level]=0; + (pp->level)--; + p=pp->block[pp->level]; +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_forward climbed to node %d off=%d", + p->pos, p->offset); +#endif + assert(!p->leaf); + assert(p->offset <= p->size); + /* skip the child we have handled */ + if (p->offset != p->size) + { + src = p->bytes + p->offset; + decode_ptr(&src, &item_len); +#if ISAMB_DEBUG + (*pp->isamb->method->log_item)(LOG_DEBUG, src, + " isamb_pp_forward " + "climb skipping old key"); +#endif + src += item_len; + decode_ptr(&src,&pos); + p->offset = src - (char*) p->bytes; + break; /* even if this puts us at the end of the block, we + need to descend to the last pos. UGLY coding, + clean up some day */ + } + } + if (!p->leaf) + { + src = p->bytes + p->offset; + if (p->offset == p->size) + cmp=-2 ; /* descend to the last node, as we have + no value to cmp */ + else + { + decode_ptr(&src, &item_len); +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_forward (B) on a high node. " + "ofs=%d sz=%d nxtpos=%d ", + p->offset,p->size,pos); + (*pp->isamb->method->log_item)(LOG_DEBUG, src, ""); +#endif + if (untilbuf) + cmp=(*pp->isamb->method->compare_item)(untilbuf,src); + else + cmp=-2; + src += item_len; + decode_ptr(&src,&nxtpos); + } + if (cmp<2) + { +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isambb_pp_forward descending l=%d p=%d ", + pp->level, pos); +#endif + descending=1; /* prevent climbing for a while */ + ++(pp->level); + p = open_block(pp->isamb,pos); + pp->block[pp->level] = p ; + pp->total_size += p->size; + (pp->accessed_nodes[pp->maxlevel - pp->level])++; + pp->no_blocks++; + if ( !p->leaf) + { /* block starts with a pos */ + src = p->bytes + p->offset; + decode_ptr(&src,&pos); + p->offset=src-(char*) p->bytes; +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_forward: block %d starts with %d", + p->pos, pos); +#endif + } + } /* descend to the node */ + else + { /* skip the node */ + p->offset = src - (char*) p->bytes; + pos=nxtpos; + (pp->skipped_nodes[pp->maxlevel - pp->level -1])++; +#if ISAMB_DEBUG + logf(LOG_DEBUG, + "isamb_pp_forward: skipping block on level %d, noting " + "on %d (%d)", + pp->level, pp->maxlevel - pp->level-1 , + pp->skipped_nodes[pp->maxlevel - pp->level-1 ]); +#endif + /* 0 is always leafs, 1 is one level above leafs etc, no + * matter how high tree */ + } + } /* not on a leaf */ + else + { /* on a leaf */ + if (p->offset == p->size) { + descending = 0; + } + else + { + assert (p->offset < p->size); + src = p->bytes + p->offset; + dst=buf; + (*pp->isamb->method->code_item)(ISAMC_DECODE, p->decodeClientData, + &dst, &src); + p->offset = src - (char*) p->bytes; + if (untilbuf) + cmp=(*pp->isamb->method->compare_item)(untilbuf,buf); + else + cmp=-2; +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_forward on a leaf. cmp=%d", + cmp); + (*pp->isamb->method->log_item)(LOG_DEBUG, buf, ""); +#endif + if (cmp <2) + { +#if ISAMB_DEBUG + if (untilbuf) + { + (*pp->isamb->method->log_item)( + LOG_DEBUG, buf, "isamb_pp_forward returning 1"); + } + else + { + (*pp->isamb->method->log_item)( + LOG_DEBUG, buf, "isamb_pp_read returning 1 (fwd)"); + } +#endif + pp->returned_numbers++; + return 1; + } + else + pp->skipped_numbers++; + } + } /* leaf */ + } /* main loop */ +} + +#endif int isamb_pp_num (ISAMB_PP pp) {