X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=isamb%2Fisamb.c;h=1a7b9bdfe673a5d3a12363fef7778d45f8a3fa36;hb=d587b2b301c06be60ad255eee6f7010cb2d56fe7;hp=2f439718372435d69d2cda4981fb6373da0afd8d;hpb=12d4ad07fc9bae26217f7d6bc17181a1b892337a;p=idzebra-moved-to-github.git diff --git a/isamb/isamb.c b/isamb/isamb.c index 2f43971..1a7b9bd 100644 --- a/isamb/isamb.c +++ b/isamb/isamb.c @@ -1,4 +1,4 @@ -/* $Id: isamb.c,v 1.35 2004-06-02 07:53:31 adam Exp $ +/* $Id: isamb.c,v 1.44 2004-06-04 13:54:56 heikki Exp $ Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003,2004 Index Data Aps @@ -97,6 +97,7 @@ struct ISAMB_block { int deleted; int offset; char *bytes; + char *cbuf; unsigned char *buf; void *decodeClientData; int log_rw; @@ -116,6 +117,9 @@ struct ISAMB_PP_s { struct ISAMB_block **block; }; +void isamb_pp_pos( ISAMB_PP pp, int *current, int *total ); + /* FIXME - this should be in a header file */ + #if ISAMB_PTR_CODEC static void encode_ptr (char **dst, unsigned pos) { @@ -326,8 +330,7 @@ void isamb_close (ISAMB isamb) xfree (isamb); } - -struct ISAMB_block *open_block (ISAMB b, ISAMC_P pos) +static struct ISAMB_block *open_block (ISAMB b, ISAMC_P pos) { int cat = pos&CAT_MASK; struct ISAMB_block *p; @@ -337,6 +340,7 @@ struct ISAMB_block *open_block (ISAMB b, ISAMC_P pos) p->pos = pos; p->cat = pos & CAT_MASK; p->buf = xmalloc (b->file[cat].head.block_size); + p->cbuf = 0; if (!get_block (b, pos, p->buf, 0)) { @@ -1008,12 +1012,70 @@ void isamb_pp_close (ISAMB_PP pp) isamb_pp_close_x (pp, 0, 0); } +/* simple recursive dumper .. */ +static void isamb_dump_r (ISAMB b, ISAMB_P pos, void (*pr)(const char *str), + int level) +{ + char buf[1024]; + char prefix_str[1024]; + if (pos) + { + struct ISAMB_block *p = open_block (b, pos); + sprintf(prefix_str, "%*s %d cat=%d size=%d max=%d", level*2, "", + pos, p->cat, p->size, b->file[p->cat].head.block_max); + (*pr)(prefix_str); + sprintf(prefix_str, "%*s %d", level*2, "", pos); + if (p->leaf) + { + while (p->offset < p->size) + { + char *src = p->bytes + p->offset; + char *dst = buf; + (*b->method->code_item)(ISAMC_DECODE, p->decodeClientData, + &dst, &src); + (*b->method->log_item)(LOG_DEBUG, buf, prefix_str); + p->offset = src - (char*) p->bytes; + } + assert(p->offset == p->size); + } + else + { + char *src = p->bytes + p->offset; + int sub; + int item_len; + + decode_ptr (&src, &sub); + p->offset = src - (char*) p->bytes; + + isamb_dump_r(b, sub, pr, level+1); + + while (p->offset < p->size) + { + decode_ptr (&src, &item_len); + (*b->method->log_item)(LOG_DEBUG, src, prefix_str); + src += item_len; + decode_ptr (&src, &sub); + + p->offset = src - (char*) p->bytes; + + isamb_dump_r(b, sub, pr, level+1); + } + } + close_block(b,p); + } +} + +void isamb_dump (ISAMB b, ISAMB_P pos, void (*pr)(const char *str)) +{ + return isamb_dump_r(b, pos, pr, 0); +} #if 0 /* Old isamb_pp_read that Adam wrote, kept as a reference in case we need to debug the more complex pp_read that also forwards. May be deleted near end of 2004, if it has not shown to be useful */ + int isamb_pp_read (ISAMB_PP pp, void *buf) { char *dst = buf; @@ -1079,6 +1141,369 @@ int isamb_pp_read (ISAMB_PP pp, void *buf) } #endif +#define NEW_FORWARD 1 + +#if NEW_FORWARD == 1 + +/* +#undef ISAMB_DEBUB +#define ISAMB_DEBUG 1 +*/ +static int isamb_pp_on_right_node(ISAMB_PP pp, int level, const void *untilbuf) +{ /* looks one node higher to see if we should be on this node at all */ + /* useful in backing off quickly, and in avoiding tail descends */ + /* call with pp->level to begin with */ + struct ISAMB_block *p; + int cmp; + char *src; + int item_len; + assert(level>=0); + if ( level == 0) { +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_on_right returning true for root"); +#endif + return 1; /* we can never skip the root node */ + } + level--; + p=pp->block[level]; + assert(p->offset <= p->size); + if (p->offset < p->size ) + { + assert(p->offset>0); + src=p->bytes + p->offset; + decode_ptr(&src,&item_len); +#if ISAMB_DEBUG + (*pp->isamb->method->log_item)(LOG_DEBUG,untilbuf,"on_leaf: until"); + (*pp->isamb->method->log_item)(LOG_DEBUG,src,"on_leaf: value"); +#endif + cmp=(*pp->isamb->method->compare_item)(untilbuf,src); + if (cmp<2) { +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_on_right returning true " + "cmp=%d lev=%d ofs=%d",cmp,level,p->offset); +#endif + return 1; + } + else { +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_on_right returning false " + "cmp=%d lev=%d ofs=%d",cmp,level,p->offset); +#endif + return 0; + } + } + else { +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_on_right at tail, looking higher " + "lev=%d",level); +#endif + return isamb_pp_on_right_node(pp, level, untilbuf); + } +} /* isamb_pp_on_right_node */ + +static int isamb_pp_read_on_leaf(ISAMB_PP pp, void *buf) +{ /* reads the next item on the current leaf, returns 0 if end of leaf*/ + struct ISAMB_block *p = pp->block[pp->level]; + char *dst; + char *src; + assert(pp); + assert(buf); + if (p->offset == p->size) { +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_read_on_leaf returning 0 on node %d",p->pos); +#endif + return 0; /* at end of leaf */ + } + src=p->bytes + p->offset; + dst=buf; + (*pp->isamb->method->code_item) + (ISAMC_DECODE, p->decodeClientData,&dst, &src); + p->offset = src - (char*) p->bytes; + /* +#if ISAMB_DEBUG + (*pp->isamb->method->log_item)(LOG_DEBUG, buf, "read_on_leaf returning 1"); +#endif +*/ + return 1; +} /* read_on_leaf */ + +static int isamb_pp_forward_on_leaf(ISAMB_PP pp, void *buf, const void *untilbuf) +{ /* forwards on the current leaf, returns 0 if not found */ + int cmp; + int skips=0; + while (1){ + if (!isamb_pp_read_on_leaf(pp,buf)) + return 0; + /* FIXME - this is an extra function call, inline the read? */ + cmp=(*pp->isamb->method->compare_item)(untilbuf,buf); + if (cmp <2){ /* found a good one */ +#if ISAMB_DEBUG + if (skips) + logf(LOG_DEBUG, "isam_pp_fwd_on_leaf skipped %d items",skips); +#endif + pp->returned_numbers++; + return 1; + } + if (!skips) + if (!isamb_pp_on_right_node(pp, pp->level, untilbuf)) + return 0; /* never mind the rest of this leaf */ + pp->skipped_numbers++; + skips++; + } +} /* forward_on_leaf */ + +static int isamb_pp_climb_level(ISAMB_PP pp, int *pos) +{ /* climbs higher in the tree, until finds a level with data left */ + /* returns the node to (consider to) descend to in *pos) */ + struct ISAMB_block *p = pp->block[pp->level]; + char *src; + int item_len; +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_climb_level starting " + "at level %d node %d ofs=%d sz=%d", + pp->level, p->pos, p->offset, p->size); +#endif + assert(pp->level >= 0); + assert(p->offset <= p->size); + if (pp->level==0) + { +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_climb_level returning 0 at root"); +#endif + return 0; + } + assert(pp->level>0); + close_block(pp->isamb, pp->block[pp->level]); + pp->block[pp->level]=0; + (pp->level)--; + p=pp->block[pp->level]; +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_climb_level climbed to level %d node %d ofs=%d", + pp->level, p->pos, p->offset); +#endif + assert(!p->leaf); + assert(p->offset <= p->size); + if (p->offset == p->size ) { + /* we came from the last pointer, climb on */ + if (!isamb_pp_climb_level(pp,pos)) + return 0; + p=pp->block[pp->level]; + } + else + { + /* skip the child we just came from */ +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isam_pp_climb_level: skipping lev=%d ofs=%d sz=%d", + pp->level, p->offset, p->size); +#endif + assert (p->offset < p->size ); + src=p->bytes + p->offset; + decode_ptr(&src, &item_len); + src += item_len; + decode_ptr(&src, pos); + p->offset=src - (char *)p->bytes; + + } + return 1; +} /* climb_level */ + + +static int isamb_pp_forward_unode(ISAMB_PP pp, int pos, const void *untilbuf) +{ /* scans a upper node until it finds a child <= untilbuf */ + /* pp points to the key value, as always. pos is the child read from */ + /* the buffer */ + /* if all values are too small, returns the last child in the node */ + /* FIXME - this can be detected, and avoided by looking at the */ + /* parent node, but that gets messy. Presumably the cost is */ + /* pretty low anyway */ + struct ISAMB_block *p = pp->block[pp->level]; + char *src=p->bytes + p->offset; + int item_len; + int cmp; + int nxtpos; +#if ISAMB_DEBUG + int skips=0; + logf(LOG_DEBUG,"isamb_pp_forward_unode starting " + "at level %d node %d ofs=%di sz=%d", + pp->level, p->pos, p->offset, p->size); +#endif + assert(!p->leaf); + assert(p->offset <= p->size); + if (p->offset == p->size) { +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_forward_unode returning at end " + "at level %d node %d ofs=%di sz=%d", + pp->level, p->pos, p->offset, p->size); +#endif + return pos; /* already at the end of it */ + } + while(p->offset < p->size) { + decode_ptr(&src,&item_len); + cmp=(*pp->isamb->method->compare_item)(untilbuf,src); + src+=item_len; + decode_ptr(&src,&nxtpos); + if (cmp<2) + { +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_forward_unode returning a hit " + "at level %d node %d ofs=%d sz=%d", + pp->level, p->pos, p->offset, p->size); +#endif + return pos; + } /* found one */ + pos=nxtpos; + p->offset=src-(char*)p->bytes; + (pp->skipped_nodes[pp->maxlevel - pp->level -1])++; +#if ISAMB_DEBUG + skips++; +#endif + } +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_forward_unode returning at tail " + "at level %d node %d ofs=%d sz=%d skips=%d", + pp->level, p->pos, p->offset, p->size, skips); +#endif + return pos; /* that's the last one in the line */ + +} /* forward_unode */ + +static void isamb_pp_descend_to_leaf(ISAMB_PP pp, int pos, const void *untilbuf) +{ /* climbs down the tree, from pos, to the leftmost leaf */ + struct ISAMB_block *p = pp->block[pp->level]; + char *src; + assert(!p->leaf); +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_descend_to_leaf " + "starting at lev %d node %d ofs=%d lf=%d u=%p", + pp->level, p->pos, p->offset, p->leaf, untilbuf); +#endif + if (untilbuf) + pos=isamb_pp_forward_unode(pp,pos,untilbuf); + ++(pp->level); + assert(pos); + p=open_block(pp->isamb, pos); + pp->block[pp->level]=p; + ++(pp->accessed_nodes[pp->maxlevel-pp->level]); + ++(pp->no_blocks); +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_descend_to_leaf " + "got lev %d node %d lf=%d", + pp->level, p->pos, p->leaf); +#endif + if (p->leaf) + return; + assert (p->offset==0 ); + src=p->bytes + p->offset; + decode_ptr(&src, &pos); + p->offset=src-(char*)p->bytes; + isamb_pp_descend_to_leaf(pp,pos,untilbuf); +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_descend_to_leaf " + "returning at lev %d node %d ofs=%d lf=%d", + pp->level, p->pos, p->offset, p->leaf); +#endif +} /* descend_to_leaf */ + +static int isamb_pp_find_next_leaf(ISAMB_PP pp) +{ /* finds the next leaf by climbing up and down */ + int pos; + if (!isamb_pp_climb_level(pp,&pos)) + return 0; + isamb_pp_descend_to_leaf(pp, pos,0); + return 1; +} + +static int isamb_pp_climb_desc(ISAMB_PP pp, void *buf, const void *untilbuf) +{ /* climbs up and descends to a leaf where values >= *untilbuf are found */ + int pos; +#if ISAMB_DEBUG + struct ISAMB_block *p = pp->block[pp->level]; + logf(LOG_DEBUG,"isamb_pp_climb_desc starting " + "at level %d node %d ofs=%d sz=%d", + pp->level, p->pos, p->offset, p->size); +#endif + if (!isamb_pp_climb_level(pp,&pos)) + return 0; + /* see if it would pay to climb one higher */ + if (!isamb_pp_on_right_node(pp, pp->level, untilbuf)) + if (!isamb_pp_climb_level(pp,&pos)) + return 0; + isamb_pp_descend_to_leaf(pp, pos,untilbuf); +#if ISAMB_DEBUG + p = pp->block[pp->level]; + logf(LOG_DEBUG,"isamb_pp_climb_desc done " + "at level %d node %d ofs=%d sz=%d", + pp->level, p->pos, p->offset, p->size); +#endif + return 1; +} /* climb_desc */ + +int isamb_pp_forward (ISAMB_PP pp, void *buf, const void *untilbuf) +{ +#if ISAMB_DEBUG + struct ISAMB_block *p = pp->block[pp->level]; + assert(p->leaf); + logf(LOG_DEBUG,"isamb_pp_forward starting " + "at level %d node %d ofs=%d sz=%d u=%p", + pp->level, p->pos, p->offset, p->size,untilbuf); +#endif + if (untilbuf) { + if (isamb_pp_forward_on_leaf( pp, buf, untilbuf)) { +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_forward (f) returning (A) " + "at level %d node %d ofs=%d sz=%d", + pp->level, p->pos, p->offset, p->size); +#endif + return 1; + } + if (! isamb_pp_climb_desc( pp, buf, untilbuf)) { +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_forward (f) returning notfound (B) " + "at level %d node %d ofs=%d sz=%d", + pp->level, p->pos, p->offset, p->size); +#endif + return 0; /* could not find a leaf */ + } + do{ + if (isamb_pp_forward_on_leaf( pp, buf, untilbuf)) { +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_forward (f) returning (C) " + "at level %d node %d ofs=%d sz=%d", + pp->level, p->pos, p->offset, p->size); +#endif + return 1; + } + }while ( isamb_pp_find_next_leaf(pp)); + return 0; /* could not find at all */ + } + else { /* no untilbuf, a straight read */ + /* FIXME - this should be moved + * directly into the pp_read */ + /* keeping here now, to keep same + * interface as the old fwd */ + if (isamb_pp_read_on_leaf( pp, buf)) { +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_forward (read) returning (D) " + "at level %d node %d ofs=%d sz=%d", + pp->level, p->pos, p->offset, p->size); +#endif + return 1; + } + if (isamb_pp_find_next_leaf(pp)) { +#if ISAMB_DEBUG + logf(LOG_DEBUG,"isamb_pp_forward (read) returning (E) " + "at level %d node %d ofs=%d sz=%d", + pp->level, p->pos, p->offset, p->size); +#endif + return isamb_pp_read_on_leaf(pp, buf); + } + else + return 0; + } +} /* isam_pp_forward (new version) */ + +#elif NEW_FORWARD == 0 + int isamb_pp_forward (ISAMB_PP pp, void *buf, const void *untilbuf) { /* pseudocode: @@ -1093,12 +1518,12 @@ int isamb_pp_forward (ISAMB_PP pp, void *buf, const void *untilbuf) * if cmp * return 1 */ - /* - * The upper nodes consist of a sequence of nodenumbers and keys - * When opening a block, the first node number is read in, and - * offset points to the first key, which is the upper limit of keys - * in the node just read. - */ + /* + * The upper nodes consist of a sequence of nodenumbers and keys + * When opening a block, the first node number is read in, and + * offset points to the first key, which is the upper limit of keys + * in the node just read. + */ char *dst = buf; char *src; struct ISAMB_block *p = pp->block[pp->level]; @@ -1141,7 +1566,7 @@ int isamb_pp_forward (ISAMB_PP pp, void *buf, const void *untilbuf) p->pos, p->offset); #endif assert(!p->leaf); - assert(p->offset <= p->size); + assert(p->offset <= p->size); /* skip the child we have handled */ if (p->offset != p->size) { @@ -1268,8 +1693,183 @@ int isamb_pp_forward (ISAMB_PP pp, void *buf, const void *untilbuf) } /* main loop */ } +#elif NEW_FORWARD == 2 + +int isamb_pp_forward (ISAMB_PP pp, void *buf, const void *untilb) +{ + char *dst = buf; + char *src; + struct ISAMB_block *p = pp->block[pp->level]; + if (!p) + return 0; + +again: + while (p->offset == p->size) + { + int pos, item_len; + while (p->offset == p->size) + { + if (pp->level == 0) + return 0; + close_block (pp->isamb, pp->block[pp->level]); + pp->block[pp->level] = 0; + (pp->level)--; + p = pp->block[pp->level]; + assert (!p->leaf); + } + + assert(!p->leaf); + src = p->bytes + p->offset; + + decode_ptr (&src, &item_len); + src += item_len; + decode_ptr (&src, &pos); + + p->offset = src - (char*) p->bytes; + + src = p->bytes + p->offset; + + while(1) + { + if (!untilb || p->offset == p->size) + break; + assert(p->offset < p->size); + decode_ptr (&src, &item_len); + if ((*pp->isamb->method->compare_item)(untilb, src) <= 1) + break; + src += item_len; + decode_ptr (&src, &pos); + p->offset = src - (char*) p->bytes; + } + + pp->level++; + + while (1) + { + pp->block[pp->level] = p = open_block (pp->isamb, pos); + + pp->total_size += p->size; + pp->no_blocks++; + + if (p->leaf) + { + break; + } + + src = p->bytes + p->offset; + while(1) + { + decode_ptr (&src, &pos); + p->offset = src - (char*) p->bytes; + + if (!untilb || p->offset == p->size) + break; + assert(p->offset < p->size); + decode_ptr (&src, &item_len); + if ((*pp->isamb->method->compare_item)(untilb, src) <= 1) + break; + src += item_len; + } + pp->level++; + } + } + assert (p->offset < p->size); + assert (p->leaf); + while(1) + { + char *dst0 = dst; + src = p->bytes + p->offset; + (*pp->isamb->method->code_item)(ISAMC_DECODE, p->decodeClientData, + &dst, &src); + p->offset = src - (char*) p->bytes; + if (!untilb || (*pp->isamb->method->compare_item)(untilb, dst0) <= 1) + break; + dst = dst0; + if (p->offset == p->size) goto again; + } + /* key_logdump_txt(LOG_DEBUG,buf, "isamb_pp_read returning 1"); */ + return 1; +} + +#endif int isamb_pp_num (ISAMB_PP pp) { return 1; } + +static void isamb_pp_leaf_pos( ISAMB_PP pp, + int *current, int *total, void *dummybuf ) +{ + struct ISAMB_block *p = pp->block[pp->level]; + char *src=p->bytes; + char *end=p->bytes+p->size; + char *cur=p->bytes+p->offset; + char *dst; + assert(p->offset <= p->size); + assert(cur <= end); + assert(p->leaf); + *current=0; + *total=0; + + while(src < end) { + dst=dummybuf; + (*pp->isamb->method->code_item) + (ISAMC_DECODE, p->decodeClientData,&dst, &src); + assert(dstoffset, p->size, pp->level); + assert(src==end); +} + +static void isamb_pp_upper_pos( ISAMB_PP pp, int *current, int *total, + int size, int level ) +{ /* estimates total/current occurrences from here up, excl leaf */ + struct ISAMB_block *p = pp->block[level]; + char *src=p->bytes; + char *end=p->bytes+p->size; + char *cur=p->bytes+p->offset; + int item_size; + int child; + assert(level>=0); + assert(!p->leaf); + logf(LOG_DEBUG,"isamb_pp_upper_pos at beginning l=%d " + "cur=%d tot=%d ofs=%d sz=%d pos=%d", + level, *current, *total, p->offset, p->size, p->pos); + assert (p->offset <= p->size); + decode_ptr (&src, &child ); /* first child */ + while(src < end) { + if (src!=cur) { + *total += size; + if (src < cur) + *current +=size; + } + decode_ptr (&src, &item_size ); + assert(src+item_size<=end); + src += item_size; + decode_ptr (&src, &child ); + } + if (level>0) + isamb_pp_upper_pos(pp, current, total, *total, level-1); +} /* upper_pos */ + +void isamb_pp_pos( ISAMB_PP pp, int *current, int *total ) +{ /* return an estimate of the current position and of the total number of */ + /* occureences in the isam tree, based on the current leaf */ + struct ISAMB_block *p = pp->block[pp->level]; + char dummy[100]; /* 100 bytes/entry must be enough */ + assert(total); + assert(current); + assert(p->leaf); + isamb_pp_leaf_pos(pp,current, total, dummy); + if (pp->level>0) + isamb_pp_upper_pos(pp, current, total, *total, pp->level-1); + /* + logf(LOG_DEBUG,"isamb_pp_pos: C=%d T=%d =%6.2f%%", + *current, *total, 100.0*(*current)/(*total)); + */ +}