From f026071ff0fbd2df52851b204b0dbcee8b45509d Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Tue, 30 Apr 2002 19:31:09 +0000 Subject: [PATCH] isamb delete; more statistics --- index/invstat.c | 31 ++++++++++++++++++++++++++++--- isamb/isamb.c | 50 ++++++++++++++++++++++++++++++++------------------ 2 files changed, 60 insertions(+), 21 deletions(-) diff --git a/index/invstat.c b/index/invstat.c index ca44bda..8e5770a 100644 --- a/index/invstat.c +++ b/index/invstat.c @@ -23,6 +23,7 @@ struct inv_stat_info { int isamb_levels[10][5]; int isamb_sizes[10]; int isamb_blocks[10]; + unsigned long cksum; }; #define SINGLETON_TYPE 8 /* the type to use for singletons that */ @@ -52,7 +53,8 @@ static int inv_stat_handle (char *name, const char *info, int pos, occur = isams_pp_num (pp); while (isams_pp_read(pp, &key)) { - //printf ("sysno=%d seqno=%d\n", key.sysno, key.seqno); + stat_info->cksum = stat_info->cksum * 65509 + + key.sysno + 11 * key.seqno; occurx++; } assert (occurx == occur); @@ -65,6 +67,7 @@ static int inv_stat_handle (char *name, const char *info, int pos, ispt = is_position (stat_info->zh->reg->isam, isam_p); occur = is_numkeys (ispt); + stat_info->no_isam_entries[is_type(isam_p)] += occur; is_pt_free (ispt); } if (stat_info->zh->reg->isamc) @@ -77,7 +80,8 @@ static int inv_stat_handle (char *name, const char *info, int pos, occur = isc_pp_num (pp); while (isc_pp_read(pp, &key)) { - //printf ("sysno=%d seqno=%d\n", key.sysno, key.seqno); + stat_info->cksum = stat_info->cksum * 65509 + + key.sysno + 11 * key.seqno; occurx++; } assert (occurx == occur); @@ -95,6 +99,8 @@ static int inv_stat_handle (char *name, const char *info, int pos, occur = isamd_pp_num (pp); while (isamd_pp_read(pp, &key)) { + stat_info->cksum = stat_info->cksum * 65509 + + key.sysno + 11 * key.seqno; occurx++; if ( pp->is->method->debug >8 ) logf (LOG_LOG,"sysno=%d seqno=%d (%x/%x) oc=%d/%d ofs=%d ", @@ -127,13 +133,18 @@ static int inv_stat_handle (char *name, const char *info, int pos, pp = isamb_pp_open_x(stat_info->zh->reg->isamb, isam_p, &level); while (isamb_pp_read(pp, &key)) + { + stat_info->cksum = stat_info->cksum * 65509 + + key.sysno + 11 * key.seqno; occur++; + } isamb_pp_close_x (pp, &size, &blocks); stat_info->isamb_blocks[cat] += blocks; stat_info->isamb_sizes[cat] += size; if (level > 4) level = 4; stat_info->isamb_levels[cat][level] ++; + stat_info->no_isam_entries[cat] += occur; } while (occur > stat_info->isam_bounds[i] && stat_info->isam_bounds[i]) @@ -149,6 +160,7 @@ void zebra_register_statistics (ZebraHandle zh) int count; int i, prev; int before = 0; + int occur; int after = 1000000000; struct inv_stat_info stat_info; char term_dict[2*IT_MAX_WORD+2]; @@ -185,6 +197,8 @@ void zebra_register_statistics (ZebraHandle zh) stat_info.isam_bounds[17] = 1000000; stat_info.isam_bounds[18] = 0; + stat_info.cksum = 0; + for (i = 0; i<20; i++) stat_info.isam_occurrences[i] = 0; @@ -265,15 +279,23 @@ void zebra_register_statistics (ZebraHandle zh) fprintf (stderr, "Block size %d\n", bsize); fprintf (stderr, "Blocks: %d\n", stat_info.isamb_blocks[i]); fprintf (stderr, "Size: %d\n", stat_info.isamb_sizes[i]); + fprintf (stderr, "Entries: %d\n", stat_info.no_isam_entries[i]); fprintf (stderr, "Total %d\n", stat_info.isamb_blocks[i]* bsize); for (j = 0; j<5; j++) if (stat_info.isamb_levels[i][j]) fprintf (stderr, "Level%d %d\n", j, stat_info.isamb_levels[i][j]); + fprintf (stderr, "\n"); } } + fprintf (stderr, "Checksum %08lX\n", stat_info.cksum); + fprintf (stderr, "Distinct words %d\n", stat_info.no_dict_entries); + occur = 0; + for (i = 0; i<9; i++) + occur += stat_info.no_isam_entries[i]; + fprintf (stderr, "Word pos %d\n", occur); fprintf (stderr, " Occurrences Words\n"); prev = 1; for (i = 0; stat_info.isam_bounds[i]; i++) @@ -293,7 +315,10 @@ void zebra_register_statistics (ZebraHandle zh) /* * * $Log: invstat.c,v $ - * Revision 1.27 2002-04-30 08:28:37 adam + * Revision 1.28 2002-04-30 19:31:09 adam + * isamb delete; more statistics + * + * Revision 1.27 2002/04/30 08:28:37 adam * isamb fixes for pp_read. Statistics * * Revision 1.26 2002/04/29 18:03:46 adam diff --git a/isamb/isamb.c b/isamb/isamb.c index 1625774..3437fc1 100644 --- a/isamb/isamb.c +++ b/isamb/isamb.c @@ -2,7 +2,7 @@ * Copyright (c) 2000-2002, Index Data. * See the file LICENSE for details. * - * $Id: isamb.c,v 1.12 2002-04-30 08:28:37 adam Exp $ + * $Id: isamb.c,v 1.13 2002-04-30 19:31:09 adam Exp $ */ #include #include @@ -91,7 +91,7 @@ ISAMB isamb_open (BFiles bfs, const char *name, int writeflag, ISAMC_M method) isamb->file[i].head.block_max = b_size - ISAMB_DATA_OFFSET; b_size = b_size * 4; isamb->file[i].head_dirty = 0; - sprintf (fname, "%s-%d", name, i); + sprintf (fname, "%s%c", name, i+'A'); isamb->file[i].bf = bf_open (bfs, fname, isamb->file[i].head.block_size, writeflag); @@ -186,13 +186,14 @@ void close_block (ISAMB b, struct ISAMB_block *p) } int insert_sub (ISAMB b, struct ISAMB_block **p, - void *new_item, + void *new_item, int *mode, ISAMC_I stream, struct ISAMB_block **sp, void *sub_item, int *sub_size, void *max_item); int insert_int (ISAMB b, struct ISAMB_block *p, void *lookahead_item, + int *mode, ISAMC_I stream, struct ISAMB_block **sp, void *split_item, int *split_size) { @@ -218,7 +219,8 @@ int insert_int (ISAMB b, struct ISAMB_block *p, void *lookahead_item, { sub_p1 = open_block (b, pos); assert (sub_p1); - more = insert_sub (b, &sub_p1, lookahead_item, stream, &sub_p2, + more = insert_sub (b, &sub_p1, lookahead_item, mode, + stream, &sub_p2, sub_item, &sub_size, src); break; } @@ -229,7 +231,7 @@ int insert_int (ISAMB b, struct ISAMB_block *p, void *lookahead_item, { sub_p1 = open_block (b, pos); assert (sub_p1); - more = insert_sub (b, &sub_p1, lookahead_item, stream, &sub_p2, + more = insert_sub (b, &sub_p1, lookahead_item, mode, stream, &sub_p2, sub_item, &sub_size, 0); } if (sub_p2) @@ -297,7 +299,7 @@ int insert_int (ISAMB b, struct ISAMB_block *p, void *lookahead_item, int insert_leaf (ISAMB b, struct ISAMB_block **sp1, void *lookahead_item, - ISAMC_I stream, struct ISAMB_block **sp2, + int *lookahead_mode, ISAMC_I stream, struct ISAMB_block **sp2, void *sub_item, int *sub_size, void *max_item) { @@ -329,17 +331,23 @@ int insert_leaf (ISAMB b, struct ISAMB_block **sp1, void *lookahead_item, char *dst_item = 0; char *dst_0 = dst; char *lookahead_next; - int lookahead_mode; int d = -1; if (lookahead_item) d = (*b->method->compare_item)(file_item_buf, lookahead_item); - if (d > 0) + if (d > 0) + { dst_item = lookahead_item; + assert (*lookahead_mode); + } else dst_item = file_item_buf; - if (!half1 && dst > cut) + if (!*lookahead_mode && d == 0) + { + p->dirty = 1; + } + else if (!half1 && dst > cut) { char *dst_item_0 = dst_item; half1 = dst; /* candidate for splitting */ @@ -365,7 +373,7 @@ int insert_leaf (ISAMB b, struct ISAMB_block **sp1, void *lookahead_item, lookahead_next = lookahead_item; if (!(*stream->read_item)(stream->clientData, &lookahead_next, - &lookahead_mode)) + lookahead_mode)) { lookahead_item = 0; more = 0; @@ -384,7 +392,7 @@ int insert_leaf (ISAMB b, struct ISAMB_block **sp1, void *lookahead_item, { lookahead_next = lookahead_item; if (!(*stream->read_item)(stream->clientData, - &lookahead_next, &lookahead_mode)) + &lookahead_next, lookahead_mode)) { lookahead_item = 0; more = 0; @@ -406,7 +414,6 @@ int insert_leaf (ISAMB b, struct ISAMB_block **sp1, void *lookahead_item, maxp = dst_buf + b->file[b->no_cat-1].head.block_max + quater; while (lookahead_item) { - int lookahead_mode; char *dst_item = lookahead_item; char *dst_0 = dst; @@ -416,7 +423,12 @@ int insert_leaf (ISAMB b, struct ISAMB_block **sp1, void *lookahead_item, yaz_log (LOG_LOG, "max_item 2"); break; } - if (!half1 && dst > cut) + if (!*lookahead_mode) + { + yaz_log (LOG_WARN, "Inconsistent register (2)"); + abort(); + } + else if (!half1 && dst > cut) { char *dst_item_0 = dst_item; half1 = dst; /* candidate for splitting */ @@ -440,7 +452,7 @@ int insert_leaf (ISAMB b, struct ISAMB_block **sp1, void *lookahead_item, p->dirty = 1; dst_item = lookahead_item; if (!(*stream->read_item)(stream->clientData, &dst_item, - &lookahead_mode)) + lookahead_mode)) { lookahead_item = 0; more = 0; @@ -506,16 +518,18 @@ int insert_leaf (ISAMB b, struct ISAMB_block **sp1, void *lookahead_item, } int insert_sub (ISAMB b, struct ISAMB_block **p, void *new_item, + int *mode, ISAMC_I stream, struct ISAMB_block **sp, void *sub_item, int *sub_size, void *max_item) { if (!*p || (*p)->leaf) - return insert_leaf (b, p, new_item, stream, sp, sub_item, sub_size, - max_item); + return insert_leaf (b, p, new_item, mode, stream, sp, sub_item, + sub_size, max_item); else - return insert_int (b, *p, new_item, stream, sp, sub_item, sub_size); + return insert_int (b, *p, new_item, mode, stream, sp, sub_item, + sub_size); } int isamb_merge (ISAMB b, ISAMC_P pos, ISAMC_I stream) @@ -535,7 +549,7 @@ int isamb_merge (ISAMB b, ISAMC_P pos, ISAMC_I stream) if (pos) p = open_block (b, pos); - more = insert_sub (b, &p, item_buf, stream, &sp, + more = insert_sub (b, &p, item_buf, &i_mode, stream, &sp, sub_item, &sub_size, 0); if (sp) { /* increase level of tree by one */ -- 1.7.10.4