X-Git-Url: http://git.indexdata.com/?p=idzebra-moved-to-github.git;a=blobdiff_plain;f=index%2Fzebraapi.c;h=46a0e51caa62921d42270db8a145ac6f0d4eaeeb;hp=885c684ba32ab201f9f8241fd4a505f8f7a2874f;hb=bf39de14630d0b8a861cfb211e1e218a05cf68ab;hpb=45a75e3bd29f9cd44bb7880544ad3864f5c6e87d diff --git a/index/zebraapi.c b/index/zebraapi.c index 885c684..46a0e51 100644 --- a/index/zebraapi.c +++ b/index/zebraapi.c @@ -2195,9 +2195,15 @@ ZEBRA_RES zebra_compact(ZebraHandle zh) } static ZEBRA_RES zebra_record_check(ZebraHandle zh, Record rec, - zint *no_keys) + zint *no_keys, int verbose_level, + zint *no_long_dict_entries, + zint *no_failed_dict_lookups, + zint *no_invalid_keys, + zint *no_invalid_dict_infos, + zint *no_invalid_isam_entries + ) { - ZEBRA_RES res = ZEBRA_FAIL; + ZEBRA_RES res = ZEBRA_OK; zebra_rec_keys_t keys = zebra_rec_keys_open(); zebra_rec_keys_set_buf(keys, rec->info[recInfo_delKeys], rec->size[recInfo_delKeys], 0); @@ -2212,75 +2218,240 @@ static ZEBRA_RES zebra_record_check(ZebraHandle zh, Record rec, size_t slen; const char *str; struct it_key key_in; - int no_long_dict_entries = 0; - int no_failed_dict_lookup = 0; + NMEM nmem = nmem_create(); while (zebra_rec_keys_read(keys, &str, &slen, &key_in)) { + int do_fail = 0; int ord = CAST_ZINT_TO_INT(key_in.mem[0]); - char ord_buf[IT_MAX_WORD]; + char ord_buf[IT_MAX_WORD+20]; int ord_len = key_SU_encode(ord, ord_buf); - + char *info = 0; + + (*no_keys)++; + + if (ord_len + slen >= sizeof(ord_buf)-1) + { + (*no_long_dict_entries)++; + res = ZEBRA_FAIL; + if (verbose_level >= 1) + { + /* so bad it can not fit into our ord_buf */ + yaz_log(YLOG_WARN, "Record " ZINT_FORMAT + ": long dictionary entry %d + %d", + rec->sysno, ord_len, (int) slen); + } + continue; + } memcpy(ord_buf + ord_len, str, slen); ord_buf[ord_len + slen] = '\0'; if (ord_len + slen >= IT_MAX_WORD) - ++no_long_dict_entries; + { + do_fail = 1; + (*no_long_dict_entries)++; + if (verbose_level >= 1) + { + yaz_log(YLOG_WARN, "Record " ZINT_FORMAT + ": long dictionary entry %d + %d", + rec->sysno, (int) ord_len, (int) slen); + } + } + info = dict_lookup(zh->reg->dict, ord_buf); + if (!info) + { + do_fail = 1; + (*no_failed_dict_lookups)++; + if (verbose_level >= 1) + { + yaz_log(YLOG_WARN, "Record " ZINT_FORMAT + ": term do not exist in dictionary", rec->sysno); + } + } else { - char *info = dict_lookup(zh->reg->dict, ord_buf); - if (!info) - no_failed_dict_lookup++; + ISAM_P pos; + + if (*info != sizeof(pos)) + { + do_fail = 1; + (*no_invalid_dict_infos)++; + if (verbose_level >= 1) + { + yaz_log(YLOG_WARN, "Record " ZINT_FORMAT + ": long dictionary entry %d + %d", + rec->sysno, (int) ord_len, (int) slen); + } + } else { - ; + int scope = 1; + memcpy(&pos, info+1, sizeof(pos)); + if (zh->reg->isamb) + { + ISAMB_PP ispt = isamb_pp_open(zh->reg->isamb, pos, + scope); + if (!ispt) + { + do_fail = 1; + (*no_invalid_isam_entries)++; + if (verbose_level >= 1) + { + yaz_log(YLOG_WARN, "Record " ZINT_FORMAT + ": isamb_pp_open entry " ZINT_FORMAT + " not found", + rec->sysno, pos); + } + } + else if (zh->m_staticrank) + { + isamb_pp_close(ispt); + } + else + { + struct it_key until_key; + struct it_key isam_key; + int r; + int i = 0; + + until_key.len = key_in.len - 1; + for (i = 0; i < until_key.len; i++) + until_key.mem[i] = key_in.mem[i+1]; + + if (until_key.mem[0] == 0) + until_key.mem[0] = rec->sysno; + r = isamb_pp_forward(ispt, &isam_key, &until_key); + if (r != 1) + { + do_fail = 1; + (*no_invalid_isam_entries)++; + if (verbose_level >= 1) + { + yaz_log(YLOG_WARN, "Record " ZINT_FORMAT + ": isamb_pp_forward " ZINT_FORMAT + " returned no entry", + rec->sysno, pos); + } + } + else + { + int cmp = key_compare(&until_key, &isam_key); + if (cmp != 0) + { + do_fail = 1; + (*no_invalid_isam_entries)++; + if (verbose_level >= 1) + { + yaz_log(YLOG_WARN, "Record " + ZINT_FORMAT + ": isamb_pp_forward " + ZINT_FORMAT + " returned different entry", + rec->sysno, pos); + + key_logdump_txt(YLOG_LOG, + &until_key, + "until"); + + key_logdump_txt(YLOG_LOG, + &isam_key, + "isam"); + + } + } + } + isamb_pp_close(ispt); + } + + } + } + } + if (key_in.len < 2 || key_in.len > 4) + { + do_fail = 1; + (*no_invalid_keys)++; + if (verbose_level >= 1) + { + yaz_log(YLOG_WARN, "Record " ZINT_FORMAT + ": unexpected key length %d", + rec->sysno, key_in.len); + } + } + if (do_fail) + { + res = ZEBRA_FAIL; + if (verbose_level >= 1) + { + zebra_it_key_str_dump(zh, &key_in, str, + slen, nmem, YLOG_LOG); + nmem_reset(nmem); } } - (*no_keys)++; } - if (no_long_dict_entries) - { - yaz_log(YLOG_WARN, "Record id " ZINT_FORMAT - " has %d dictionary entries that are too long", - rec->sysno, no_long_dict_entries); - } - if (no_failed_dict_lookup) - { - yaz_log(YLOG_WARN, "Record id " ZINT_FORMAT - " has %d terms that do not exist in dictionary", - rec->sysno, no_failed_dict_lookup); - } - res = ZEBRA_OK; + nmem_destroy(nmem); } zebra_rec_keys_close(keys); return res; } -ZEBRA_RES zebra_register_check(ZebraHandle zh) +ZEBRA_RES zebra_register_check(ZebraHandle zh, int verbose_level) { ZEBRA_RES res = ZEBRA_FAIL; if (zebra_begin_read(zh) == ZEBRA_OK) { - zint no_records = 0; + zint no_records_total = 0; + zint no_records_fail = 0; zint total_keys = 0; + + if (zh->reg) { Record rec = rec_get_root(zh->reg->records); + zint no_long_dict_entries = 0; + zint no_failed_dict_lookups = 0; + zint no_invalid_keys = 0; + zint no_invalid_dict_infos = 0; + zint no_invalid_isam_entries = 0; + + res = ZEBRA_OK; while (rec) { Record r1; zint no_keys; - - zebra_record_check(zh, rec, &no_keys); + + if (zebra_record_check(zh, rec, &no_keys, verbose_level, + &no_long_dict_entries, + &no_failed_dict_lookups, + &no_invalid_keys, + &no_invalid_dict_infos, + &no_invalid_isam_entries + ) + != ZEBRA_OK) + { + res = ZEBRA_FAIL; + no_records_fail++; + } + r1 = rec_get_next(zh->reg->records, rec); rec_free(&rec); rec = r1; - no_records++; + no_records_total++; total_keys += no_keys; } - res = ZEBRA_OK; - yaz_log(YLOG_LOG, "records: " ZINT_FORMAT, no_records); - yaz_log(YLOG_LOG, "keys: " ZINT_FORMAT, total_keys); + yaz_log(YLOG_LOG, "records total: " ZINT_FORMAT, + no_records_total); + yaz_log(YLOG_LOG, "records fail: " ZINT_FORMAT, + no_records_fail); + yaz_log(YLOG_LOG, "total keys: " ZINT_FORMAT, + total_keys); + yaz_log(YLOG_LOG, "long dict entries: " ZINT_FORMAT, + no_long_dict_entries); + yaz_log(YLOG_LOG, "failed dict lookups: " ZINT_FORMAT, + no_failed_dict_lookups); + yaz_log(YLOG_LOG, "invalid dict infos: " ZINT_FORMAT, + no_invalid_dict_infos); + yaz_log(YLOG_LOG, "invalid isam entries: " ZINT_FORMAT, + no_invalid_isam_entries); } zebra_end_read(zh); }