X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fextract.c;h=86f868b3a9e7c355a1b6c92e29ed1bf4ee6448d4;hb=f7a3769dede0071696bdcc13ae2ee1efe6d52d96;hp=10ff2bfd21c07fa568311b1d92ac136068cacadf;hpb=3168ab325caf86d3bac38c33f7d3b701d1110539;p=idzebra-moved-to-github.git diff --git a/index/extract.c b/index/extract.c index 10ff2bf..86f868b 100644 --- a/index/extract.c +++ b/index/extract.c @@ -1,5 +1,5 @@ -/* $Id: extract.c,v 1.208 2006-05-09 11:31:37 marc Exp $ - Copyright (C) 1995-2005 +/* $Id: extract.c,v 1.214 2006-05-18 12:03:05 adam Exp $ + Copyright (C) 1995-2006 Index Data ApS This file is part of the Zebra server. @@ -36,12 +36,19 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA #include #include -#if _FILE_OFFSET_BITS == 64 -#define PRINTF_OFF_T "%Ld" +#ifdef WIN32 +#define PRINTF_OFF_T "%I64d" +#else +/* !WIN32 */ +#if SIZEOF_OFF_T == SIZEOF_LONG_LONG +#define PRINTF_OFF_T "%lld" #else #define PRINTF_OFF_T "%ld" #endif +#endif + + #define USE_SHELLSORT 0 #if USE_SHELLSORT @@ -81,6 +88,8 @@ static void logRecord (ZebraHandle zh) } } +static void extract_add_index_string (RecWord *p, const char *str, int length); + static void extract_set_store_data_prepare(struct recExtractCtrl *p); static void extract_init (struct recExtractCtrl *p, RecWord *w) @@ -379,6 +388,16 @@ static void init_extractCtrl(ZebraHandle zh, struct recExtractCtrl *ctrl) ctrl->flagShowRecords = !zh->m_flag_rw; } +static void all_matches_add(struct recExtractCtrl *ctrl) +{ + RecWord word; + extract_init(ctrl, &word); + word.index_name = "allrecords"; + word.index_type = 'w'; + word.seqno = 1; + extract_add_index_string (&word, "", 0); +} + static ZEBRA_RES file_extract_record(ZebraHandle zh, SYSNO *sysno, const char *fname, int deleteFlag, @@ -387,6 +406,7 @@ static ZEBRA_RES file_extract_record(ZebraHandle zh, RecType recType, void *recTypeClientData) { + const char *match_str_to_print = ""; RecordAttr *recordAttr; int r; const char *matchStr = 0; @@ -408,11 +428,7 @@ static ZEBRA_RES file_extract_record(ZebraHandle zh, /* we are going to read from a file, so prepare the extraction */ zebra_rec_keys_reset(zh->reg->keys); -#if NATTR zebra_rec_keys_reset(zh->reg->sortKeys); -#else - zh->reg->sortKeys.buf_used = 0; -#endif recordOffset = fi->file_moffset; extractCtrl.handle = zh; extractCtrl.offset = fi->file_moffset; @@ -471,11 +487,18 @@ static ZEBRA_RES file_extract_record(ZebraHandle zh, } return ZEBRA_FAIL; } + all_matches_add(&extractCtrl); if (extractCtrl.match_criteria[0]) - matchStr = extractCtrl.match_criteria; + matchStr = extractCtrl.match_criteria; } - /* perform match if sysno not known and if match criteria is specified */ + /* if matchStr is set now - we assume it's printable . + For internal matchStr (see below) we don't print */ + if (matchStr) + match_str_to_print = matchStr; + + /* perform internal match if sysno not known and if match criteria is + specified already */ if (!sysno) { sysnotmp = 0; @@ -488,6 +511,11 @@ static ZEBRA_RES file_extract_record(ZebraHandle zh, if (!matchStr) { yaz_log(YLOG_WARN, "Bad match criteria"); + + if (zebra_rec_keys_empty(zh->reg->keys)) + { + yaz_log(YLOG_WARN, "And no index keys"); + } return ZEBRA_FAIL; } } @@ -532,17 +560,12 @@ static ZEBRA_RES file_extract_record(ZebraHandle zh, *sysno = rec->sysno; if (zh->records_processed < zh->m_file_verbose_limit) - if (matchStr) + { yaz_log(YLOG_LOG, "add %s %s " PRINTF_OFF_T " " ZINT_FORMAT " %s" , zh->m_record_type, - fname, recordOffset, *sysno, matchStr); - else - yaz_log(YLOG_LOG, "add %s %s " PRINTF_OFF_T - " " ZINT_FORMAT , - zh->m_record_type, - fname, recordOffset, *sysno); - + fname, recordOffset, *sysno, match_str_to_print); + } recordAttr = rec_init_attr (zh->reg->zei, rec); recordAttr->staticrank = extractCtrl.staticrank; @@ -554,11 +577,7 @@ static ZEBRA_RES file_extract_record(ZebraHandle zh, } -#if NATTR extract_flushSortKeys (zh, *sysno, 1, zh->reg->sortKeys); -#else - extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); -#endif extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys, recordAttr->staticrank); zh->records_inserted++; @@ -568,11 +587,7 @@ static ZEBRA_RES file_extract_record(ZebraHandle zh, /* record already exists */ zebra_rec_keys_t delkeys = zebra_rec_keys_open(); -#if NATTR zebra_rec_keys_t sortKeys = zebra_rec_keys_open(); -#else - struct sortKeys sortKeys; -#endif rec = rec_get (zh->reg->records, *sysno); assert (rec); @@ -584,18 +599,11 @@ static ZEBRA_RES file_extract_record(ZebraHandle zh, rec->size[recInfo_delKeys], 0); -#if NATTR zebra_rec_keys_set_buf(sortKeys, rec->info[recInfo_sortKeys], rec->size[recInfo_sortKeys], 0); extract_flushSortKeys (zh, *sysno, 0, sortKeys); -#else - sortKeys.buf_used = rec->size[recInfo_sortKeys]; - sortKeys.buf = rec->info[recInfo_sortKeys]; - extract_flushSortKeys (zh, *sysno, 0, &sortKeys); -#endif - extract_flushRecordKeys (zh, *sysno, 0, delkeys, recordAttr->staticrank); /* old values */ if (deleteFlag) @@ -611,19 +619,12 @@ static ZEBRA_RES file_extract_record(ZebraHandle zh, else { if (zh->records_processed < zh->m_file_verbose_limit) - if (matchStr) + { yaz_log(YLOG_LOG, "delete %s %s " PRINTF_OFF_T " " ZINT_FORMAT " %s" , zh->m_record_type, - fname, recordOffset, *sysno, matchStr); - else - yaz_log(YLOG_LOG, "delete %s %s " PRINTF_OFF_T - " " ZINT_FORMAT , - zh->m_record_type, - fname, recordOffset, *sysno); - - - + fname, recordOffset, *sysno, match_str_to_print); + } zh->records_deleted++; if (matchStr) { @@ -640,31 +641,20 @@ static ZEBRA_RES file_extract_record(ZebraHandle zh, { /* flush new keys for sort&search etc */ if (zh->records_processed < zh->m_file_verbose_limit) - if (matchStr) - yaz_log(YLOG_LOG, "update %s %s " PRINTF_OFF_T - " " ZINT_FORMAT " %s" , - zh->m_record_type, - fname, recordOffset, *sysno, matchStr); - else - yaz_log(YLOG_LOG, "update %s %s " PRINTF_OFF_T - " " ZINT_FORMAT , - zh->m_record_type, - fname, recordOffset, *sysno); - + { + yaz_log(YLOG_LOG, "update %s %s " PRINTF_OFF_T + " " ZINT_FORMAT " %s" , + zh->m_record_type, + fname, recordOffset, *sysno, match_str_to_print); + } recordAttr->staticrank = extractCtrl.staticrank; -#if NATTR extract_flushSortKeys (zh, *sysno, 1, zh->reg->sortKeys); -#else - extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); -#endif extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys, recordAttr->staticrank); zh->records_updated++; } zebra_rec_keys_close(delkeys); -#if NATTR zebra_rec_keys_close(sortKeys); -#endif } /* update file type */ xfree (rec->info[recInfo_fileType]); @@ -693,16 +683,9 @@ static ZEBRA_RES file_extract_record(ZebraHandle zh, /* update sort keys */ xfree (rec->info[recInfo_sortKeys]); -#if NATTR zebra_rec_keys_get_buf(zh->reg->sortKeys, &rec->info[recInfo_sortKeys], &rec->size[recInfo_sortKeys]); -#else - rec->size[recInfo_sortKeys] = zh->reg->sortKeys.buf_used; - rec->info[recInfo_sortKeys] = zh->reg->sortKeys.buf; - zh->reg->sortKeys.buf = NULL; - zh->reg->sortKeys.buf_max = 0; -#endif /* save file size of original record */ zebraExplain_recordBytesIncrement (zh->reg->zei, @@ -927,12 +910,8 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh, extractCtrl.fh = &fc; zebra_rec_keys_reset(zh->reg->keys); - -#if NATTR zebra_rec_keys_reset(zh->reg->sortKeys); -#else - zh->reg->sortKeys.buf_used = 0; -#endif + if (zebraExplain_curDatabase (zh->reg->zei, zh->basenames[0])) { if (zebraExplain_newDatabase (zh->reg->zei, zh->basenames[0], @@ -994,6 +973,8 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh, return ZEBRA_FAIL; } + all_matches_add(&extractCtrl); + if (extractCtrl.match_criteria[0]) match_criteria = extractCtrl.match_criteria; @@ -1062,15 +1043,7 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh, } -#if NATTR extract_flushSortKeys (zh, *sysno, 1, zh->reg->sortKeys); -#else - extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); -#endif - -#if 0 - print_rec_keys(zh, zh->reg->keys); -#endif extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys, recordAttr->staticrank); zh->records_inserted++; @@ -1079,12 +1052,7 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh, { /* record already exists */ zebra_rec_keys_t delkeys = zebra_rec_keys_open(); -#if NATTR zebra_rec_keys_t sortKeys = zebra_rec_keys_open(); -#else - struct sortKeys sortKeys; -#endif - if (!allow_update) { yaz_log (YLOG_LOG, "skipped %s %s %ld", @@ -1102,21 +1070,12 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh, rec->info[recInfo_delKeys], rec->size[recInfo_delKeys], 0); -#if NATTR zebra_rec_keys_set_buf(sortKeys, rec->info[recInfo_sortKeys], rec->size[recInfo_sortKeys], 0); -#else - sortKeys.buf_used = rec->size[recInfo_sortKeys]; - sortKeys.buf = rec->info[recInfo_sortKeys]; -#endif -#if NATTR extract_flushSortKeys (zh, *sysno, 0, sortKeys); -#else - extract_flushSortKeys (zh, *sysno, 0, &sortKeys); -#endif extract_flushRecordKeys (zh, *sysno, 0, delkeys, recordAttr->staticrank); if (delete_flag) @@ -1152,19 +1111,13 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh, yaz_log (YLOG_LOG, "update %s %s %ld", recordType, pr_fname, (long) recordOffset); recordAttr->staticrank = extractCtrl.staticrank; -#if NATTR extract_flushSortKeys (zh, *sysno, 1, zh->reg->sortKeys); -#else - extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); -#endif extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys, recordAttr->staticrank); zh->records_updated++; } zebra_rec_keys_close(delkeys); -#if NATTR zebra_rec_keys_close(sortKeys); -#endif } /* update file type */ xfree (rec->info[recInfo_fileType]); @@ -1192,16 +1145,9 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh, /* update sort keys */ xfree (rec->info[recInfo_sortKeys]); -#if NATTR zebra_rec_keys_get_buf(zh->reg->sortKeys, &rec->info[recInfo_sortKeys], &rec->size[recInfo_sortKeys]); -#else - rec->size[recInfo_sortKeys] = zh->reg->sortKeys.buf_used; - rec->info[recInfo_sortKeys] = zh->reg->sortKeys.buf; - zh->reg->sortKeys.buf = NULL; - zh->reg->sortKeys.buf_max = 0; -#endif /* save file size of original record */ zebraExplain_recordBytesIncrement (zh->reg->zei, @@ -1271,12 +1217,8 @@ int explain_extract (void *handle, Record rec, data1_node *n) } zebra_rec_keys_reset(zh->reg->keys); - -#if NATTR zebra_rec_keys_reset(zh->reg->sortKeys); -#else - zh->reg->sortKeys.buf_used = 0; -#endif + extractCtrl.init = extract_init; extractCtrl.tokenAdd = extract_token_add; extractCtrl.schemaAdd = extract_schema_add; @@ -1299,36 +1241,23 @@ int explain_extract (void *handle, Record rec, data1_node *n) { zebra_rec_keys_t delkeys = zebra_rec_keys_open(); -#if NATTR zebra_rec_keys_t sortkeys = zebra_rec_keys_open(); -#else - struct sortKeys sortkeys; -#endif zebra_rec_keys_set_buf(delkeys, rec->info[recInfo_delKeys], rec->size[recInfo_delKeys], 0); extract_flushRecordKeys (zh, rec->sysno, 0, delkeys, 0); zebra_rec_keys_close(delkeys); -#if NATTR + zebra_rec_keys_set_buf(sortkeys, rec->info[recInfo_sortKeys], rec->size[recInfo_sortKeys], 0); extract_flushSortKeys (zh, rec->sysno, 0, sortkeys); zebra_rec_keys_close(sortkeys); -#else - sortkeys.buf_used = rec->size[recInfo_sortKeys]; - sortkeys.buf = rec->info[recInfo_sortKeys]; - extract_flushSortKeys (zh, rec->sysno, 0, &sortkeys); -#endif } extract_flushRecordKeys (zh, rec->sysno, 1, zh->reg->keys, 0); -#if NATTR extract_flushSortKeys (zh, rec->sysno, 1, zh->reg->sortKeys); -#else - extract_flushSortKeys (zh, rec->sysno, 1, &zh->reg->sortKeys); -#endif xfree (rec->info[recInfo_delKeys]); zebra_rec_keys_get_buf(zh->reg->keys, @@ -1336,20 +1265,65 @@ int explain_extract (void *handle, Record rec, data1_node *n) &rec->size[recInfo_delKeys]); xfree (rec->info[recInfo_sortKeys]); -#if NATTR zebra_rec_keys_get_buf(zh->reg->sortKeys, &rec->info[recInfo_sortKeys], &rec->size[recInfo_sortKeys]); -#else - rec->size[recInfo_sortKeys] = zh->reg->sortKeys.buf_used; - rec->info[recInfo_sortKeys] = zh->reg->sortKeys.buf; - zh->reg->sortKeys.buf = NULL; - zh->reg->sortKeys.buf_max = 0; -#endif return 0; } +void extract_rec_keys_adjust(ZebraHandle zh, int is_insert, + zebra_rec_keys_t reckeys) +{ + ZebraExplainInfo zei = zh->reg->zei; + struct ord_stat { + int no; + int ord; + struct ord_stat *next; + }; + + if (zebra_rec_keys_rewind(reckeys)) + { + struct ord_stat *ord_list = 0; + struct ord_stat *p; + size_t slen; + const char *str; + struct it_key key_in; + while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in)) + { + int ord = key_in.mem[0]; + + for (p = ord_list; p ; p = p->next) + if (p->ord == ord) + { + p->no++; + break; + } + if (!p) + { + p = xmalloc(sizeof(*p)); + p->no = 1; + p->ord = ord; + p->next = ord_list; + ord_list = p; + } + } + + p = ord_list; + while (p) + { + struct ord_stat *p1 = p; + + if (is_insert) + zebraExplain_ord_adjust_occurrences(zei, p->ord, p->no, 1); + else + zebraExplain_ord_adjust_occurrences(zei, p->ord, - p->no, -1); + p = p->next; + xfree(p1); + } + } +} + void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, int cmd, zebra_rec_keys_t reckeys, @@ -1357,6 +1331,8 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, { ZebraExplainInfo zei = zh->reg->zei; + extract_rec_keys_adjust(zh, cmd, reckeys); + if (!zh->reg->key_buf) { int mem= 1024*1024* atoi( res_get_def( zh->res, "memmax", "8")); @@ -1403,7 +1379,7 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, zh->reg->key_buf_used += key_SU_encode(ch, (char*)zh->reg->key_buf + zh->reg->key_buf_used); - + /* copy the 0-terminated stuff from str to output */ memcpy((char*)zh->reg->key_buf + zh->reg->key_buf_used, str, slen); zh->reg->key_buf_used += slen; @@ -1637,7 +1613,7 @@ void print_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys) } } -void extract_add_index_string (RecWord *p, const char *str, int length) +void extract_add_index_string(RecWord *p, const char *str, int length) { struct it_key key; @@ -1689,8 +1665,7 @@ void extract_add_index_string (RecWord *p, const char *str, int length) zebra_rec_keys_write(zh->reg->keys, str, length, &key); } -#if NATTR -static void extract_add_sort_string (RecWord *p, const char *str, int length) +static void extract_add_sort_string(RecWord *p, const char *str, int length) { struct it_key key; @@ -1706,7 +1681,15 @@ static void extract_add_sort_string (RecWord *p, const char *str, int length) } else { +#if NATTR return; +#else + ch = zebraExplain_lookup_attr_su(zei, p->index_type, + VAL_IDXPATH, p->attrUse); + if (ch < 0) + ch = zebraExplain_add_attr_su(zei, p->index_type, + VAL_IDXPATH, p->attrUse); +#endif } key.len = 4; key.mem[0] = ch; @@ -1714,45 +1697,24 @@ static void extract_add_sort_string (RecWord *p, const char *str, int length) key.mem[2] = p->section_id; key.mem[3] = p->seqno; - zebra_rec_keys_write(zh->reg->sortKeys, str, length, &key); -} -#else -static void extract_add_sort_string (RecWord *p, const char *str, int length) -{ - ZebraHandle zh = p->extractCtrl->handle; - struct sortKeys *sk = &zh->reg->sortKeys; - int off = 0; - - while (off < sk->buf_used) +#if 0 + if (1) { - int set, use, slen; + char strz[80]; + int i; - off += key_SU_decode(&set, (unsigned char *) sk->buf + off); - off += key_SU_decode(&use, (unsigned char *) sk->buf + off); - off += key_SU_decode(&slen, (unsigned char *) sk->buf + off); - off += slen; - if (p->attrSet == set && p->attrUse == use) - return; + strz[0] = 0; + for (i = 0; iattrSet, p->attrUse, p->record_id, p->section_id, p->seqno, + strz); } - assert (off == sk->buf_used); - - if (sk->buf_used + IT_MAX_WORD > sk->buf_max) - { - char *b; - - b = (char *) xmalloc (sk->buf_max += 128000); - if (sk->buf_used > 0) - memcpy (b, sk->buf, sk->buf_used); - xfree (sk->buf); - sk->buf = b; - } - off += key_SU_encode(p->attrSet, sk->buf + off); - off += key_SU_encode(p->attrUse, sk->buf + off); - off += key_SU_encode(length, sk->buf + off); - memcpy (sk->buf + off, str, length); - sk->buf_used = off + length; -} #endif + zebra_rec_keys_write(zh->reg->sortKeys, str, length, &key); +} void extract_add_string (RecWord *p, const char *string, int length) { @@ -1926,67 +1888,30 @@ void extract_schema_add (struct recExtractCtrl *p, Odr_oid *oid) zebraExplain_addSchema (zh->reg->zei, oid); } -#if NATTR -#error not done yet with zebra_rec_keys_t void extract_flushSortKeys (ZebraHandle zh, SYSNO sysno, - int cmd, struct recKeys *reckeys) + int cmd, zebra_rec_keys_t reckeys) { - SortIdx sortIdx = zh->reg->sortIdx; - void *decode_handle = iscz1_start(); - int off = 0; - int ch = 0; - - while (off < reckeys->buf_used) + if (zebra_rec_keys_rewind(reckeys)) { - const char *src = reckeys->buf + off; - struct it_key key; - char *dst = (char*) &key; - - iscz1_decode(decode_handle, &dst, &src); - assert(key.len == 4); - - ch = (int) key.mem[0]; /* ordinal for field/use/attribute */ - - sortIdx_type(sortIdx, ch); - if (cmd == 1) - sortIdx_add(sortIdx, src, strlen(src)); - else - sortIdx_add(sortIdx, "", 1); - - src += strlen(src); - src++; - - off = src - reckeys->buf; - } - assert (off == reckeys->buf_used); - iscz1_stop(decode_handle); -} -#else -void extract_flushSortKeys (ZebraHandle zh, SYSNO sysno, - int cmd, struct sortKeys *sk) -{ - SortIdx sortIdx = zh->reg->sortIdx; - int off = 0; + SortIdx sortIdx = zh->reg->sortIdx; + size_t slen; + const char *str; + struct it_key key_in; - sortIdx_sysno (sortIdx, sysno); + sortIdx_sysno (sortIdx, sysno); - while (off < sk->buf_used) - { - int set, use, slen; - - off += key_SU_decode(&set, (unsigned char *) sk->buf + off); - off += key_SU_decode(&use, (unsigned char *) sk->buf + off); - off += key_SU_decode(&slen, (unsigned char *) sk->buf + off); - - sortIdx_type(sortIdx, use); - if (cmd == 1) - sortIdx_add(sortIdx, sk->buf + off, slen); - else - sortIdx_add(sortIdx, "", 1); - off += slen; + while (zebra_rec_keys_read(reckeys, &str, &slen, &key_in)) + { + int ord = (int) key_in.mem[0]; + + sortIdx_type(sortIdx, ord); + if (cmd == 1) + sortIdx_add(sortIdx, str, slen); + else + sortIdx_add(sortIdx, "", 1); + } } } -#endif void encode_key_init (struct encode_info *i) { @@ -2159,3 +2084,11 @@ void encode_key_flush (struct encode_info *i, FILE *outf) i->prevseq=0; } #endif +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ +