X-Git-Url: http://git.indexdata.com/?p=idzebra-moved-to-github.git;a=blobdiff_plain;f=index%2Fextract.c;h=3f218627ae218be4b17f323d3b2af6ed05288958;hp=ee771ac6587a8f0d91f68177d3cffbacf58ec1ea;hb=ecb3935e78cd9bcfdebafdee0834cfb1060d7b5e;hpb=00d69171180b9edc123986794b572be57caa5bb2 diff --git a/index/extract.c b/index/extract.c index ee771ac..3f21862 100644 --- a/index/extract.c +++ b/index/extract.c @@ -1,6 +1,6 @@ -/* $Id: extract.c,v 1.133 2002-12-16 22:59:34 adam Exp $ - Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002 - Index Data Aps +/* $Id: extract.c,v 1.209 2006-05-10 08:13:21 adam Exp $ + Copyright (C) 1995-2005 + Index Data ApS This file is part of the Zebra server. @@ -20,17 +20,19 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ - #include #include +#include #ifdef WIN32 #include -#else +#endif +#if HAVE_UNISTD_H #include #endif #include #include "index.h" +#include "orddict.h" #include #include @@ -72,104 +74,72 @@ static void logRecord (ZebraHandle zh) ++zh->records_processed; if (!(zh->records_processed % 1000)) { - logf (LOG_LOG, "Records: %7d i/u/d %d/%d/%d", + yaz_log (YLOG_LOG, "Records: "ZINT_FORMAT" i/u/d " + ZINT_FORMAT"/"ZINT_FORMAT"/"ZINT_FORMAT, zh->records_processed, zh->records_inserted, zh->records_updated, zh->records_deleted); } } +static void extract_set_store_data_prepare(struct recExtractCtrl *p); + static void extract_init (struct recExtractCtrl *p, RecWord *w) { w->zebra_maps = p->zebra_maps; w->seqno = 1; +#if NATTR +#else w->attrSet = VAL_BIB1; w->attrUse = 1016; - w->reg_type = 'w'; +#endif + w->index_name = 0; + w->index_type = 'w'; w->extractCtrl = p; + w->record_id = 0; + w->section_id = 0; } -static const char **searchRecordKey (ZebraHandle zh, - struct recKeys *reckeys, - int attrSetS, int attrUseS) +static void searchRecordKey(ZebraHandle zh, + zebra_rec_keys_t reckeys, + int attrSetS, int attrUseS, + const char **ws, int ws_length) { - static const char *ws[32]; - int off = 0; - int startSeq = -1; int i; - int seqno = 0; -#if SU_SCHEME - int chS, ch; -#else - short attrUse; - char attrSet; -#endif + int ch; - for (i = 0; i<32; i++) + for (i = 0; ireg->zei, attrSetS, attrUseS); - if (chS < 0) - return ws; -#endif - while (off < reckeys->buf_used) - { + ch = zebraExplain_lookup_attr_su_any_index(zh->reg->zei, + attrSetS, attrUseS); + if (ch < 0) + return ; - const char *src = reckeys->buf + off; - const char *wstart; - int lead; - - lead = *src++; -#if SU_SCHEME - if ((lead & 3)<3) + if (zebra_rec_keys_rewind(reckeys)) + { + int startSeq = -1; + const char *str; + size_t slen; + struct it_key key; + zint seqno; + while (zebra_rec_keys_read(reckeys, &str, &slen, &key)) { - memcpy (&ch, src, sizeof(ch)); - src += sizeof(ch); - } -#else - if (!(lead & 1)) - { - memcpy (&attrSet, src, sizeof(attrSet)); - src += sizeof(attrSet); - } - if (!(lead & 2)) - { - memcpy (&attrUse, src, sizeof(attrUse)); - src += sizeof(attrUse); - } -#endif - wstart = src; - while (*src++) - ; - if (lead & 60) - seqno += ((lead>>2) & 15)-1; - else - { - memcpy (&seqno, src, sizeof(seqno)); - src += sizeof(seqno); - } - if ( -#if SU_SCHEME - ch == chS -#else - attrUseS == attrUse && attrSetS == attrSet -#endif - ) - { - int woff; - - - if (startSeq == -1) - startSeq = seqno; - woff = seqno - startSeq; - if (woff >= 0 && woff < 31) - ws[woff] = wstart; - } + assert(key.len <= 4 && key.len > 2); - off = src - reckeys->buf; + seqno = key.mem[key.len-1]; + + if (key.mem[0] == ch) + { + int woff; + + if (startSeq == -1) + startSeq = seqno; + woff = seqno - startSeq; + if (woff >= 0 && woff < ws_length) + ws[woff] = str; + } + } } - assert (off == reckeys->buf_used); - return ws; } struct file_read_info { @@ -178,8 +148,6 @@ struct file_read_info { off_t file_moffset; /* offset of rec/rec boundary */ int file_more; int fd; - char *sdrbuf; - int sdrmax; }; static struct file_read_info *file_read_start (int fd) @@ -190,8 +158,8 @@ static struct file_read_info *file_read_start (int fd) fi->fd = fd; fi->file_max = 0; fi->file_moffset = 0; - fi->sdrbuf = 0; - fi->sdrmax = 0; + fi->file_offset = 0; + fi->file_more = 0; return fi; } @@ -204,8 +172,6 @@ static off_t file_seek (void *handle, off_t offset) { struct file_read_info *p = (struct file_read_info *) handle; p->file_offset = offset; - if (p->sdrbuf) - return offset; return lseek (p->fd, offset, SEEK_SET); } @@ -220,16 +186,7 @@ static int file_read (void *handle, char *buf, size_t count) struct file_read_info *p = (struct file_read_info *) handle; int fd = p->fd; int r; - if (p->sdrbuf) - { - r = count; - if (r > p->sdrmax - p->file_offset) - r = p->sdrmax - p->file_offset; - if (r) - memcpy (buf, p->sdrbuf + p->file_offset, r); - } - else - r = read (fd, buf, count); + r = read (fd, buf, count); if (r > 0) { p->file_offset += r; @@ -239,60 +196,59 @@ static int file_read (void *handle, char *buf, size_t count) return r; } -static void file_begin (void *handle) -{ - struct file_read_info *p = (struct file_read_info *) handle; - - p->file_offset = p->file_moffset; - if (!p->sdrbuf && p->file_moffset) - lseek (p->fd, p->file_moffset, SEEK_SET); - p->file_more = 0; -} - static void file_end (void *handle, off_t offset) { struct file_read_info *p = (struct file_read_info *) handle; - assert (p->file_more == 0); - p->file_more = 1; - p->file_moffset = offset; + if (offset != p->file_moffset) + { + p->file_moffset = offset; + p->file_more = 1; + } } +#define FILE_MATCH_BLANK "\t " + static char *fileMatchStr (ZebraHandle zh, - struct recKeys *reckeys, struct recordGroup *rGroup, + zebra_rec_keys_t reckeys, const char *fname, const char *spec) { static char dstBuf[2048]; /* static here ??? */ char *dst = dstBuf; const char *s = spec; - static const char **w; while (1) { - while (*s == ' ' || *s == '\t') - s++; + for (; *s && strchr(FILE_MATCH_BLANK, *s); s++) + ; if (!*s) break; if (*s == '(') { + const char *ws[32]; char attset_str[64], attname_str[64]; data1_attset *attset; int i; - char matchFlag[32]; int attSet = 1, attUse = 1; int first = 1; - - s++; - for (i = 0; *s && *s != ',' && *s != ')'; s++) - if (i < 63) + + for (s++; strchr(FILE_MATCH_BLANK, *s); s++) + ; + for (i = 0; *s && *s != ',' && *s != ')' && + !strchr(FILE_MATCH_BLANK, *s); s++) + if (i+1 < sizeof(attset_str)) attset_str[i++] = *s; attset_str[i] = '\0'; - + + for (; strchr(FILE_MATCH_BLANK, *s); s++) + ; if (*s == ',') { - s++; - for (i = 0; *s && *s != ')'; s++) - if (i < 63) + for (s++; strchr(FILE_MATCH_BLANK, *s); s++) + ; + for (i = 0; *s && *s != ')' && + !strchr(FILE_MATCH_BLANK, *s); s++) + if (i+1 < sizeof(attname_str)) attname_str[i++] = *s; attname_str[i] = '\0'; } @@ -307,36 +263,30 @@ static char *fileMatchStr (ZebraHandle zh, else attUse = atoi (attname_str); } - w = searchRecordKey (zh, reckeys, attSet, attUse); - assert (w); + searchRecordKey (zh, reckeys, attSet, attUse, ws, 32); - if (*s == ')') + if (*s != ')') { - for (i = 0; i<32; i++) - matchFlag[i] = 1; - } - else - { - logf (LOG_WARN, "Missing ) in match criteria %s in group %s", - spec, rGroup->groupName ? rGroup->groupName : "none"); + yaz_log (YLOG_WARN, "Missing ) in match criteria %s in group %s", + spec, zh->m_group ? zh->m_group : "none"); return NULL; } s++; for (i = 0; i<32; i++) - if (matchFlag[i] && w[i]) + if (ws[i]) { if (first) { *dst++ = ' '; first = 0; } - strcpy (dst, w[i]); - dst += strlen(w[i]); + strcpy (dst, ws[i]); + dst += strlen(ws[i]); } if (first) { - logf (LOG_WARN, "Record didn't contain match" + yaz_log (YLOG_WARN, "Record didn't contain match" " fields in (%s,%s)", attset_str, attname_str); return NULL; } @@ -347,25 +297,25 @@ static char *fileMatchStr (ZebraHandle zh, char special[64]; const char *spec_src = NULL; const char *s1 = ++s; - while (*s1 && *s1 != ' ' && *s1 != '\t') + while (*s1 && !strchr(FILE_MATCH_BLANK, *s1)) s1++; spec_len = s1 - s; - if (spec_len > 63) - spec_len = 63; + if (spec_len > sizeof(special)-1) + spec_len = sizeof(special)-1; memcpy (special, s, spec_len); special[spec_len] = '\0'; s = s1; if (!strcmp (special, "group")) - spec_src = rGroup->groupName; + spec_src = zh->m_group; else if (!strcmp (special, "database")) - spec_src = rGroup->databaseName; + spec_src = zh->basenames[0]; else if (!strcmp (special, "filename")) { spec_src = fname; } else if (!strcmp (special, "type")) - spec_src = rGroup->recordType; + spec_src = zh->m_record_type; else spec_src = NULL; if (spec_src) @@ -382,7 +332,7 @@ static char *fileMatchStr (ZebraHandle zh, while (*s && *s != stopMarker) { - if (i < 63) + if (i+1 < sizeof(tmpString)) tmpString[i++] = *s++; } if (*s) @@ -393,16 +343,16 @@ static char *fileMatchStr (ZebraHandle zh, } else { - logf (LOG_WARN, "Syntax error in match criteria %s in group %s", - spec, rGroup->groupName ? rGroup->groupName : "none"); + yaz_log (YLOG_WARN, "Syntax error in match criteria %s in group %s", + spec, zh->m_group ? zh->m_group : "none"); return NULL; } *dst++ = 1; } if (dst == dstBuf) { - logf (LOG_WARN, "No match criteria for record %s in group %s", - fname, rGroup->groupName ? rGroup->groupName : "none"); + yaz_log (YLOG_WARN, "No match criteria for record %s in group %s", + fname, zh->m_group ? zh->m_group : "none"); return NULL; } *dst = '\0'; @@ -414,133 +364,156 @@ struct recordLogInfo { int recordOffset; struct recordGroup *rGroup; }; - -static int recordExtract (ZebraHandle zh, - SYSNO *sysno, const char *fname, - struct recordGroup *rGroup, int deleteFlag, - struct file_read_info *fi, - RecType recType, char *subType, void *clientData) + +static void init_extractCtrl(ZebraHandle zh, struct recExtractCtrl *ctrl) +{ + int i; + for (i = 0; i<256; i++) + { + if (zebra_maps_is_positioned(zh->reg->zebra_maps, i)) + ctrl->seqno[i] = 1; + else + ctrl->seqno[i] = 0; + } + ctrl->zebra_maps = zh->reg->zebra_maps; + ctrl->flagShowRecords = !zh->m_flag_rw; +} + +static ZEBRA_RES file_extract_record(ZebraHandle zh, + SYSNO *sysno, const char *fname, + int deleteFlag, + struct file_read_info *fi, + int force_update, + RecType recType, + void *recTypeClientData) { RecordAttr *recordAttr; int r; - char *matchStr; + const char *matchStr = 0; SYSNO sysnotmp; Record rec; off_t recordOffset = 0; + struct recExtractCtrl extractCtrl; + + /* announce database */ + if (zebraExplain_curDatabase (zh->reg->zei, zh->basenames[0])) + { + if (zebraExplain_newDatabase (zh->reg->zei, zh->basenames[0], + zh->m_explain_database)) + return ZEBRA_FAIL; + } if (fi->fd != -1) { - struct recExtractCtrl extractCtrl; - /* we are going to read from a file, so prepare the extraction */ - int i; + zebra_rec_keys_reset(zh->reg->keys); - zh->reg->keys.buf_used = 0; - zh->reg->keys.prevAttrUse = -1; - zh->reg->keys.prevAttrSet = -1; - zh->reg->keys.prevSeqNo = 0; +#if NATTR + zebra_rec_keys_reset(zh->reg->sortKeys); +#else zh->reg->sortKeys.buf_used = 0; - zh->reg->sortKeys.buf_max = 0; - zh->reg->sortKeys.buf = 0; - +#endif recordOffset = fi->file_moffset; + extractCtrl.handle = zh; extractCtrl.offset = fi->file_moffset; extractCtrl.readf = file_read; extractCtrl.seekf = file_seek; extractCtrl.tellf = file_tell; extractCtrl.endf = file_end; extractCtrl.fh = fi; - extractCtrl.subType = subType; extractCtrl.init = extract_init; extractCtrl.tokenAdd = extract_token_add; extractCtrl.schemaAdd = extract_schema_add; extractCtrl.dh = zh->reg->dh; - extractCtrl.handle = zh; - for (i = 0; i<256; i++) - { - if (zebra_maps_is_positioned(zh->reg->zebra_maps, i)) - extractCtrl.seqno[i] = 1; - else - extractCtrl.seqno[i] = 0; - } - extractCtrl.zebra_maps = zh->reg->zebra_maps; - extractCtrl.flagShowRecords = !rGroup->flagRw; + extractCtrl.match_criteria[0] = '\0'; + extractCtrl.staticrank = 0; + + extractCtrl.first_record = fi->file_offset ? 0 : 1; + + extract_set_store_data_prepare(&extractCtrl); + + init_extractCtrl(zh, &extractCtrl); - if (!rGroup->flagRw) + if (!zh->m_flag_rw) printf ("File: %s " PRINTF_OFF_T "\n", fname, recordOffset); - if (rGroup->flagRw) + if (zh->m_flag_rw) { char msg[512]; sprintf (msg, "%s:" PRINTF_OFF_T , fname, recordOffset); yaz_log_init_prefix2 (msg); } - r = (*recType->extract)(clientData, &extractCtrl); + r = (*recType->extract)(recTypeClientData, &extractCtrl); yaz_log_init_prefix2 (0); if (r == RECCTRL_EXTRACT_EOF) - return 0; + return ZEBRA_FAIL; else if (r == RECCTRL_EXTRACT_ERROR_GENERIC) { /* error occured during extraction ... */ - if (rGroup->flagRw && - zh->records_processed < rGroup->fileVerboseLimit) + if (zh->m_flag_rw && + zh->records_processed < zh->m_file_verbose_limit) { - logf (LOG_WARN, "fail %s %s " PRINTF_OFF_T, rGroup->recordType, + yaz_log (YLOG_WARN, "fail %s %s " PRINTF_OFF_T, zh->m_record_type, fname, recordOffset); } - return 0; + return ZEBRA_FAIL; } else if (r == RECCTRL_EXTRACT_ERROR_NO_SUCH_FILTER) { /* error occured during extraction ... */ - if (rGroup->flagRw && - zh->records_processed < rGroup->fileVerboseLimit) + if (zh->m_flag_rw && + zh->records_processed < zh->m_file_verbose_limit) { - logf (LOG_WARN, "no filter for %s %s " - PRINTF_OFF_T, rGroup->recordType, + yaz_log (YLOG_WARN, "no filter for %s %s " + PRINTF_OFF_T, zh->m_record_type, fname, recordOffset); } - return 0; - } - if (zh->reg->keys.buf_used == 0) - { - /* the extraction process returned no information - the record - is probably empty - unless flagShowRecords is in use */ - if (!rGroup->flagRw) - return 1; - - logf (LOG_WARN, "empty %s %s " PRINTF_OFF_T, rGroup->recordType, - fname, recordOffset); - return 1; + return ZEBRA_FAIL; } + if (extractCtrl.match_criteria[0]) + matchStr = extractCtrl.match_criteria; } /* perform match if sysno not known and if match criteria is specified */ - - matchStr = NULL; if (!sysno) { sysnotmp = 0; sysno = &sysnotmp; - if (rGroup->recordId && *rGroup->recordId) + + if (matchStr == 0 && zh->m_record_id && *zh->m_record_id) { - char *rinfo; - - matchStr = fileMatchStr (zh, &zh->reg->keys, rGroup, fname, - rGroup->recordId); - if (matchStr) - { - rinfo = dict_lookup (zh->reg->matchDict, matchStr); - if (rinfo) - memcpy (sysno, rinfo+1, sizeof(*sysno)); - } - else - { - logf (LOG_WARN, "Bad match criteria"); - return 0; - } - } + matchStr = fileMatchStr (zh, zh->reg->keys, fname, + zh->m_record_id); + if (!matchStr) + { + yaz_log(YLOG_WARN, "Bad match criteria"); + return ZEBRA_FAIL; + } + } + if (matchStr) + { + int db_ord = zebraExplain_get_database_ord(zh->reg->zei); + char *rinfo = dict_lookup_ord(zh->reg->matchDict, db_ord, + matchStr); + if (rinfo) + { + assert(*rinfo == sizeof(*sysno)); + memcpy (sysno, rinfo+1, sizeof(*sysno)); + } + } + } + if (! *sysno && zebra_rec_keys_empty(zh->reg->keys) ) + { + /* the extraction process returned no information - the record + is probably empty - unless flagShowRecords is in use */ + if (!zh->m_flag_rw) + return ZEBRA_OK; + + if (zh->records_processed < zh->m_file_verbose_limit) + yaz_log (YLOG_WARN, "empty %s %s " PRINTF_OFF_T, zh->m_record_type, + fname, recordOffset); + return ZEBRA_OK; } if (! *sysno) @@ -548,106 +521,155 @@ static int recordExtract (ZebraHandle zh, /* new record */ if (deleteFlag) { - logf (LOG_LOG, "delete %s %s " PRINTF_OFF_T, rGroup->recordType, + yaz_log (YLOG_LOG, "delete %s %s " PRINTF_OFF_T, zh->m_record_type, fname, recordOffset); - logf (LOG_WARN, "cannot delete record above (seems new)"); - return 1; + yaz_log (YLOG_WARN, "cannot delete record above (seems new)"); + return ZEBRA_OK; } - if (zh->records_processed < rGroup->fileVerboseLimit) - logf (LOG_LOG, "add %s %s " PRINTF_OFF_T, rGroup->recordType, - fname, recordOffset); - rec = rec_new (zh->reg->records); + rec = rec_new (zh->reg->records); + *sysno = rec->sysno; - + + if (zh->records_processed < zh->m_file_verbose_limit) + if (matchStr) + yaz_log(YLOG_LOG, "add %s %s " PRINTF_OFF_T + " " ZINT_FORMAT " %s" , + zh->m_record_type, + fname, recordOffset, *sysno, matchStr); + else + yaz_log(YLOG_LOG, "add %s %s " PRINTF_OFF_T + " " ZINT_FORMAT , + zh->m_record_type, + fname, recordOffset, *sysno); + recordAttr = rec_init_attr (zh->reg->zei, rec); + recordAttr->staticrank = extractCtrl.staticrank; if (matchStr) { - dict_insert (zh->reg->matchDict, matchStr, sizeof(*sysno), sysno); + int db_ord = zebraExplain_get_database_ord(zh->reg->zei); + dict_insert_ord(zh->reg->matchDict, db_ord, matchStr, + sizeof(*sysno), sysno); } - extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); - extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys); + +#if NATTR + extract_flushSortKeys (zh, *sysno, 1, zh->reg->sortKeys); +#else + extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); +#endif + extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys, + recordAttr->staticrank); zh->records_inserted++; } else { /* record already exists */ - struct recKeys delkeys; + zebra_rec_keys_t delkeys = zebra_rec_keys_open(); + +#if NATTR + zebra_rec_keys_t sortKeys = zebra_rec_keys_open(); +#else struct sortKeys sortKeys; +#endif rec = rec_get (zh->reg->records, *sysno); assert (rec); recordAttr = rec_init_attr (zh->reg->zei, rec); - if (recordAttr->runNumber == - zebraExplain_runNumberIncrement (zh->reg->zei, 0)) - { - yaz_log (LOG_LOG, "run number = %d", recordAttr->runNumber); - yaz_log (LOG_LOG, "skipped %s %s " PRINTF_OFF_T, - rGroup->recordType, fname, recordOffset); - extract_flushSortKeys (zh, *sysno, -1, &zh->reg->sortKeys); - rec_rm (&rec); - logRecord (zh); - return 1; - } - delkeys.buf_used = rec->size[recInfo_delKeys]; - delkeys.buf = rec->info[recInfo_delKeys]; - + zebra_rec_keys_set_buf(delkeys, + rec->info[recInfo_delKeys], + rec->size[recInfo_delKeys], + 0); + +#if NATTR + zebra_rec_keys_set_buf(sortKeys, + rec->info[recInfo_sortKeys], + rec->size[recInfo_sortKeys], + 0); + extract_flushSortKeys (zh, *sysno, 0, sortKeys); +#else sortKeys.buf_used = rec->size[recInfo_sortKeys]; sortKeys.buf = rec->info[recInfo_sortKeys]; - extract_flushSortKeys (zh, *sysno, 0, &sortKeys); - extract_flushRecordKeys (zh, *sysno, 0, &delkeys); +#endif + + extract_flushRecordKeys (zh, *sysno, 0, delkeys, + recordAttr->staticrank); /* old values */ if (deleteFlag) { /* record going to be deleted */ - if (!delkeys.buf_used) + if (zebra_rec_keys_empty(delkeys)) { - logf (LOG_LOG, "delete %s %s " PRINTF_OFF_T, - rGroup->recordType, fname, recordOffset); - logf (LOG_WARN, "cannot delete file above, storeKeys false"); + yaz_log (YLOG_LOG, "delete %s %s " PRINTF_OFF_T + " " ZINT_FORMAT, + zh->m_record_type, fname, recordOffset, *sysno); + yaz_log (YLOG_WARN, "cannot delete file above, storeKeys false (1)"); } else { - if (zh->records_processed < rGroup->fileVerboseLimit) - logf (LOG_LOG, "delete %s %s " PRINTF_OFF_T, - rGroup->recordType, fname, recordOffset); + if (zh->records_processed < zh->m_file_verbose_limit) + if (matchStr) + yaz_log(YLOG_LOG, "delete %s %s " PRINTF_OFF_T + " " ZINT_FORMAT " %s" , + zh->m_record_type, + fname, recordOffset, *sysno, matchStr); + else + yaz_log(YLOG_LOG, "delete %s %s " PRINTF_OFF_T + " " ZINT_FORMAT , + zh->m_record_type, + fname, recordOffset, *sysno); + + + zh->records_deleted++; if (matchStr) - dict_delete (zh->reg->matchDict, matchStr); + { + int db_ord = zebraExplain_get_database_ord(zh->reg->zei); + dict_delete_ord(zh->reg->matchDict, db_ord, matchStr); + } rec_del (zh->reg->records, &rec); } rec_rm (&rec); logRecord (zh); - return 1; + return ZEBRA_OK; } else { - /* record going to be updated */ - if (!delkeys.buf_used) - { - logf (LOG_LOG, "update %s %s " PRINTF_OFF_T, - rGroup->recordType, fname, recordOffset); - logf (LOG_WARN, "cannot update file above, storeKeys false"); - } - else - { - if (zh->records_processed < rGroup->fileVerboseLimit) - logf (LOG_LOG, "update %s %s " PRINTF_OFF_T, - rGroup->recordType, fname, recordOffset); - extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); - extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys); - zh->records_updated++; - } + /* flush new keys for sort&search etc */ + if (zh->records_processed < zh->m_file_verbose_limit) + if (matchStr) + yaz_log(YLOG_LOG, "update %s %s " PRINTF_OFF_T + " " ZINT_FORMAT " %s" , + zh->m_record_type, + fname, recordOffset, *sysno, matchStr); + else + yaz_log(YLOG_LOG, "update %s %s " PRINTF_OFF_T + " " ZINT_FORMAT , + zh->m_record_type, + fname, recordOffset, *sysno); + + recordAttr->staticrank = extractCtrl.staticrank; +#if NATTR + extract_flushSortKeys (zh, *sysno, 1, zh->reg->sortKeys); +#else + extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); +#endif + extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys, + recordAttr->staticrank); + zh->records_updated++; } + zebra_rec_keys_close(delkeys); +#if NATTR + zebra_rec_keys_close(sortKeys); +#endif } /* update file type */ xfree (rec->info[recInfo_fileType]); rec->info[recInfo_fileType] = - rec_strdup (rGroup->recordType, &rec->size[recInfo_fileType]); + rec_strdup (zh->m_record_type, &rec->size[recInfo_fileType]); /* update filename */ xfree (rec->info[recInfo_filename]); @@ -656,12 +678,11 @@ static int recordExtract (ZebraHandle zh, /* update delete keys */ xfree (rec->info[recInfo_delKeys]); - if (zh->reg->keys.buf_used > 0 && rGroup->flagStoreKeys == 1) + if (!zebra_rec_keys_empty(zh->reg->keys) && zh->m_store_keys == 1) { - rec->size[recInfo_delKeys] = zh->reg->keys.buf_used; - rec->info[recInfo_delKeys] = zh->reg->keys.buf; - zh->reg->keys.buf = NULL; - zh->reg->keys.buf_max = 0; + zebra_rec_keys_get_buf(zh->reg->keys, + &rec->info[recInfo_delKeys], + &rec->size[recInfo_delKeys]); } else { @@ -672,10 +693,16 @@ static int recordExtract (ZebraHandle zh, /* update sort keys */ xfree (rec->info[recInfo_sortKeys]); +#if NATTR + zebra_rec_keys_get_buf(zh->reg->sortKeys, + &rec->info[recInfo_sortKeys], + &rec->size[recInfo_sortKeys]); +#else rec->size[recInfo_sortKeys] = zh->reg->sortKeys.buf_used; rec->info[recInfo_sortKeys] = zh->reg->sortKeys.buf; zh->reg->sortKeys.buf = NULL; zh->reg->sortKeys.buf_max = 0; +#endif /* save file size of original record */ zebraExplain_recordBytesIncrement (zh->reg->zei, @@ -692,21 +719,27 @@ static int recordExtract (ZebraHandle zh, /* update store data */ xfree (rec->info[recInfo_storeData]); - if (rGroup->flagStoreData == 1) + if (zh->store_data_buf) + { + rec->size[recInfo_storeData] = zh->store_data_size; + rec->info[recInfo_storeData] = zh->store_data_buf; + zh->store_data_buf = 0; + } + else if (zh->m_store_data) { rec->size[recInfo_storeData] = recordAttr->recordSize; rec->info[recInfo_storeData] = (char *) xmalloc (recordAttr->recordSize); if (lseek (fi->fd, recordOffset, SEEK_SET) < 0) { - logf (LOG_ERRNO|LOG_FATAL, "seek to " PRINTF_OFF_T " in %s", + yaz_log (YLOG_ERRNO|YLOG_FATAL, "seek to " PRINTF_OFF_T " in %s", recordOffset, fname); exit (1); } if (read (fi->fd, rec->info[recInfo_storeData], recordAttr->recordSize) < recordAttr->recordSize) { - logf (LOG_ERRNO|LOG_FATAL, "read %d bytes of %s", + yaz_log (YLOG_ERRNO|YLOG_FATAL, "read %d bytes of %s", recordAttr->recordSize, fname); exit (1); } @@ -719,7 +752,7 @@ static int recordExtract (ZebraHandle zh, /* update database name */ xfree (rec->info[recInfo_databaseName]); rec->info[recInfo_databaseName] = - rec_strdup (rGroup->databaseName, &rec->size[recInfo_databaseName]); + rec_strdup (zh->basenames[0], &rec->size[recInfo_databaseName]); /* update offset */ recordAttr->recordOffset = recordOffset; @@ -727,31 +760,28 @@ static int recordExtract (ZebraHandle zh, /* commit this record */ rec_put (zh->reg->records, &rec); logRecord (zh); - return 1; + return ZEBRA_OK; } -int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname, - const struct recordGroup *rGroupP, int deleteFlag) +ZEBRA_RES zebra_extract_file(ZebraHandle zh, SYSNO *sysno, const char *fname, + int deleteFlag) { - int r, i, fd; + ZEBRA_RES r = ZEBRA_OK; + int i, fd; char gprefix[128]; char ext[128]; char ext_res[128]; - char subType[128]; - RecType recType; - struct recordGroup rGroupM; - struct recordGroup *rGroup = &rGroupM; struct file_read_info *fi; - void *clientData; + const char *original_record_type = 0; + RecType recType; + void *recTypeClientData; - memcpy (rGroup, rGroupP, sizeof(*rGroupP)); - - if (!rGroup->groupName || !*rGroup->groupName) + if (!zh->m_group || !*zh->m_group) *gprefix = '\0'; else - sprintf (gprefix, "%s.", rGroup->groupName); - - logf (LOG_DEBUG, "fileExtract %s", fname); + sprintf (gprefix, "%s.", zh->m_group); + + yaz_log (YLOG_DEBUG, "fileExtract %s", fname); /* determine file extension */ *ext = '\0'; @@ -764,99 +794,40 @@ int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname, break; } /* determine file type - depending on extension */ - if (!rGroup->recordType) + original_record_type = zh->m_record_type; + if (!zh->m_record_type) { sprintf (ext_res, "%srecordType.%s", gprefix, ext); - if (!(rGroup->recordType = res_get (zh->res, ext_res))) - { - sprintf (ext_res, "%srecordType", gprefix); - rGroup->recordType = res_get (zh->res, ext_res); - } - } - if (!rGroup->recordType) - { - if (zh->records_processed < rGroup->fileVerboseLimit) - logf (LOG_LOG, "? %s", fname); - return 0; + zh->m_record_type = res_get (zh->res, ext_res); } - if (!*rGroup->recordType) - return 0; - if (!(recType = - recType_byName (zh->reg->recTypes, rGroup->recordType, subType, - &clientData))) + if (!zh->m_record_type) { - logf (LOG_WARN, "No such record type: %s", rGroup->recordType); + if (zh->records_processed < zh->m_file_verbose_limit) + yaz_log (YLOG_LOG, "? %s", fname); return 0; } - /* determine match criteria */ - if (!rGroup->recordId) + if (!zh->m_record_id) { sprintf (ext_res, "%srecordId.%s", gprefix, ext); - rGroup->recordId = res_get (zh->res, ext_res); - } - - /* determine database name */ - if (!rGroup->databaseName) - { - sprintf (ext_res, "%sdatabase.%s", gprefix, ext); - if (!(rGroup->databaseName = res_get (zh->res, ext_res))) - { - sprintf (ext_res, "%sdatabase", gprefix); - rGroup->databaseName = res_get (zh->res, ext_res); - } + zh->m_record_id = res_get (zh->res, ext_res); } - if (!rGroup->databaseName) - rGroup->databaseName = "Default"; - - /* determine if explain database */ - - sprintf (ext_res, "%sexplainDatabase", gprefix); - rGroup->explainDatabase = - atoi (res_get_def (zh->res, ext_res, "0")); - /* announce database */ - if (zebraExplain_curDatabase (zh->reg->zei, rGroup->databaseName)) + if (!(recType = + recType_byName (zh->reg->recTypes, zh->res, zh->m_record_type, + &recTypeClientData))) { - if (zebraExplain_newDatabase (zh->reg->zei, rGroup->databaseName, - rGroup->explainDatabase)) - return 0; + yaz_log(YLOG_WARN, "No such record type: %s", zh->m_record_type); + return ZEBRA_FAIL; } - if (rGroup->flagStoreData == -1) + switch(recType->version) { - const char *sval; - sprintf (ext_res, "%sstoreData.%s", gprefix, ext); - if (!(sval = res_get (zh->res, ext_res))) - { - sprintf (ext_res, "%sstoreData", gprefix); - sval = res_get (zh->res, ext_res); - } - if (sval) - rGroup->flagStoreData = atoi (sval); + case 0: + break; + default: + yaz_log(YLOG_WARN, "Bad filter version: %s", zh->m_record_type); } - if (rGroup->flagStoreData == -1) - rGroup->flagStoreData = 0; - - if (rGroup->flagStoreKeys == -1) - { - const char *sval; - - sprintf (ext_res, "%sstoreKeys.%s", gprefix, ext); - sval = res_get (zh->res, ext_res); - if (!sval) - { - sprintf (ext_res, "%sstoreKeys", gprefix); - sval = res_get (zh->res, ext_res); - } - if (!sval) - sval = res_get (zh->res, "storeKeys"); - if (sval) - rGroup->flagStoreKeys = atoi (sval); - } - if (rGroup->flagStoreKeys == -1) - rGroup->flagStoreKeys = 0; - if (sysno && deleteFlag) fd = -1; else @@ -872,81 +843,73 @@ int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname, else strcpy (full_rep, fname); - if ((fd = open (full_rep, O_BINARY|O_RDONLY)) == -1) { - logf (LOG_WARN|LOG_ERRNO, "open %s", full_rep); - return 0; + yaz_log (YLOG_WARN|YLOG_ERRNO, "open %s", full_rep); + zh->m_record_type = original_record_type; + return ZEBRA_FAIL; } } fi = file_read_start (fd); - do - { - file_begin (fi); - r = recordExtract (zh, sysno, fname, rGroup, deleteFlag, fi, - recType, subType, clientData); - } while (r && !sysno && fi->file_more); + while(1) + { + fi->file_moffset = fi->file_offset; + fi->file_more = 0; /* file_end not called (yet) */ + r = file_extract_record (zh, sysno, fname, deleteFlag, fi, 1, + recType, recTypeClientData); + if (fi->file_more) + { /* file_end has been called so reset offset .. */ + fi->file_offset = fi->file_moffset; + lseek(fi->fd, fi->file_moffset, SEEK_SET); + } + if (r != ZEBRA_OK) + { + break; + } + if (sysno) + { + break; + } + } file_read_stop (fi); if (fd != -1) close (fd); + zh->m_record_type = original_record_type; return r; } -int extract_rec_in_mem (ZebraHandle zh, const char *recordType, - const char *buf, size_t buf_size, - const char *databaseName, int delete_flag, - int test_mode, int *sysno, - int store_keys, int store_data, - const char *match_criteria) -{ - struct recordGroup rGroup; - rGroup.groupName = NULL; - rGroup.databaseName = (char *)databaseName; - rGroup.path = NULL; - rGroup.recordId = NULL; - rGroup.recordType = (char *)recordType; - rGroup.flagStoreData = store_data; - rGroup.flagStoreKeys = store_keys; - rGroup.flagRw = 1; - rGroup.databaseNamePath = 0; - rGroup.explainDatabase = 0; - rGroup.fileVerboseLimit = 100000; - rGroup.followLinks = -1; - return (bufferExtractRecord (zh, - buf, buf_size, - &rGroup, - delete_flag, - test_mode, - sysno, - match_criteria, - "")); -} + /* If sysno is provided, then it's used to identify the reocord. If not, and match_criteria is provided, then sysno is guessed If not, and a record is provided, then sysno is got from there */ -int bufferExtractRecord (ZebraHandle zh, - const char *buf, size_t buf_size, - struct recordGroup *rGroup, - int delete_flag, - int test_mode, - int *sysno, - const char *match_criteria, - const char *fname) - +ZEBRA_RES buffer_extract_record(ZebraHandle zh, + const char *buf, size_t buf_size, + int delete_flag, + int test_mode, + const char *recordType, + SYSNO *sysno, + const char *match_criteria, + const char *fname, + int force_update, + int allow_update) { + SYSNO sysno0 = 0; RecordAttr *recordAttr; struct recExtractCtrl extractCtrl; - int i, r; - char *matchStr = 0; - RecType recType; - char subType[1024]; + int r; + const char *matchStr = 0; + RecType recType = NULL; void *clientData; - SYSNO sysnotmp; Record rec; long recordOffset = 0; struct zebra_fetch_control fc; + const char *pr_fname = fname; /* filename to print .. */ + int show_progress = zh->records_processed < zh->m_file_verbose_limit ? 1:0; + + if (!pr_fname) + pr_fname = ""; /* make it printable if file is omitted */ fc.fd = -1; fc.record_int_buf = buf; @@ -960,99 +923,115 @@ int bufferExtractRecord (ZebraHandle zh, extractCtrl.seekf = zebra_record_int_seek; extractCtrl.tellf = zebra_record_int_tell; extractCtrl.endf = zebra_record_int_end; + extractCtrl.first_record = 1; extractCtrl.fh = &fc; - /* announce database */ - if (zebraExplain_curDatabase (zh->reg->zei, rGroup->databaseName)) + zebra_rec_keys_reset(zh->reg->keys); + +#if NATTR + zebra_rec_keys_reset(zh->reg->sortKeys); +#else + zh->reg->sortKeys.buf_used = 0; +#endif + if (zebraExplain_curDatabase (zh->reg->zei, zh->basenames[0])) { - if (zebraExplain_newDatabase (zh->reg->zei, rGroup->databaseName, 0)) - return 0; + if (zebraExplain_newDatabase (zh->reg->zei, zh->basenames[0], + zh->m_explain_database)) + return ZEBRA_FAIL; } - - if (!(rGroup->recordType)) { - logf (LOG_WARN, "No such record type defined"); - return 0; + + if (recordType && *recordType) + { + yaz_log (YLOG_DEBUG, "Record type explicitly specified: %s", recordType); + recType = recType_byName (zh->reg->recTypes, zh->res, recordType, + &clientData); + } + else + { + if (!(zh->m_record_type)) + { + yaz_log (YLOG_WARN, "No such record type defined"); + return ZEBRA_FAIL; + } + yaz_log (YLOG_DEBUG, "Get record type from rgroup: %s",zh->m_record_type); + recType = recType_byName (zh->reg->recTypes, zh->res, + zh->m_record_type, &clientData); + recordType = zh->m_record_type; } - - if (!(recType = - recType_byName (zh->reg->recTypes, rGroup->recordType, subType, - &clientData))) + + if (!recType) { - logf (LOG_WARN, "No such record type: %s", rGroup->recordType); - return 0; + yaz_log (YLOG_WARN, "No such record type: %s", recordType); + return ZEBRA_FAIL; } - zh->reg->keys.buf_used = 0; - zh->reg->keys.prevAttrUse = -1; - zh->reg->keys.prevAttrSet = -1; - zh->reg->keys.prevSeqNo = 0; - zh->reg->sortKeys.buf_used = 0; - zh->reg->sortKeys.buf_max = 0; - zh->reg->sortKeys.buf = 0; - - extractCtrl.subType = subType; + extractCtrl.init = extract_init; extractCtrl.tokenAdd = extract_token_add; extractCtrl.schemaAdd = extract_schema_add; extractCtrl.dh = zh->reg->dh; extractCtrl.handle = zh; - extractCtrl.zebra_maps = zh->reg->zebra_maps; - extractCtrl.flagShowRecords = 0; - for (i = 0; i<256; i++) - { - if (zebra_maps_is_positioned(zh->reg->zebra_maps, i)) - extractCtrl.seqno[i] = 1; - else - extractCtrl.seqno[i] = 0; - } + extractCtrl.match_criteria[0] = '\0'; + extractCtrl.staticrank = 0; + + init_extractCtrl(zh, &extractCtrl); + + extract_set_store_data_prepare(&extractCtrl); r = (*recType->extract)(clientData, &extractCtrl); if (r == RECCTRL_EXTRACT_EOF) - return 0; + return ZEBRA_FAIL; else if (r == RECCTRL_EXTRACT_ERROR_GENERIC) { /* error occured during extraction ... */ - yaz_log (LOG_WARN, "extract error: generic"); - return 0; + yaz_log (YLOG_WARN, "extract error: generic"); + return ZEBRA_FAIL; } else if (r == RECCTRL_EXTRACT_ERROR_NO_SUCH_FILTER) { /* error occured during extraction ... */ - yaz_log (LOG_WARN, "extract error: no such filter"); - return 0; + yaz_log (YLOG_WARN, "extract error: no such filter"); + return ZEBRA_FAIL; + } + + if (extractCtrl.match_criteria[0]) + match_criteria = extractCtrl.match_criteria; + + if (!sysno) { + + sysno = &sysno0; + + if (match_criteria && *match_criteria) { + matchStr = match_criteria; + } else { + if (zh->m_record_id && *zh->m_record_id) { + matchStr = fileMatchStr (zh, zh->reg->keys, pr_fname, + zh->m_record_id); + if (!matchStr) + { + yaz_log (YLOG_WARN, "Bad match criteria (recordID)"); + return ZEBRA_FAIL; + } + } + } + if (matchStr) + { + int db_ord = zebraExplain_get_database_ord(zh->reg->zei); + char *rinfo = dict_lookup_ord(zh->reg->matchDict, db_ord, + matchStr); + if (rinfo) + { + assert(*rinfo == sizeof(*sysno)); + memcpy (sysno, rinfo+1, sizeof(*sysno)); + } + } } - if (zh->reg->keys.buf_used == 0) + if (zebra_rec_keys_empty(zh->reg->keys)) { /* the extraction process returned no information - the record is probably empty - unless flagShowRecords is in use */ if (test_mode) - return 1; - logf (LOG_WARN, "No keys generated for record"); - logf (LOG_WARN, " The file is probably empty"); - return 1; - } - /* match criteria */ - matchStr = NULL; - - if (! *sysno) { - char *rinfo; - if (strlen(match_criteria) > 0) { - matchStr = (char *)match_criteria; - } else { - if (rGroup->recordId && *rGroup->recordId) { - matchStr = fileMatchStr (zh, &zh->reg->keys, rGroup, fname, - rGroup->recordId); - } - } - if (matchStr) { - rinfo = dict_lookup (zh->reg->matchDict, matchStr); - if (rinfo) - memcpy (sysno, rinfo+1, sizeof(*sysno)); - } else { - logf (LOG_WARN, "Bad match criteria (recordID)"); - return 0; - } - + return ZEBRA_OK; } if (! *sysno) @@ -1060,103 +1039,137 @@ int bufferExtractRecord (ZebraHandle zh, /* new record */ if (delete_flag) { - logf (LOG_LOG, "delete %s %s %ld", rGroup->recordType, - fname, (long) recordOffset); - logf (LOG_WARN, "cannot delete record above (seems new)"); - return 1; + yaz_log (YLOG_LOG, "delete %s %s %ld", recordType, + pr_fname, (long) recordOffset); + yaz_log (YLOG_WARN, "cannot delete record above (seems new)"); + return ZEBRA_FAIL; } - logf (LOG_LOG, "add %s %s %ld", rGroup->recordType, fname, - (long) recordOffset); + if (show_progress) + yaz_log (YLOG_LOG, "add %s %s %ld", recordType, pr_fname, + (long) recordOffset); rec = rec_new (zh->reg->records); *sysno = rec->sysno; recordAttr = rec_init_attr (zh->reg->zei, rec); + recordAttr->staticrank = extractCtrl.staticrank; if (matchStr) { - dict_insert (zh->reg->matchDict, matchStr, - sizeof(*sysno), sysno); + int db_ord = zebraExplain_get_database_ord(zh->reg->zei); + dict_insert_ord(zh->reg->matchDict, db_ord, matchStr, + sizeof(*sysno), sysno); } - extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); - extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys); + +#if NATTR + extract_flushSortKeys (zh, *sysno, 1, zh->reg->sortKeys); +#else + extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); +#endif + +#if 0 + print_rec_keys(zh, zh->reg->keys); +#endif + extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys, + recordAttr->staticrank); zh->records_inserted++; - } + } else { /* record already exists */ - struct recKeys delkeys; + zebra_rec_keys_t delkeys = zebra_rec_keys_open(); +#if NATTR + zebra_rec_keys_t sortKeys = zebra_rec_keys_open(); +#else struct sortKeys sortKeys; +#endif + + if (!allow_update) + { + yaz_log (YLOG_LOG, "skipped %s %s %ld", + recordType, pr_fname, (long) recordOffset); + logRecord(zh); + return ZEBRA_FAIL; + } rec = rec_get (zh->reg->records, *sysno); assert (rec); recordAttr = rec_init_attr (zh->reg->zei, rec); - if (recordAttr->runNumber == - zebraExplain_runNumberIncrement (zh->reg->zei, 0)) - { - logf (LOG_LOG, "skipped %s %s %ld", rGroup->recordType, - fname, (long) recordOffset); - extract_flushSortKeys (zh, *sysno, -1, &zh->reg->sortKeys); - rec_rm (&rec); - logRecord(zh); - return 1; - } - delkeys.buf_used = rec->size[recInfo_delKeys]; - delkeys.buf = rec->info[recInfo_delKeys]; - + zebra_rec_keys_set_buf(delkeys, + rec->info[recInfo_delKeys], + rec->size[recInfo_delKeys], + 0); +#if NATTR + zebra_rec_keys_set_buf(sortKeys, + rec->info[recInfo_sortKeys], + rec->size[recInfo_sortKeys], + 0); +#else sortKeys.buf_used = rec->size[recInfo_sortKeys]; sortKeys.buf = rec->info[recInfo_sortKeys]; +#endif +#if NATTR + extract_flushSortKeys (zh, *sysno, 0, sortKeys); +#else extract_flushSortKeys (zh, *sysno, 0, &sortKeys); - extract_flushRecordKeys (zh, *sysno, 0, &delkeys); +#endif + extract_flushRecordKeys (zh, *sysno, 0, delkeys, + recordAttr->staticrank); if (delete_flag) { /* record going to be deleted */ - if (!delkeys.buf_used) + if (zebra_rec_keys_empty(delkeys)) { - logf (LOG_LOG, "delete %s %s %ld", rGroup->recordType, - fname, (long) recordOffset); - logf (LOG_WARN, "cannot delete file above, storeKeys false"); - } + yaz_log (YLOG_LOG, "delete %s %s %ld", recordType, + pr_fname, (long) recordOffset); + yaz_log (YLOG_WARN, "cannot delete file above, " + "storeKeys false (3)"); + } else { - logf (LOG_LOG, "delete %s %s %ld", rGroup->recordType, - fname, (long) recordOffset); + if (show_progress) + yaz_log (YLOG_LOG, "delete %s %s %ld", recordType, + pr_fname, (long) recordOffset); zh->records_deleted++; if (matchStr) - dict_delete (zh->reg->matchDict, matchStr); + { + int db_ord = zebraExplain_get_database_ord(zh->reg->zei); + dict_delete_ord(zh->reg->matchDict, db_ord, matchStr); + } rec_del (zh->reg->records, &rec); } rec_rm (&rec); logRecord(zh); - return 1; + return ZEBRA_OK; } else { - /* record going to be updated */ - if (!delkeys.buf_used) - { - logf (LOG_LOG, "update %s %s %ld", rGroup->recordType, - fname, (long) recordOffset); - logf (LOG_WARN, "cannot update file above, storeKeys false"); - } - else - { - logf (LOG_LOG, "update %s %s %ld", rGroup->recordType, - fname, (long) recordOffset); - extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); - extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys); - zh->records_updated++; - } + if (show_progress) + yaz_log (YLOG_LOG, "update %s %s %ld", recordType, + pr_fname, (long) recordOffset); + recordAttr->staticrank = extractCtrl.staticrank; +#if NATTR + extract_flushSortKeys (zh, *sysno, 1, zh->reg->sortKeys); +#else + extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); +#endif + extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys, + recordAttr->staticrank); + zh->records_updated++; } + zebra_rec_keys_close(delkeys); +#if NATTR + zebra_rec_keys_close(sortKeys); +#endif } /* update file type */ xfree (rec->info[recInfo_fileType]); rec->info[recInfo_fileType] = - rec_strdup (rGroup->recordType, &rec->size[recInfo_fileType]); + rec_strdup (recordType, &rec->size[recInfo_fileType]); /* update filename */ xfree (rec->info[recInfo_filename]); @@ -1165,26 +1178,30 @@ int bufferExtractRecord (ZebraHandle zh, /* update delete keys */ xfree (rec->info[recInfo_delKeys]); - if (zh->reg->keys.buf_used > 0 && rGroup->flagStoreKeys == 1) + if (!zebra_rec_keys_empty(zh->reg->keys) && zh->m_store_keys == 1) { - rec->size[recInfo_delKeys] = zh->reg->keys.buf_used; - rec->info[recInfo_delKeys] = zh->reg->keys.buf; - zh->reg->keys.buf = NULL; - zh->reg->keys.buf_max = 0; + zebra_rec_keys_get_buf(zh->reg->keys, + &rec->info[recInfo_delKeys], + &rec->size[recInfo_delKeys]); } else { rec->info[recInfo_delKeys] = NULL; rec->size[recInfo_delKeys] = 0; } - /* update sort keys */ xfree (rec->info[recInfo_sortKeys]); +#if NATTR + zebra_rec_keys_get_buf(zh->reg->sortKeys, + &rec->info[recInfo_sortKeys], + &rec->size[recInfo_sortKeys]); +#else rec->size[recInfo_sortKeys] = zh->reg->sortKeys.buf_used; rec->info[recInfo_sortKeys] = zh->reg->sortKeys.buf; zh->reg->sortKeys.buf = NULL; zh->reg->sortKeys.buf_max = 0; +#endif /* save file size of original record */ zebraExplain_recordBytesIncrement (zh->reg->zei, @@ -1205,28 +1222,20 @@ int bufferExtractRecord (ZebraHandle zh, /* update store data */ xfree (rec->info[recInfo_storeData]); - if (rGroup->flagStoreData == 1) + + /* update store data */ + if (zh->store_data_buf) + { + rec->size[recInfo_storeData] = zh->store_data_size; + rec->info[recInfo_storeData] = zh->store_data_buf; + zh->store_data_buf = 0; + } + else if (zh->m_store_data) { rec->size[recInfo_storeData] = recordAttr->recordSize; rec->info[recInfo_storeData] = (char *) xmalloc (recordAttr->recordSize); -#if 1 memcpy (rec->info[recInfo_storeData], buf, recordAttr->recordSize); -#else - if (lseek (fi->fd, recordOffset, SEEK_SET) < 0) - { - logf (LOG_ERRNO|LOG_FATAL, "seek to %ld in %s", - (long) recordOffset, fname); - exit (1); - } - if (read (fi->fd, rec->info[recInfo_storeData], recordAttr->recordSize) - < recordAttr->recordSize) - { - logf (LOG_ERRNO|LOG_FATAL, "read %d bytes of %s", - recordAttr->recordSize, fname); - exit (1); - } -#endif } else { @@ -1236,7 +1245,7 @@ int bufferExtractRecord (ZebraHandle zh, /* update database name */ xfree (rec->info[recInfo_databaseName]); rec->info[recInfo_databaseName] = - rec_strdup (rGroup->databaseName, &rec->size[recInfo_databaseName]); + rec_strdup (zh->basenames[0], &rec->size[recInfo_databaseName]); /* update offset */ recordAttr->recordOffset = recordOffset; @@ -1244,16 +1253,13 @@ int bufferExtractRecord (ZebraHandle zh, /* commit this record */ rec_put (zh->reg->records, &rec); logRecord(zh); - return 0; + return ZEBRA_OK; } - - int explain_extract (void *handle, Record rec, data1_node *n) { ZebraHandle zh = (ZebraHandle) handle; struct recExtractCtrl extractCtrl; - int i; if (zebraExplain_curDatabase (zh->reg->zei, rec->info[recInfo_databaseName])) @@ -1264,67 +1270,91 @@ int explain_extract (void *handle, Record rec, data1_node *n) abort (); } - zh->reg->keys.buf_used = 0; - zh->reg->keys.prevAttrUse = -1; - zh->reg->keys.prevAttrSet = -1; - zh->reg->keys.prevSeqNo = 0; - zh->reg->sortKeys.buf_used = 0; + zebra_rec_keys_reset(zh->reg->keys); +#if NATTR + zebra_rec_keys_reset(zh->reg->sortKeys); +#else + zh->reg->sortKeys.buf_used = 0; +#endif extractCtrl.init = extract_init; extractCtrl.tokenAdd = extract_token_add; extractCtrl.schemaAdd = extract_schema_add; extractCtrl.dh = zh->reg->dh; - for (i = 0; i<256; i++) - extractCtrl.seqno[i] = 0; - extractCtrl.zebra_maps = zh->reg->zebra_maps; + + init_extractCtrl(zh, &extractCtrl); + extractCtrl.flagShowRecords = 0; + extractCtrl.match_criteria[0] = '\0'; + extractCtrl.staticrank = 0; extractCtrl.handle = handle; + extractCtrl.first_record = 1; - grs_extract_tree(&extractCtrl, n); + extract_set_store_data_prepare(&extractCtrl); + + if (n) + grs_extract_tree(&extractCtrl, n); if (rec->size[recInfo_delKeys]) { - struct recKeys delkeys; + zebra_rec_keys_t delkeys = zebra_rec_keys_open(); + +#if NATTR + zebra_rec_keys_t sortkeys = zebra_rec_keys_open(); +#else struct sortKeys sortkeys; +#endif - delkeys.buf_used = rec->size[recInfo_delKeys]; - delkeys.buf = rec->info[recInfo_delKeys]; - + zebra_rec_keys_set_buf(delkeys, rec->info[recInfo_delKeys], + rec->size[recInfo_delKeys], + 0); + extract_flushRecordKeys (zh, rec->sysno, 0, delkeys, 0); + zebra_rec_keys_close(delkeys); +#if NATTR + zebra_rec_keys_set_buf(sortkeys, rec->info[recInfo_sortKeys], + rec->size[recInfo_sortKeys], + 0); + + extract_flushSortKeys (zh, rec->sysno, 0, sortkeys); + zebra_rec_keys_close(sortkeys); +#else sortkeys.buf_used = rec->size[recInfo_sortKeys]; sortkeys.buf = rec->info[recInfo_sortKeys]; - extract_flushSortKeys (zh, rec->sysno, 0, &sortkeys); - extract_flushRecordKeys (zh, rec->sysno, 0, &delkeys); +#endif } - extract_flushRecordKeys (zh, rec->sysno, 1, &zh->reg->keys); + extract_flushRecordKeys (zh, rec->sysno, 1, zh->reg->keys, 0); +#if NATTR + extract_flushSortKeys (zh, rec->sysno, 1, zh->reg->sortKeys); +#else extract_flushSortKeys (zh, rec->sysno, 1, &zh->reg->sortKeys); +#endif xfree (rec->info[recInfo_delKeys]); - rec->size[recInfo_delKeys] = zh->reg->keys.buf_used; - rec->info[recInfo_delKeys] = zh->reg->keys.buf; - zh->reg->keys.buf = NULL; - zh->reg->keys.buf_max = 0; + zebra_rec_keys_get_buf(zh->reg->keys, + &rec->info[recInfo_delKeys], + &rec->size[recInfo_delKeys]); xfree (rec->info[recInfo_sortKeys]); +#if NATTR + zebra_rec_keys_get_buf(zh->reg->sortKeys, + &rec->info[recInfo_sortKeys], + &rec->size[recInfo_sortKeys]); +#else rec->size[recInfo_sortKeys] = zh->reg->sortKeys.buf_used; rec->info[recInfo_sortKeys] = zh->reg->sortKeys.buf; zh->reg->sortKeys.buf = NULL; zh->reg->sortKeys.buf_max = 0; +#endif return 0; } void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, - int cmd, struct recKeys *reckeys) + int cmd, + zebra_rec_keys_t reckeys, + zint staticrank) { -#if SU_SCHEME -#else - unsigned char attrSet = (unsigned char) -1; - unsigned short attrUse = (unsigned short) -1; -#endif - int seqno = 0; - int off = 0; - int ch = 0; ZebraExplainInfo zei = zh->reg->zei; if (!zh->reg->key_buf) @@ -1332,7 +1362,7 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, int mem= 1024*1024* atoi( res_get_def( zh->res, "memmax", "8")); if (mem <= 0) { - logf(LOG_WARN, "Invalid memory setting, using default 8 MB"); + yaz_log(YLOG_WARN, "Invalid memory setting, using default 8 MB"); mem= 1024*1024*8; } /* FIXME: That "8" should be in a default settings include */ @@ -1344,97 +1374,140 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, zh->reg->key_file_no = 0; } zebraExplain_recordCountIncrement (zei, cmd ? 1 : -1); - while (off < reckeys->buf_used) - { - const char *src = reckeys->buf + off; - struct it_key key; - int lead; - - lead = *src++; -#if SU_SCHEME - if ((lead & 3) < 3) + if (zebra_rec_keys_rewind(reckeys)) + { + size_t slen; + const char *str; + struct it_key key_in; + while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in)) { - memcpy (&ch, src, sizeof(ch)); - src += sizeof(ch); - } -#else - if (!(lead & 1)) - { - memcpy (&attrSet, src, sizeof(attrSet)); - src += sizeof(attrSet); - } - if (!(lead & 2)) - { - memcpy (&attrUse, src, sizeof(attrUse)); - src += sizeof(attrUse); - } -#endif - if (zh->reg->key_buf_used + 1024 > - (zh->reg->ptr_top -zh->reg->ptr_i)*sizeof(char*)) - extract_flushWriteKeys (zh); - ++(zh->reg->ptr_i); - (zh->reg->key_buf)[zh->reg->ptr_top - zh->reg->ptr_i] = - (char*)zh->reg->key_buf + zh->reg->key_buf_used; -#if SU_SCHEME -#else - ch = zebraExplain_lookupSU (zei, attrSet, attrUse); - if (ch < 0) - ch = zebraExplain_addSU (zei, attrSet, attrUse); -#endif - assert (ch > 0); - zh->reg->key_buf_used += - key_SU_encode (ch,((char*)zh->reg->key_buf) + - zh->reg->key_buf_used); + int ch = 0; + struct it_key key_out; + zint *keyp = key_out.mem; - while (*src) - ((char*)zh->reg->key_buf) [(zh->reg->key_buf_used)++] = *src++; - src++; - ((char*)(zh->reg->key_buf))[(zh->reg->key_buf_used)++] = '\0'; - ((char*)(zh->reg->key_buf))[(zh->reg->key_buf_used)++] = cmd; + assert(key_in.len == 4); + + /* check for buffer overflow */ + if (zh->reg->key_buf_used + 1024 > + (zh->reg->ptr_top -zh->reg->ptr_i)*sizeof(char*)) + extract_flushWriteKeys (zh, 0); + + ++(zh->reg->ptr_i); + assert(zh->reg->ptr_i > 0); + (zh->reg->key_buf)[zh->reg->ptr_top - zh->reg->ptr_i] = + (char*)zh->reg->key_buf + zh->reg->key_buf_used; + + /* encode the ordinal value (field/use/attribute) .. */ + ch = (int) key_in.mem[0]; + zh->reg->key_buf_used += + key_SU_encode(ch, (char*)zh->reg->key_buf + + zh->reg->key_buf_used); + + /* copy the 0-terminated stuff from str to output */ + memcpy((char*)zh->reg->key_buf + zh->reg->key_buf_used, str, slen); + zh->reg->key_buf_used += slen; + ((char*)zh->reg->key_buf)[(zh->reg->key_buf_used)++] = '\0'; - if (lead & 60) - seqno += ((lead>>2) & 15)-1; - else - { - memcpy (&seqno, src, sizeof(seqno)); - src += sizeof(seqno); - } - key.seqno = seqno; - key.sysno = sysno; - memcpy ((char*)zh->reg->key_buf + zh->reg->key_buf_used, &key, sizeof(key)); - (zh->reg->key_buf_used) += sizeof(key); - off = src - reckeys->buf; + /* the delete/insert indicator */ + ((char*)zh->reg->key_buf)[(zh->reg->key_buf_used)++] = cmd; + + if (zh->m_staticrank) /* rank config enabled ? */ + { + if (staticrank < 0) + { + yaz_log(YLOG_WARN, "staticrank = %ld. Setting to 0", + (long) staticrank); + staticrank = 0; + } + *keyp++ = staticrank; + key_out.len = 4; + } + else + key_out.len = 3; + + if (key_in.mem[1]) /* filter specified record ID */ + *keyp++ = key_in.mem[1]; + else + *keyp++ = sysno; + *keyp++ = key_in.mem[2]; /* section_id */ + *keyp++ = key_in.mem[3]; /* sequence .. */ + + memcpy((char*)zh->reg->key_buf + zh->reg->key_buf_used, + &key_out, sizeof(key_out)); + (zh->reg->key_buf_used) += sizeof(key_out); + } } - assert (off == reckeys->buf_used); } -void extract_flushWriteKeys (ZebraHandle zh) +void extract_flushWriteKeys (ZebraHandle zh, int final) + /* optimizing: if final=1, and no files written yet */ + /* push the keys directly to merge, sidestepping the */ + /* temp file altogether. Speeds small updates */ { FILE *outf; char out_fname[200]; char *prevcp, *cp; struct encode_info encode_info; int ptr_i = zh->reg->ptr_i; + int temp_policy; #if SORT_EXTRA int i; #endif if (!zh->reg->key_buf || ptr_i <= 0) + { + yaz_log (YLOG_DEBUG, " nothing to flush section=%d buf=%p i=%d", + zh->reg->key_file_no, zh->reg->key_buf, ptr_i); + yaz_log (YLOG_DEBUG, " buf=%p ", + zh->reg->key_buf); + yaz_log (YLOG_DEBUG, " ptr=%d ",zh->reg->ptr_i); + yaz_log (YLOG_DEBUG, " reg=%p ",zh->reg); + return; + } (zh->reg->key_file_no)++; - logf (LOG_LOG, "sorting section %d", (zh->reg->key_file_no)); + yaz_log (YLOG_LOG, "sorting section %d", (zh->reg->key_file_no)); + yaz_log (YLOG_DEBUG, " sort_buff at %p n=%d", + zh->reg->key_buf + zh->reg->ptr_top - ptr_i,ptr_i); #if !SORT_EXTRA qsort (zh->reg->key_buf + zh->reg->ptr_top - ptr_i, ptr_i, sizeof(char*), key_qsort_compare); + + /* zebra.cfg: tempfiles: + Y: always use temp files (old way) + A: use temp files, if more than one (auto) + = if this is both the last and the first + N: never bother with temp files (new) */ + + temp_policy=toupper(res_get_def(zh->res,"tempfiles","auto")[0]); + if (temp_policy != 'Y' && temp_policy != 'N' && temp_policy != 'A') { + yaz_log (YLOG_WARN, "Illegal tempfiles setting '%c'. using 'Auto' ", + temp_policy); + temp_policy='A'; + } + + if ( ( temp_policy =='N' ) || /* always from memory */ + ( ( temp_policy =='A' ) && /* automatic */ + (zh->reg->key_file_no == 1) && /* this is first time */ + (final) ) ) /* and last (=only) time */ + { /* go directly from memory */ + zh->reg->key_file_no =0; /* signal not to read files */ + zebra_index_merge(zh); + zh->reg->ptr_i = 0; + zh->reg->key_buf_used = 0; + return; + } + + /* Not doing directly from memory, write into a temp file */ extract_get_fname_tmp (zh, out_fname, zh->reg->key_file_no); if (!(outf = fopen (out_fname, "wb"))) { - logf (LOG_FATAL|LOG_ERRNO, "fopen %s", out_fname); + yaz_log (YLOG_FATAL|YLOG_ERRNO, "fopen %s", out_fname); exit (1); } - logf (LOG_LOG, "writing section %d", zh->reg->key_file_no); + yaz_log (YLOG_LOG, "writing section %d", zh->reg->key_file_no); prevcp = cp = (zh->reg->key_buf)[zh->reg->ptr_top - ptr_i]; encode_key_init (&encode_info); @@ -1460,10 +1533,10 @@ void extract_flushWriteKeys (ZebraHandle zh) if (!(outf = fopen (out_fname, "wb"))) { - logf (LOG_FATAL|LOG_ERRNO, "fopen %s", out_fname); + yaz_log (YLOG_FATAL|YLOG_ERRNO, "fopen %s", out_fname); exit (1); } - logf (LOG_LOG, "writing section %d", key_file_no); + yaz_log (YLOG_LOG, "writing section %d", key_file_no); i = ptr_i; prevcp = key_buf[ptr_top-i]; while (1) @@ -1471,7 +1544,7 @@ void extract_flushWriteKeys (ZebraHandle zh) { key_y_len = strlen(prevcp)+1; #if 0 - logf (LOG_LOG, "key_y_len: %2d %02x %02x %s", + yaz_log (YLOG_LOG, "key_y_len: %2d %02x %02x %s", key_y_len, prevcp[0], prevcp[1], 2+prevcp); #endif qsort (key_buf + ptr_top-ptr_i, ptr_i - i, @@ -1493,113 +1566,170 @@ void extract_flushWriteKeys (ZebraHandle zh) #endif if (fclose (outf)) { - logf (LOG_FATAL|LOG_ERRNO, "fclose %s", out_fname); + yaz_log (YLOG_FATAL|YLOG_ERRNO, "fclose %s", out_fname); exit (1); } - logf (LOG_LOG, "finished section %d", zh->reg->key_file_no); + yaz_log (YLOG_LOG, "finished section %d", zh->reg->key_file_no); zh->reg->ptr_i = 0; zh->reg->key_buf_used = 0; } -void extract_add_index_string (RecWord *p, const char *string, - int length) +ZEBRA_RES zebra_snippets_rec_keys(ZebraHandle zh, + zebra_rec_keys_t reckeys, + zebra_snippets *snippets) { - char *dst; - unsigned char attrSet; - unsigned short attrUse; - int lead = 0; - int diff = 0; - int *pseqno = &p->seqno; - ZebraHandle zh = p->extractCtrl->handle; - ZebraExplainInfo zei = zh->reg->zei; - struct recKeys *keys = &zh->reg->keys; - - if (keys->buf_used+1024 > keys->buf_max) + NMEM nmem = nmem_create(); + if (zebra_rec_keys_rewind(reckeys)) { - char *b; + const char *str; + size_t slen; + struct it_key key; + while (zebra_rec_keys_read(reckeys, &str, &slen, &key)) + { + char dst_buf[IT_MAX_WORD]; + char *dst_term = dst_buf; + int ord, seqno; + int index_type; + assert(key.len <= 4 && key.len > 2); + seqno = (int) key.mem[key.len-1]; + ord = key.mem[0]; + + zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, + 0/* db */, 0/* set */, 0/* use */, + 0 /* string_index */); + assert(index_type); + zebra_term_untrans_iconv(zh, nmem, index_type, + &dst_term, str); + zebra_snippets_append(snippets, seqno, ord, dst_term); + nmem_reset(nmem); + } + } + nmem_destroy(nmem); + return ZEBRA_OK; +} - b = (char *) xmalloc (keys->buf_max += 128000); - if (keys->buf_used > 0) - memcpy (b, keys->buf, keys->buf_used); - xfree (keys->buf); - keys->buf = b; +void print_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys) +{ + yaz_log(YLOG_LOG, "print_rec_keys"); + if (zebra_rec_keys_rewind(reckeys)) + { + const char *str; + size_t slen; + struct it_key key; + while (zebra_rec_keys_read(reckeys, &str, &slen, &key)) + { + char dst_buf[IT_MAX_WORD]; + int seqno; + int index_type; + const char *db = 0; + assert(key.len <= 4 && key.len > 2); + + zebraExplain_lookup_ord(zh->reg->zei, + key.mem[0], &index_type, &db, 0, 0, 0); + + seqno = (int) key.mem[key.len-1]; + + zebra_term_untrans(zh, index_type, dst_buf, str); + + yaz_log(YLOG_LOG, "ord=" ZINT_FORMAT " seqno=%d term=%s", + key.mem[0], seqno, dst_buf); + } } - dst = keys->buf + keys->buf_used; +} - attrSet = p->attrSet; - if (keys->buf_used > 0 && keys->prevAttrSet == attrSet) - lead |= 1; - else - keys->prevAttrSet = attrSet; - attrUse = p->attrUse; - if (keys->buf_used > 0 && keys->prevAttrUse == attrUse) - lead |= 2; - else - keys->prevAttrUse = attrUse; -#if 1 - diff = 1 + *pseqno - keys->prevSeqNo; - if (diff >= 1 && diff <= 15) - lead |= (diff << 2); - else - diff = 0; -#endif - keys->prevSeqNo = *pseqno; - - *dst++ = lead; +void extract_add_index_string (RecWord *p, const char *str, int length) +{ + struct it_key key; + + ZebraHandle zh = p->extractCtrl->handle; + ZebraExplainInfo zei = zh->reg->zei; + int ch; -#if SU_SCHEME - if ((lead & 3) < 3) + if (p->index_name) { - int ch = zebraExplain_lookupSU (zei, attrSet, attrUse); - if (ch < 0) - { - ch = zebraExplain_addSU (zei, attrSet, attrUse); - yaz_log (LOG_DEBUG, "addSU set=%d use=%d SU=%d", - attrSet, attrUse, ch); - } - assert (ch > 0); - memcpy (dst, &ch, sizeof(ch)); - dst += sizeof(ch); + ch = zebraExplain_lookup_attr_str(zei, p->index_type, p->index_name); + if (ch < 0) + ch = zebraExplain_add_attr_str(zei, p->index_type, p->index_name); } -#else - if (!(lead & 1)) + else { - memcpy (dst, &attrSet, sizeof(attrSet)); - dst += sizeof(attrSet); +#if NATTR + return; +#else + ch = zebraExplain_lookup_attr_su(zei, p->index_type, + p->attrSet, p->attrUse); + if (ch < 0) + ch = zebraExplain_add_attr_su(zei, p->index_type, + p->attrSet, p->attrUse); +#endif } - if (!(lead & 2)) + key.len = 4; + key.mem[0] = ch; + key.mem[1] = p->record_id; + key.mem[2] = p->section_id; + key.mem[3] = p->seqno; + +#if 0 + if (1) { - memcpy (dst, &attrUse, sizeof(attrUse)); - dst += sizeof(attrUse); + char strz[80]; + int i; + + strz[0] = 0; + for (i = 0; iattrSet, p->attrUse, p->record_id, p->section_id, p->seqno, + strz); } #endif - *dst++ = p->reg_type; - memcpy (dst, string, length); - dst += length; - *dst++ = '\0'; - if (!diff) + zebra_rec_keys_write(zh->reg->keys, str, length, &key); +} + +#if NATTR +static void extract_add_sort_string (RecWord *p, const char *str, int length) +{ + struct it_key key; + + ZebraHandle zh = p->extractCtrl->handle; + ZebraExplainInfo zei = zh->reg->zei; + int ch; + + if (p->index_name) { - memcpy (dst, pseqno, sizeof(*pseqno)); - dst += sizeof(*pseqno); + ch = zebraExplain_lookup_attr_str(zei, p->index_type, p->index_name); + if (ch < 0) + ch = zebraExplain_add_attr_str(zei, p->index_type, p->index_name); } - keys->buf_used = dst - keys->buf; -} + else + { + return; + } + key.len = 4; + key.mem[0] = ch; + key.mem[1] = p->record_id; + key.mem[2] = p->section_id; + key.mem[3] = p->seqno; -static void extract_add_sort_string (RecWord *p, const char *string, - int length) + zebra_rec_keys_write(zh->reg->sortKeys, str, length, &key); +} +#else +static void extract_add_sort_string (RecWord *p, const char *str, int length) { ZebraHandle zh = p->extractCtrl->handle; struct sortKeys *sk = &zh->reg->sortKeys; - size_t off = 0; + int off = 0; while (off < sk->buf_used) { int set, use, slen; - off += key_SU_decode(&set, sk->buf + off); - off += key_SU_decode(&use, sk->buf + off); - off += key_SU_decode(&slen, sk->buf + off); + off += key_SU_decode(&set, (unsigned char *) sk->buf + off); + off += key_SU_decode(&use, (unsigned char *) sk->buf + off); + off += key_SU_decode(&slen, (unsigned char *) sk->buf + off); off += slen; if (p->attrSet == set && p->attrUse == use) return; @@ -1619,14 +1749,15 @@ static void extract_add_sort_string (RecWord *p, const char *string, off += key_SU_encode(p->attrSet, sk->buf + off); off += key_SU_encode(p->attrUse, sk->buf + off); off += key_SU_encode(length, sk->buf + off); - memcpy (sk->buf + off, string, length); + memcpy (sk->buf + off, str, length); sk->buf_used = off + length; } +#endif void extract_add_string (RecWord *p, const char *string, int length) { assert (length > 0); - if (zebra_maps_is_sort (p->zebra_maps, p->reg_type)) + if (zebra_maps_is_sort (p->zebra_maps, p->index_type)) extract_add_sort_string (p, string, length); else extract_add_index_string (p, string, length); @@ -1634,12 +1765,14 @@ void extract_add_string (RecWord *p, const char *string, int length) static void extract_add_incomplete_field (RecWord *p) { - const char *b = p->string; - int remain = p->length; + const char *b = p->term_buf; + int remain = p->term_len; const char **map = 0; + + yaz_log(YLOG_DEBUG, "Incomplete field, w='%.*s'", p->term_len, p->term_buf); if (remain > 0) - map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain); + map = zebra_maps_input(p->zebra_maps, p->index_type, &b, remain, 0); while (map) { @@ -1649,9 +1782,10 @@ static void extract_add_incomplete_field (RecWord *p) /* Skip spaces */ while (map && *map && **map == *CHR_SPACE) { - remain = p->length - (b - p->string); + remain = p->term_len - (b - p->term_buf); if (remain > 0) - map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain); + map = zebra_maps_input(p->zebra_maps, p->index_type, &b, + remain, 0); else map = 0; } @@ -1664,9 +1798,9 @@ static void extract_add_incomplete_field (RecWord *p) while (i < IT_MAX_WORD && *cp) buf[i++] = *(cp++); - remain = p->length - (b - p->string); + remain = p->term_len - (b - p->term_buf); if (remain > 0) - map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain); + map = zebra_maps_input(p->zebra_maps, p->index_type, &b, remain, 0); else map = 0; } @@ -1679,21 +1813,28 @@ static void extract_add_incomplete_field (RecWord *p) static void extract_add_complete_field (RecWord *p) { - const char *b = p->string; + const char *b = p->term_buf; char buf[IT_MAX_WORD+1]; const char **map = 0; - int i = 0, remain = p->length; + int i = 0, remain = p->term_len; + + yaz_log(YLOG_DEBUG, "Complete field, w='%.*s'", + p->term_len, p->term_buf); if (remain > 0) - map = zebra_maps_input (p->zebra_maps, p->reg_type, &b, remain); + map = zebra_maps_input (p->zebra_maps, p->index_type, &b, remain, 1); while (remain > 0 && i < IT_MAX_WORD) { while (map && *map && **map == *CHR_SPACE) { - remain = p->length - (b - p->string); + remain = p->term_len - (b - p->term_buf); + if (remain > 0) - map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain); + { + int first = i ? 0 : 1; /* first position */ + map = zebra_maps_input(p->zebra_maps, p->index_type, &b, remain, first); + } else map = 0; } @@ -1706,14 +1847,24 @@ static void extract_add_complete_field (RecWord *p) { const char *cp = *map; - if (i >= IT_MAX_WORD) - break; - while (i < IT_MAX_WORD && *cp) - buf[i++] = *(cp++); - remain = p->length - (b - p->string); + if (**map == *CHR_CUT) + { + i = 0; + } + else + { + if (i >= IT_MAX_WORD) + break; + yaz_log(YLOG_DEBUG, "Adding string to index '%d'", **map); + while (i < IT_MAX_WORD && *cp) + buf[i++] = *(cp++); + } + remain = p->term_len - (b - p->term_buf); if (remain > 0) - map = zebra_maps_input (p->zebra_maps, p->reg_type, &b, - remain); + { + map = zebra_maps_input (p->zebra_maps, p->index_type, &b, + remain, 0); + } else map = 0; } @@ -1726,44 +1877,106 @@ static void extract_add_complete_field (RecWord *p) void extract_token_add (RecWord *p) { WRBUF wrbuf; - - yaz_log (LOG_LOG, "reg_type=%c attrSet=%d attrUse=%d seqno=%d s=%.*s", +#if 0 + yaz_log (YLOG_LOG, "token_add " + "reg_type=%c attrSet=%d attrUse=%d seqno=%d s=%.*s", p->reg_type, p->attrSet, p->attrUse, p->seqno, p->length, p->string); - - if ((wrbuf = zebra_replace(p->zebra_maps, p->reg_type, 0, - p->string, p->length))) +#endif + if ((wrbuf = zebra_replace(p->zebra_maps, p->index_type, 0, + p->term_buf, p->term_len))) { - p->string = wrbuf_buf(wrbuf); - p->length = wrbuf_len(wrbuf); + p->term_buf = wrbuf_buf(wrbuf); + p->term_len = wrbuf_len(wrbuf); } - if (zebra_maps_is_complete (p->zebra_maps, p->reg_type)) + if (zebra_maps_is_complete (p->zebra_maps, p->index_type)) extract_add_complete_field (p); else extract_add_incomplete_field(p); } +static void extract_set_store_data_cb(struct recExtractCtrl *p, + void *buf, size_t sz) +{ + ZebraHandle zh = (ZebraHandle) p->handle; + + xfree(zh->store_data_buf); + zh->store_data_buf = 0; + zh->store_data_size = 0; + if (buf && sz) + { + zh->store_data_buf = xmalloc(sz); + zh->store_data_size = sz; + memcpy(zh->store_data_buf, buf, sz); + } +} + +static void extract_set_store_data_prepare(struct recExtractCtrl *p) +{ + ZebraHandle zh = (ZebraHandle) p->handle; + xfree(zh->store_data_buf); + zh->store_data_buf = 0; + zh->store_data_size = 0; + p->setStoreData = extract_set_store_data_cb; +} + void extract_schema_add (struct recExtractCtrl *p, Odr_oid *oid) { - ZebraHandle zh = (ZebraHandle) (p->handle); + ZebraHandle zh = (ZebraHandle) p->handle; zebraExplain_addSchema (zh->reg->zei, oid); } +#if NATTR +#error not done yet with zebra_rec_keys_t +void extract_flushSortKeys (ZebraHandle zh, SYSNO sysno, + int cmd, struct recKeys *reckeys) +{ + SortIdx sortIdx = zh->reg->sortIdx; + void *decode_handle = iscz1_start(); + int off = 0; + int ch = 0; + + while (off < reckeys->buf_used) + { + const char *src = reckeys->buf + off; + struct it_key key; + char *dst = (char*) &key; + + iscz1_decode(decode_handle, &dst, &src); + assert(key.len == 4); + + ch = (int) key.mem[0]; /* ordinal for field/use/attribute */ + + sortIdx_type(sortIdx, ch); + if (cmd == 1) + sortIdx_add(sortIdx, src, strlen(src)); + else + sortIdx_add(sortIdx, "", 1); + + src += strlen(src); + src++; + + off = src - reckeys->buf; + } + assert (off == reckeys->buf_used); + iscz1_stop(decode_handle); +} +#else void extract_flushSortKeys (ZebraHandle zh, SYSNO sysno, int cmd, struct sortKeys *sk) { SortIdx sortIdx = zh->reg->sortIdx; - size_t off = 0; + int off = 0; sortIdx_sysno (sortIdx, sysno); while (off < sk->buf_used) { - int set, use, slen, l; + int set, use, slen; - off += key_SU_decode(&set, sk->buf + off); - off += key_SU_decode(&use, sk->buf + off); - off += key_SU_decode(&slen, sk->buf + off); + off += key_SU_decode(&set, (unsigned char *) sk->buf + off); + off += key_SU_decode(&use, (unsigned char *) sk->buf + off); + off += key_SU_decode(&slen, (unsigned char *) sk->buf + off); sortIdx_type(sortIdx, use); if (cmd == 1) @@ -1773,6 +1986,7 @@ void extract_flushSortKeys (ZebraHandle zh, SYSNO sysno, off += slen; } } +#endif void encode_key_init (struct encode_info *i) { @@ -1783,32 +1997,11 @@ void encode_key_init (struct encode_info *i) i->prevseq=0; i->prevcmd=-1; i->keylen=0; + i->encode_handle = iscz1_start(); + i->decode_handle = iscz1_start(); } -char *encode_key_int (int d, char *bp) -{ - if (d <= 63) - *bp++ = d; - else if (d <= 16383) - { - *bp++ = 64 + (d>>8); - *bp++ = d & 255; - } - else if (d <= 4194303) - { - *bp++ = 128 + (d>>16); - *bp++ = (d>>8) & 255; - *bp++ = d & 255; - } - else - { - *bp++ = 192 + (d>>24); - *bp++ = (d>>16) & 255; - *bp++ = (d>>8) & 255; - *bp++ = d & 255; - } - return bp; -} +#define OLDENCODE 1 #ifdef OLDENCODE /* this is the old encode_key_write @@ -1818,31 +2011,51 @@ char *encode_key_int (int d, char *bp) void encode_key_write (char *k, struct encode_info *i, FILE *outf) { struct it_key key; - char *bp = i->buf; + char *bp = i->buf, *bp0; + const char *src = (char *) &key; + /* copy term to output buf */ while ((*bp++ = *k++)) ; - memcpy (&key, k+1, sizeof(struct it_key)); - bp = encode_key_int ( (key.sysno - i->sysno) * 2 + *k, bp); - if (i->sysno != key.sysno) - { - i->sysno = key.sysno; - i->seqno = 0; - } - else if (!i->seqno && !key.seqno && i->cmd == *k) - return; - bp = encode_key_int (key.seqno - i->seqno, bp); - i->seqno = key.seqno; - i->cmd = *k; + /* and copy & align key so we can mangle */ + memcpy (&key, k+1, sizeof(struct it_key)); /* *k is insert/delete */ + +#if 0 + /* debugging */ + key_logdump_txt(YLOG_LOG, &key, *k ? "i" : "d"); +#endif + assert(key.mem[0] >= 0); + + bp0 = bp++; + iscz1_encode(i->encode_handle, &bp, &src); + + *bp0 = (*k * 128) + bp - bp0 - 1; /* length and insert/delete combined */ if (fwrite (i->buf, bp - i->buf, 1, outf) != 1) { - logf (LOG_FATAL|LOG_ERRNO, "fwrite"); + yaz_log (YLOG_FATAL|YLOG_ERRNO, "fwrite"); exit (1); } + +#if 0 + /* debugging */ + if (1) + { + struct it_key key2; + const char *src = bp0+1; + char *dst = (char*) &key2; + iscz1_decode(i->decode_handle, &dst, &src); + + key_logdump_txt(YLOG_LOG, &key2, *k ? "i" : "d"); + + assert(key2.mem[1]); + } +#endif } void encode_key_flush (struct encode_info *i, FILE *outf) -{ /* dummy routine */ +{ + iscz1_stop(i->encode_handle); + iscz1_stop(i->decode_handle); } #else @@ -1904,7 +2117,7 @@ void encode_key_write (char *k, struct encode_info *i, FILE *outf) i->cmd = i->prevcmd; if (fwrite (i->buf, bp - i->buf, 1, outf) != 1) { - logf (LOG_FATAL|LOG_ERRNO, "fwrite"); + yaz_log (YLOG_FATAL|YLOG_ERRNO, "fwrite"); exit (1); } i->keylen=0; /* ok, it's written, forget it */ @@ -1938,7 +2151,7 @@ void encode_key_flush (struct encode_info *i, FILE *outf) i->cmd = i->prevcmd; if (fwrite (i->buf, bp - i->buf, 1, outf) != 1) { - logf (LOG_FATAL|LOG_ERRNO, "fwrite"); + yaz_log (YLOG_FATAL|YLOG_ERRNO, "fwrite"); exit (1); } i->keylen=0; /* ok, it's written, forget it */ @@ -1946,3 +2159,11 @@ void encode_key_flush (struct encode_info *i, FILE *outf) i->prevseq=0; } #endif +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ +