X-Git-Url: http://git.indexdata.com/?p=idzebra-moved-to-github.git;a=blobdiff_plain;f=index%2Fextract.c;h=18cac3d597c58577d9346d69cc32ed0805d2f728;hp=e3255287e911323efb4939fc1bcd955a5d70411c;hb=7c3a0352f0492609a3b6b26b63a72b0b2d207aab;hpb=90307a85924c3b9a6ce11db3d13d0c7f24fd396d diff --git a/index/extract.c b/index/extract.c index e325528..18cac3d 100644 --- a/index/extract.c +++ b/index/extract.c @@ -1,5 +1,5 @@ -/* $Id: extract.c,v 1.144 2003-04-15 16:46:18 adam Exp $ - Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003 +/* $Id: extract.c,v 1.161 2004-09-14 14:38:07 quinn Exp $ + Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003,2004 Index Data Aps This file is part of the Zebra server. @@ -20,9 +20,9 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ - #include #include +#include #ifdef WIN32 #include #else @@ -72,7 +72,8 @@ static void logRecord (ZebraHandle zh) ++zh->records_processed; if (!(zh->records_processed % 1000)) { - logf (LOG_LOG, "Records: %7d i/u/d %d/%d/%d", + logf (LOG_LOG, "Records: "ZINT_FORMAT" i/u/d " + ZINT_FORMAT"/"ZINT_FORMAT"/"ZINT_FORMAT, zh->records_processed, zh->records_inserted, zh->records_updated, zh->records_deleted); } @@ -92,6 +93,51 @@ static const char **searchRecordKey (ZebraHandle zh, struct recKeys *reckeys, int attrSetS, int attrUseS) { +#if IT_KEY_NEW +/* #error searchRecordKey does not work yet in this mode.. */ + static const char *ws[32]; + void *decode_handle = iscz1_start(); + int off = 0; + int startSeq = -1; + int seqno = 0; + int i; + + for (i = 0; i<32; i++) + ws[i] = NULL; + + while (off < reckeys->buf_used) + { + const char *src = reckeys->buf + off; + struct it_key key; + char *dst = (char*) &key; + int attrSet, attrUse; + + iscz1_decode(decode_handle, &dst, &src); + assert(key.len < 4 && key.len > 2); + + attrSet = (int) key.mem[0]; + attrUse = (int) key.mem[1]; + seqno = (int) key.mem[2]; + + if (attrUseS == attrUse && attrSetS == attrSet) + { + int woff; + + if (startSeq == -1) + startSeq = seqno; + woff = seqno - startSeq; + if (woff >= 0 && woff < 31) + ws[woff] = src; + } + + while (*src++) + ; + off = src - reckeys->buf; + } + iscz1_stop(decode_handle); + assert (off == reckeys->buf_used); + return ws; +#else static const char *ws[32]; int off = 0; int startSeq = -1; @@ -170,6 +216,7 @@ static const char **searchRecordKey (ZebraHandle zh, } assert (off == reckeys->buf_used); return ws; +#endif } struct file_read_info { @@ -259,7 +306,7 @@ static void file_end (void *handle, off_t offset) } static char *fileMatchStr (ZebraHandle zh, - struct recKeys *reckeys, struct recordGroup *rGroup, + struct recKeys *reckeys, const char *fname, const char *spec) { static char dstBuf[2048]; /* static here ??? */ @@ -318,7 +365,7 @@ static char *fileMatchStr (ZebraHandle zh, else { logf (LOG_WARN, "Missing ) in match criteria %s in group %s", - spec, rGroup->groupName ? rGroup->groupName : "none"); + spec, zh->m_group ? zh->m_group : "none"); return NULL; } s++; @@ -358,14 +405,14 @@ static char *fileMatchStr (ZebraHandle zh, s = s1; if (!strcmp (special, "group")) - spec_src = rGroup->groupName; + spec_src = zh->m_group; else if (!strcmp (special, "database")) - spec_src = rGroup->databaseName; + spec_src = zh->basenames[0]; else if (!strcmp (special, "filename")) { spec_src = fname; } else if (!strcmp (special, "type")) - spec_src = rGroup->recordType; + spec_src = zh->m_record_type; else spec_src = NULL; if (spec_src) @@ -394,7 +441,7 @@ static char *fileMatchStr (ZebraHandle zh, else { logf (LOG_WARN, "Syntax error in match criteria %s in group %s", - spec, rGroup->groupName ? rGroup->groupName : "none"); + spec, zh->m_group ? zh->m_group : "none"); return NULL; } *dst++ = 1; @@ -402,7 +449,7 @@ static char *fileMatchStr (ZebraHandle zh, if (dst == dstBuf) { logf (LOG_WARN, "No match criteria for record %s in group %s", - fname, rGroup->groupName ? rGroup->groupName : "none"); + fname, zh->m_group ? zh->m_group : "none"); return NULL; } *dst = '\0'; @@ -414,20 +461,50 @@ struct recordLogInfo { int recordOffset; struct recordGroup *rGroup; }; + +void create_rec_keys_codec(struct recKeys *keys) +{ + keys->buf_used = 0; +#if IT_KEY_NEW + iscz1_reset(keys->codec_handle); +#else + keys->prevAttrUse = -1; + keys->prevAttrSet = -1; + keys->prevSeqNo = 0; +#endif +} -static int recordExtract (ZebraHandle zh, - SYSNO *sysno, const char *fname, - struct recordGroup *rGroup, int deleteFlag, - struct file_read_info *fi, - RecType recType, char *subType, void *clientData, - int force_update) +static int file_extract_record(ZebraHandle zh, + SYSNO *sysno, const char *fname, + int deleteFlag, + struct file_read_info *fi, + int force_update) { RecordAttr *recordAttr; int r; - char *matchStr; + const char *matchStr; SYSNO sysnotmp; Record rec; off_t recordOffset = 0; + RecType recType; + void *clientData; + char subType[128]; + + if (!(recType = + recType_byName (zh->reg->recTypes, zh->m_record_type, subType, + &clientData))) + { + logf (LOG_WARN, "No such record type: %s", zh->m_record_type); + return 0; + } + + /* announce database */ + if (zebraExplain_curDatabase (zh->reg->zei, zh->basenames[0])) + { + if (zebraExplain_newDatabase (zh->reg->zei, zh->basenames[0], + zh->m_explain_database)) + return 0; + } if (fi->fd != -1) { @@ -436,10 +513,8 @@ static int recordExtract (ZebraHandle zh, /* we are going to read from a file, so prepare the extraction */ int i; - zh->reg->keys.buf_used = 0; - zh->reg->keys.prevAttrUse = -1; - zh->reg->keys.prevAttrSet = -1; - zh->reg->keys.prevSeqNo = 0; + create_rec_keys_codec(&zh->reg->keys); + zh->reg->sortKeys.buf_used = 0; recordOffset = fi->file_moffset; @@ -463,11 +538,11 @@ static int recordExtract (ZebraHandle zh, extractCtrl.seqno[i] = 0; } extractCtrl.zebra_maps = zh->reg->zebra_maps; - extractCtrl.flagShowRecords = !rGroup->flagRw; + extractCtrl.flagShowRecords = !zh->m_flag_rw; - if (!rGroup->flagRw) + if (!zh->m_flag_rw) printf ("File: %s " PRINTF_OFF_T "\n", fname, recordOffset); - if (rGroup->flagRw) + if (zh->m_flag_rw) { char msg[512]; sprintf (msg, "%s:" PRINTF_OFF_T , fname, recordOffset); @@ -482,10 +557,10 @@ static int recordExtract (ZebraHandle zh, else if (r == RECCTRL_EXTRACT_ERROR_GENERIC) { /* error occured during extraction ... */ - if (rGroup->flagRw && - zh->records_processed < rGroup->fileVerboseLimit) + if (zh->m_flag_rw && + zh->records_processed < zh->m_file_verbose_limit) { - logf (LOG_WARN, "fail %s %s " PRINTF_OFF_T, rGroup->recordType, + logf (LOG_WARN, "fail %s %s " PRINTF_OFF_T, zh->m_record_type, fname, recordOffset); } return 0; @@ -493,11 +568,11 @@ static int recordExtract (ZebraHandle zh, else if (r == RECCTRL_EXTRACT_ERROR_NO_SUCH_FILTER) { /* error occured during extraction ... */ - if (rGroup->flagRw && - zh->records_processed < rGroup->fileVerboseLimit) + if (zh->m_flag_rw && + zh->records_processed < zh->m_file_verbose_limit) { logf (LOG_WARN, "no filter for %s %s " - PRINTF_OFF_T, rGroup->recordType, + PRINTF_OFF_T, zh->m_record_type, fname, recordOffset); } return 0; @@ -506,10 +581,10 @@ static int recordExtract (ZebraHandle zh, { /* the extraction process returned no information - the record is probably empty - unless flagShowRecords is in use */ - if (!rGroup->flagRw) + if (!zh->m_flag_rw) return 1; - logf (LOG_WARN, "empty %s %s " PRINTF_OFF_T, rGroup->recordType, + logf (LOG_WARN, "empty %s %s " PRINTF_OFF_T, zh->m_record_type, fname, recordOffset); return 1; } @@ -522,17 +597,20 @@ static int recordExtract (ZebraHandle zh, { sysnotmp = 0; sysno = &sysnotmp; - if (rGroup->recordId && *rGroup->recordId) + if (zh->m_record_id && *zh->m_record_id) { char *rinfo; - matchStr = fileMatchStr (zh, &zh->reg->keys, rGroup, fname, - rGroup->recordId); + matchStr = fileMatchStr (zh, &zh->reg->keys, fname, + zh->m_record_id); if (matchStr) { rinfo = dict_lookup (zh->reg->matchDict, matchStr); if (rinfo) + { + assert(*rinfo == sizeof(*sysno)); memcpy (sysno, rinfo+1, sizeof(*sysno)); + } } else { @@ -547,13 +625,13 @@ static int recordExtract (ZebraHandle zh, /* new record */ if (deleteFlag) { - logf (LOG_LOG, "delete %s %s " PRINTF_OFF_T, rGroup->recordType, + logf (LOG_LOG, "delete %s %s " PRINTF_OFF_T, zh->m_record_type, fname, recordOffset); logf (LOG_WARN, "cannot delete record above (seems new)"); return 1; } - if (zh->records_processed < rGroup->fileVerboseLimit) - logf (LOG_LOG, "add %s %s " PRINTF_OFF_T, rGroup->recordType, + if (zh->records_processed < zh->m_file_verbose_limit) + logf (LOG_LOG, "add %s %s " PRINTF_OFF_T, zh->m_record_type, fname, recordOffset); rec = rec_new (zh->reg->records); @@ -586,7 +664,7 @@ static int recordExtract (ZebraHandle zh, { yaz_log (LOG_LOG, "run number = %d", recordAttr->runNumber); yaz_log (LOG_LOG, "skipped %s %s " PRINTF_OFF_T, - rGroup->recordType, fname, recordOffset); + zh->m_record_type, fname, recordOffset); extract_flushSortKeys (zh, *sysno, -1, &zh->reg->sortKeys); rec_rm (&rec); logRecord (zh); @@ -606,14 +684,14 @@ static int recordExtract (ZebraHandle zh, if (!delkeys.buf_used) { logf (LOG_LOG, "delete %s %s " PRINTF_OFF_T, - rGroup->recordType, fname, recordOffset); + zh->m_record_type, fname, recordOffset); logf (LOG_WARN, "cannot delete file above, storeKeys false"); } else { - if (zh->records_processed < rGroup->fileVerboseLimit) + if (zh->records_processed < zh->m_file_verbose_limit) logf (LOG_LOG, "delete %s %s " PRINTF_OFF_T, - rGroup->recordType, fname, recordOffset); + zh->m_record_type, fname, recordOffset); zh->records_deleted++; if (matchStr) dict_delete (zh->reg->matchDict, matchStr); @@ -629,14 +707,14 @@ static int recordExtract (ZebraHandle zh, if (!delkeys.buf_used) { logf (LOG_LOG, "update %s %s " PRINTF_OFF_T, - rGroup->recordType, fname, recordOffset); + zh->m_record_type, fname, recordOffset); logf (LOG_WARN, "cannot update file above, storeKeys false"); } else { - if (zh->records_processed < rGroup->fileVerboseLimit) + if (zh->records_processed < zh->m_file_verbose_limit) logf (LOG_LOG, "update %s %s " PRINTF_OFF_T, - rGroup->recordType, fname, recordOffset); + zh->m_record_type, fname, recordOffset); extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys); zh->records_updated++; @@ -646,7 +724,7 @@ static int recordExtract (ZebraHandle zh, /* update file type */ xfree (rec->info[recInfo_fileType]); rec->info[recInfo_fileType] = - rec_strdup (rGroup->recordType, &rec->size[recInfo_fileType]); + rec_strdup (zh->m_record_type, &rec->size[recInfo_fileType]); /* update filename */ xfree (rec->info[recInfo_filename]); @@ -655,7 +733,7 @@ static int recordExtract (ZebraHandle zh, /* update delete keys */ xfree (rec->info[recInfo_delKeys]); - if (zh->reg->keys.buf_used > 0 && rGroup->flagStoreKeys == 1) + if (zh->reg->keys.buf_used > 0 && zh->m_store_keys == 1) { rec->size[recInfo_delKeys] = zh->reg->keys.buf_used; rec->info[recInfo_delKeys] = zh->reg->keys.buf; @@ -691,7 +769,7 @@ static int recordExtract (ZebraHandle zh, /* update store data */ xfree (rec->info[recInfo_storeData]); - if (rGroup->flagStoreData == 1) + if (zh->m_store_data) { rec->size[recInfo_storeData] = recordAttr->recordSize; rec->info[recInfo_storeData] = (char *) @@ -718,7 +796,7 @@ static int recordExtract (ZebraHandle zh, /* update database name */ xfree (rec->info[recInfo_databaseName]); rec->info[recInfo_databaseName] = - rec_strdup (rGroup->databaseName, &rec->size[recInfo_databaseName]); + rec_strdup (zh->basenames[0], &rec->size[recInfo_databaseName]); /* update offset */ recordAttr->recordOffset = recordOffset; @@ -730,26 +808,20 @@ static int recordExtract (ZebraHandle zh, } int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname, - const struct recordGroup *rGroupP, int deleteFlag) + int deleteFlag) { int r, i, fd; char gprefix[128]; char ext[128]; char ext_res[128]; - char subType[128]; - RecType recType; - struct recordGroup rGroupM; - struct recordGroup *rGroup = &rGroupM; struct file_read_info *fi; - void *clientData; + const char *original_record_type = 0; - memcpy (rGroup, rGroupP, sizeof(*rGroupP)); - - if (!rGroup->groupName || !*rGroup->groupName) + if (!zh->m_group || !*zh->m_group) *gprefix = '\0'; else - sprintf (gprefix, "%s.", rGroup->groupName); - + sprintf (gprefix, "%s.", zh->m_group); + logf (LOG_DEBUG, "fileExtract %s", fname); /* determine file extension */ @@ -763,98 +835,24 @@ int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname, break; } /* determine file type - depending on extension */ - if (!rGroup->recordType) + original_record_type = zh->m_record_type; + if (!zh->m_record_type) { sprintf (ext_res, "%srecordType.%s", gprefix, ext); - if (!(rGroup->recordType = res_get (zh->res, ext_res))) - { - sprintf (ext_res, "%srecordType", gprefix); - rGroup->recordType = res_get (zh->res, ext_res); - } + zh->m_record_type = res_get (zh->res, ext_res); } - if (!rGroup->recordType) + if (!zh->m_record_type) { - if (zh->records_processed < rGroup->fileVerboseLimit) + if (zh->records_processed < zh->m_file_verbose_limit) logf (LOG_LOG, "? %s", fname); return 0; } - if (!*rGroup->recordType) - return 0; - if (!(recType = - recType_byName (zh->reg->recTypes, rGroup->recordType, subType, - &clientData))) - { - logf (LOG_WARN, "No such record type: %s", rGroup->recordType); - return 0; - } - /* determine match criteria */ - if (!rGroup->recordId) + if (!zh->m_record_id) { sprintf (ext_res, "%srecordId.%s", gprefix, ext); - rGroup->recordId = res_get (zh->res, ext_res); - } - - /* determine database name */ - if (!rGroup->databaseName) - { - sprintf (ext_res, "%sdatabase.%s", gprefix, ext); - if (!(rGroup->databaseName = res_get (zh->res, ext_res))) - { - sprintf (ext_res, "%sdatabase", gprefix); - rGroup->databaseName = res_get (zh->res, ext_res); - } - } - if (!rGroup->databaseName) - rGroup->databaseName = "Default"; - - /* determine if explain database */ - - sprintf (ext_res, "%sexplainDatabase", gprefix); - rGroup->explainDatabase = - atoi (res_get_def (zh->res, ext_res, "0")); - - /* announce database */ - if (zebraExplain_curDatabase (zh->reg->zei, rGroup->databaseName)) - { - if (zebraExplain_newDatabase (zh->reg->zei, rGroup->databaseName, - rGroup->explainDatabase)) - return 0; - } - - if (rGroup->flagStoreData == -1) - { - const char *sval; - sprintf (ext_res, "%sstoreData.%s", gprefix, ext); - if (!(sval = res_get (zh->res, ext_res))) - { - sprintf (ext_res, "%sstoreData", gprefix); - sval = res_get (zh->res, ext_res); - } - if (sval) - rGroup->flagStoreData = atoi (sval); - } - if (rGroup->flagStoreData == -1) - rGroup->flagStoreData = 0; - - if (rGroup->flagStoreKeys == -1) - { - const char *sval; - - sprintf (ext_res, "%sstoreKeys.%s", gprefix, ext); - sval = res_get (zh->res, ext_res); - if (!sval) - { - sprintf (ext_res, "%sstoreKeys", gprefix); - sval = res_get (zh->res, ext_res); - } - if (!sval) - sval = res_get (zh->res, "storeKeys"); - if (sval) - rGroup->flagStoreKeys = atoi (sval); + zh->m_record_id = res_get (zh->res, ext_res); } - if (rGroup->flagStoreKeys == -1) - rGroup->flagStoreKeys = 0; if (sysno && deleteFlag) fd = -1; @@ -875,6 +873,7 @@ int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname, if ((fd = open (full_rep, O_BINARY|O_RDONLY)) == -1) { logf (LOG_WARN|LOG_ERRNO, "open %s", full_rep); + zh->m_record_type = original_record_type; return 0; } } @@ -882,74 +881,46 @@ int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname, do { file_begin (fi); - r = recordExtract (zh, sysno, fname, rGroup, deleteFlag, fi, - recType, subType, clientData, 1); + r = file_extract_record (zh, sysno, fname, deleteFlag, fi, 1); } while (r && !sysno && fi->file_more); file_read_stop (fi); if (fd != -1) close (fd); + zh->m_record_type = original_record_type; return r; } -int extract_rec_in_mem (ZebraHandle zh, const char *recordType, - const char *buf, size_t buf_size, - const char *databaseName, int delete_flag, - int test_mode, int *sysno, - int store_keys, int store_data, - const char *match_criteria) -{ - struct recordGroup rGroup; - rGroup.groupName = NULL; - rGroup.databaseName = (char *)databaseName; - rGroup.path = NULL; - rGroup.recordId = NULL; - rGroup.recordType = (char *)recordType; - rGroup.flagStoreData = store_data; - rGroup.flagStoreKeys = store_keys; - rGroup.flagRw = 1; - rGroup.databaseNamePath = 0; - rGroup.explainDatabase = 0; - rGroup.fileVerboseLimit = 100000; - rGroup.followLinks = -1; - return (bufferExtractRecord (zh, - buf, buf_size, - &rGroup, - delete_flag, - test_mode, - recordType, - sysno, - match_criteria, - "", - 0,1)); -} + /* If sysno is provided, then it's used to identify the reocord. If not, and match_criteria is provided, then sysno is guessed If not, and a record is provided, then sysno is got from there */ -int bufferExtractRecord (ZebraHandle zh, - const char *buf, size_t buf_size, - struct recordGroup *rGroup, - int delete_flag, - int test_mode, - const char *recordType, - int *sysno, - const char *match_criteria, - const char *fname, - int force_update, - int allow_update) - +int buffer_extract_record (ZebraHandle zh, + const char *buf, size_t buf_size, + int delete_flag, + int test_mode, + const char *recordType, + SYSNO *sysno, + const char *match_criteria, + const char *fname, + int force_update, + int allow_update) { RecordAttr *recordAttr; struct recExtractCtrl extractCtrl; int i, r; - char *matchStr = 0; + const char *matchStr = 0; RecType recType = NULL; char subType[1024]; void *clientData; Record rec; long recordOffset = 0; struct zebra_fetch_control fc; + const char *pr_fname = fname; /* filename to print .. */ + + if (!pr_fname) + pr_fname = ""; /* make it printable if file is omitted */ fc.fd = -1; fc.record_int_buf = buf; @@ -965,42 +936,34 @@ int bufferExtractRecord (ZebraHandle zh, extractCtrl.endf = zebra_record_int_end; extractCtrl.fh = &fc; - zh->reg->keys.buf_used = 0; - zh->reg->keys.prevAttrUse = -1; - zh->reg->keys.prevAttrSet = -1; - zh->reg->keys.prevSeqNo = 0; - zh->reg->sortKeys.buf_used = 0; + create_rec_keys_codec(&zh->reg->keys); - /* announce database */ + zh->reg->sortKeys.buf_used = 0; - if (!(rGroup->databaseName)) { - logf (LOG_WARN, "Invalid record group, no database name given"); - return 0; - } - - if (zebraExplain_curDatabase (zh->reg->zei, rGroup->databaseName)) + if (zebraExplain_curDatabase (zh->reg->zei, zh->basenames[0])) { - if (zebraExplain_newDatabase (zh->reg->zei, rGroup->databaseName, 0)) + if (zebraExplain_newDatabase (zh->reg->zei, zh->basenames[0], + zh->m_explain_database)) return 0; } - if (*recordType) { + if (recordType && *recordType) { logf (LOG_DEBUG, "Record type explicitly specified: %s", recordType); recType = recType_byName (zh->reg->recTypes, recordType, subType, &clientData); } else { - if (!(rGroup->recordType)) { + if (!(zh->m_record_type)) { logf (LOG_WARN, "No such record type defined"); return 0; } - logf (LOG_DEBUG, "Get record type from rgroup: %s",rGroup->recordType); - recType = recType_byName (zh->reg->recTypes, rGroup->recordType, subType, + logf (LOG_DEBUG, "Get record type from rgroup: %s",zh->m_record_type); + recType = recType_byName (zh->reg->recTypes, zh->m_record_type, subType, &clientData); - recordType = rGroup->recordType; + recordType = zh->m_record_type; } if (!recType) { - logf (LOG_WARN, "No such record type: %s", rGroup->recordType); + logf (LOG_WARN, "No such record type: %s", zh->m_record_type); return 0; } @@ -1049,23 +1012,28 @@ int bufferExtractRecord (ZebraHandle zh, /* match criteria */ matchStr = NULL; - if (! *sysno && match_criteria) { + if (! *sysno) { char *rinfo; - if (*match_criteria) { - matchStr = (char *)match_criteria; + if (match_criteria && *match_criteria) { + matchStr = match_criteria; } else { - if (rGroup->recordId && *rGroup->recordId) { - matchStr = fileMatchStr (zh, &zh->reg->keys, rGroup, fname, - rGroup->recordId); + if (zh->m_record_id && *zh->m_record_id) { + matchStr = fileMatchStr (zh, &zh->reg->keys, pr_fname, + zh->m_record_id); + if (!matchStr) + { + logf (LOG_WARN, "Bad match criteria (recordID)"); + return 1; + } } } if (matchStr) { rinfo = dict_lookup (zh->reg->matchDict, matchStr); if (rinfo) + { + assert(*rinfo == sizeof(*sysno)); memcpy (sysno, rinfo+1, sizeof(*sysno)); - } else { - logf (LOG_WARN, "Bad match criteria (recordID)"); - return 0; + } } } @@ -1075,11 +1043,11 @@ int bufferExtractRecord (ZebraHandle zh, if (delete_flag) { logf (LOG_LOG, "delete %s %s %ld", recordType, - fname, (long) recordOffset); + pr_fname, (long) recordOffset); logf (LOG_WARN, "cannot delete record above (seems new)"); return 1; } - logf (LOG_LOG, "add %s %s %ld", recordType, fname, + logf (LOG_LOG, "add %s %s %ld", recordType, pr_fname, (long) recordOffset); rec = rec_new (zh->reg->records); @@ -1105,7 +1073,7 @@ int bufferExtractRecord (ZebraHandle zh, if (!allow_update) { logf (LOG_LOG, "skipped %s %s %ld", - recordType, fname, (long) recordOffset); + recordType, pr_fname, (long) recordOffset); logRecord(zh); return -1; } @@ -1114,17 +1082,17 @@ int bufferExtractRecord (ZebraHandle zh, assert (rec); recordAttr = rec_init_attr (zh->reg->zei, rec); - + if (!force_update) { - if (recordAttr->runNumber == - zebraExplain_runNumberIncrement (zh->reg->zei, 0)) + if (recordAttr->runNumber == + zebraExplain_runNumberIncrement (zh->reg->zei, 0)) { - logf (LOG_LOG, "skipped %s %s %ld", recordType, - fname, (long) recordOffset); - extract_flushSortKeys (zh, *sysno, -1, &zh->reg->sortKeys); - rec_rm (&rec); - logRecord(zh); - return 1; + logf (LOG_LOG, "skipped %s %s %ld", recordType, + pr_fname, (long) recordOffset); + extract_flushSortKeys (zh, *sysno, -1, &zh->reg->sortKeys); + rec_rm (&rec); + logRecord(zh); + return -1; } } @@ -1142,13 +1110,13 @@ int bufferExtractRecord (ZebraHandle zh, if (!delkeys.buf_used) { logf (LOG_LOG, "delete %s %s %ld", recordType, - fname, (long) recordOffset); + pr_fname, (long) recordOffset); logf (LOG_WARN, "cannot delete file above, storeKeys false"); } else { logf (LOG_LOG, "delete %s %s %ld", recordType, - fname, (long) recordOffset); + pr_fname, (long) recordOffset); zh->records_deleted++; if (matchStr) dict_delete (zh->reg->matchDict, matchStr); @@ -1156,7 +1124,7 @@ int bufferExtractRecord (ZebraHandle zh, } rec_rm (&rec); logRecord(zh); - return 1; + return 0; } else { @@ -1164,13 +1132,13 @@ int bufferExtractRecord (ZebraHandle zh, if (!delkeys.buf_used) { logf (LOG_LOG, "update %s %s %ld", recordType, - fname, (long) recordOffset); + pr_fname, (long) recordOffset); logf (LOG_WARN, "cannot update file above, storeKeys false"); } else { logf (LOG_LOG, "update %s %s %ld", recordType, - fname, (long) recordOffset); + pr_fname, (long) recordOffset); extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys); zh->records_updated++; @@ -1189,7 +1157,7 @@ int bufferExtractRecord (ZebraHandle zh, /* update delete keys */ xfree (rec->info[recInfo_delKeys]); - if (zh->reg->keys.buf_used > 0 && rGroup->flagStoreKeys == 1) + if (zh->reg->keys.buf_used > 0 && zh->m_store_keys == 1) { rec->size[recInfo_delKeys] = zh->reg->keys.buf_used; rec->info[recInfo_delKeys] = zh->reg->keys.buf; @@ -1229,28 +1197,12 @@ int bufferExtractRecord (ZebraHandle zh, /* update store data */ xfree (rec->info[recInfo_storeData]); - if (rGroup->flagStoreData == 1) + if (zh->m_store_data) { rec->size[recInfo_storeData] = recordAttr->recordSize; rec->info[recInfo_storeData] = (char *) xmalloc (recordAttr->recordSize); -#if 1 memcpy (rec->info[recInfo_storeData], buf, recordAttr->recordSize); -#else - if (lseek (fi->fd, recordOffset, SEEK_SET) < 0) - { - logf (LOG_ERRNO|LOG_FATAL, "seek to %ld in %s", - (long) recordOffset, fname); - exit (1); - } - if (read (fi->fd, rec->info[recInfo_storeData], recordAttr->recordSize) - < recordAttr->recordSize) - { - logf (LOG_ERRNO|LOG_FATAL, "read %d bytes of %s", - recordAttr->recordSize, fname); - exit (1); - } -#endif } else { @@ -1260,7 +1212,7 @@ int bufferExtractRecord (ZebraHandle zh, /* update database name */ xfree (rec->info[recInfo_databaseName]); rec->info[recInfo_databaseName] = - rec_strdup (rGroup->databaseName, &rec->size[recInfo_databaseName]); + rec_strdup (zh->basenames[0], &rec->size[recInfo_databaseName]); /* update offset */ recordAttr->recordOffset = recordOffset; @@ -1271,8 +1223,6 @@ int bufferExtractRecord (ZebraHandle zh, return 0; } - - int explain_extract (void *handle, Record rec, data1_node *n) { ZebraHandle zh = (ZebraHandle) handle; @@ -1288,10 +1238,8 @@ int explain_extract (void *handle, Record rec, data1_node *n) abort (); } - zh->reg->keys.buf_used = 0; - zh->reg->keys.prevAttrUse = -1; - zh->reg->keys.prevAttrSet = -1; - zh->reg->keys.prevSeqNo = 0; + create_rec_keys_codec(&zh->reg->keys); + zh->reg->sortKeys.buf_used = 0; extractCtrl.init = extract_init; @@ -1303,8 +1251,9 @@ int explain_extract (void *handle, Record rec, data1_node *n) extractCtrl.zebra_maps = zh->reg->zebra_maps; extractCtrl.flagShowRecords = 0; extractCtrl.handle = handle; - - grs_extract_tree(&extractCtrl, n); + + if (n) + grs_extract_tree(&extractCtrl, n); if (rec->size[recInfo_delKeys]) { @@ -1341,12 +1290,16 @@ int explain_extract (void *handle, Record rec, data1_node *n) void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, int cmd, struct recKeys *reckeys) { +#if IT_KEY_NEW + void *decode_handle = iscz1_start(); +#else + int seqno = 0; #if SU_SCHEME #else unsigned char attrSet = (unsigned char) -1; unsigned short attrUse = (unsigned short) -1; #endif - int seqno = 0; +#endif int off = 0; int ch = 0; ZebraExplainInfo zei = zh->reg->zei; @@ -1368,6 +1321,52 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, zh->reg->key_file_no = 0; } zebraExplain_recordCountIncrement (zei, cmd ? 1 : -1); +#if IT_KEY_NEW + while (off < reckeys->buf_used) + { + const char *src = reckeys->buf + off; + struct it_key key; + char *dst = (char*) &key; + int attrSet, attrUse; + + iscz1_decode(decode_handle, &dst, &src); + assert(key.len < 4 && key.len > 2); + + attrSet = (int) key.mem[0]; + attrUse = (int) key.mem[1]; /* sequence in mem[2] */ + + if (zh->reg->key_buf_used + 1024 > + (zh->reg->ptr_top -zh->reg->ptr_i)*sizeof(char*)) + extract_flushWriteKeys (zh,0); + ++(zh->reg->ptr_i); + assert(zh->reg->ptr_i > 0); + (zh->reg->key_buf)[zh->reg->ptr_top - zh->reg->ptr_i] = + (char*)zh->reg->key_buf + zh->reg->key_buf_used; + + ch = zebraExplain_lookupSU (zei, attrSet, attrUse); + if (ch < 0) + ch = zebraExplain_addSU (zei, attrSet, attrUse); + + assert (ch > 0); + zh->reg->key_buf_used += + key_SU_encode (ch,((char*)zh->reg->key_buf) + + zh->reg->key_buf_used); + while (*src) + ((char*)zh->reg->key_buf) [(zh->reg->key_buf_used)++] = *src++; + src++; + ((char*)(zh->reg->key_buf))[(zh->reg->key_buf_used)++] = '\0'; + ((char*)(zh->reg->key_buf))[(zh->reg->key_buf_used)++] = cmd; + + key.len = 2; + key.mem[0] = sysno; + key.mem[1] = key.mem[2]; /* sequence .. */ + + memcpy ((char*)zh->reg->key_buf + zh->reg->key_buf_used, + &key, sizeof(key)); + (zh->reg->key_buf_used) += sizeof(key); + off = src - reckeys->buf; + } +#else while (off < reckeys->buf_used) { const char *src = reckeys->buf + off; @@ -1396,8 +1395,10 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, #endif if (zh->reg->key_buf_used + 1024 > (zh->reg->ptr_top -zh->reg->ptr_i)*sizeof(char*)) - extract_flushWriteKeys (zh); + extract_flushWriteKeys (zh,0); + assert(zh->reg->ptr_i >= 0); ++(zh->reg->ptr_i); + assert(zh->reg->ptr_i > 0); (zh->reg->key_buf)[zh->reg->ptr_top - zh->reg->ptr_i] = (char*)zh->reg->key_buf + zh->reg->key_buf_used; #if SU_SCHEME @@ -1430,27 +1431,73 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, (zh->reg->key_buf_used) += sizeof(key); off = src - reckeys->buf; } +#endif assert (off == reckeys->buf_used); +#if IT_KEY_NEW + iscz1_stop(decode_handle); +#endif } -void extract_flushWriteKeys (ZebraHandle zh) +void extract_flushWriteKeys (ZebraHandle zh, int final) + /* optimizing: if final=1, and no files written yet */ + /* push the keys directly to merge, sidestepping the */ + /* temp file altogether. Speeds small updates */ { FILE *outf; char out_fname[200]; char *prevcp, *cp; struct encode_info encode_info; int ptr_i = zh->reg->ptr_i; + int temp_policy; #if SORT_EXTRA int i; #endif if (!zh->reg->key_buf || ptr_i <= 0) + { + logf (LOG_DEBUG, " nothing to flush section=%d buf=%p i=%d", + zh->reg->key_file_no, zh->reg->key_buf, ptr_i); + logf (LOG_DEBUG, " buf=%p ", + zh->reg->key_buf); + logf (LOG_DEBUG, " ptr=%d ",zh->reg->ptr_i); + logf (LOG_DEBUG, " reg=%p ",zh->reg); + return; + } (zh->reg->key_file_no)++; logf (LOG_LOG, "sorting section %d", (zh->reg->key_file_no)); + logf (LOG_DEBUG, " sort_buff at %p n=%d", + zh->reg->key_buf + zh->reg->ptr_top - ptr_i,ptr_i); #if !SORT_EXTRA qsort (zh->reg->key_buf + zh->reg->ptr_top - ptr_i, ptr_i, sizeof(char*), key_qsort_compare); + + /* zebra.cfg: tempfiles: + Y: always use temp files (old way) + A: use temp files, if more than one (auto) + = if this is both the last and the first + N: never bother with temp files (new) */ + + temp_policy=toupper(res_get_def(zh->res,"tempfiles","auto")[0]); + if (temp_policy != 'Y' && temp_policy != 'N' && temp_policy != 'A') { + logf (LOG_WARN, "Illegal tempfiles setting '%c'. using 'Auto' ", + temp_policy); + temp_policy='A'; + } + + if ( ( temp_policy =='N' ) || /* always from memory */ + ( ( temp_policy =='A' ) && /* automatic */ + (zh->reg->key_file_no == 1) && /* this is first time */ + (final) ) ) /* and last (=only) time */ + { /* go directly from memory */ + zh->reg->key_file_no =0; /* signal not to read files */ + zebra_index_merge(zh); + zh->reg->ptr_i = 0; + zh->reg->key_buf_used = 0; + return; + } + + /* Not doing directly from memory, write into a temp file */ extract_get_fname_tmp (zh, out_fname, zh->reg->key_file_no); if (!(outf = fopen (out_fname, "wb"))) @@ -1525,18 +1572,22 @@ void extract_flushWriteKeys (ZebraHandle zh) zh->reg->key_buf_used = 0; } -void extract_add_index_string (RecWord *p, const char *string, - int length) +void extract_add_index_string (RecWord *p, const char *str, int length) { char *dst; + ZebraHandle zh = p->extractCtrl->handle; + struct recKeys *keys = &zh->reg->keys; +#if IT_KEY_NEW + struct it_key key; + const char *src = (char*) &key; +#else unsigned char attrSet; unsigned short attrUse; int lead = 0; int diff = 0; int *pseqno = &p->seqno; - ZebraHandle zh = p->extractCtrl->handle; ZebraExplainInfo zei = zh->reg->zei; - struct recKeys *keys = &zh->reg->keys; +#endif if (keys->buf_used+1024 > keys->buf_max) { @@ -1550,6 +1601,29 @@ void extract_add_index_string (RecWord *p, const char *string, } dst = keys->buf + keys->buf_used; +#if IT_KEY_NEW + key.len = 3; + key.mem[0] = p->attrSet; + key.mem[1] = p->attrUse; + key.mem[2] = p->seqno; + +#if 0 + /* just for debugging .. */ + yaz_log(LOG_LOG, "set=%d use=%d seqno=%d", p->attrSet, p->attrUse, + p->seqno); +#endif + + iscz1_encode(keys->codec_handle, &dst, &src); + + *dst++ = p->reg_type; + memcpy (dst, str, length); + dst += length; + *dst++ = '\0'; +#else + /* leader byte is encoded as follows: + bit 0 : 1 if attrset is unchanged; 0 if attrset is changed + bit 1 : 1 if attruse is unchanged; 0 if attruse is changed + */ attrSet = p->attrSet; if (keys->buf_used > 0 && keys->prevAttrSet == attrSet) lead |= 1; @@ -1560,13 +1634,13 @@ void extract_add_index_string (RecWord *p, const char *string, lead |= 2; else keys->prevAttrUse = attrUse; -#if 1 + diff = 1 + *pseqno - keys->prevSeqNo; if (diff >= 1 && diff <= 15) lead |= (diff << 2); else diff = 0; -#endif + keys->prevSeqNo = *pseqno; *dst++ = lead; @@ -1598,7 +1672,7 @@ void extract_add_index_string (RecWord *p, const char *string, } #endif *dst++ = p->reg_type; - memcpy (dst, string, length); + memcpy (dst, str, length); dst += length; *dst++ = '\0'; @@ -1607,10 +1681,11 @@ void extract_add_index_string (RecWord *p, const char *string, memcpy (dst, pseqno, sizeof(*pseqno)); dst += sizeof(*pseqno); } +#endif keys->buf_used = dst - keys->buf; } -static void extract_add_sort_string (RecWord *p, const char *string, +static void extract_add_sort_string (RecWord *p, const char *str, int length) { ZebraHandle zh = p->extractCtrl->handle; @@ -1643,7 +1718,7 @@ static void extract_add_sort_string (RecWord *p, const char *string, off += key_SU_encode(p->attrSet, sk->buf + off); off += key_SU_encode(p->attrUse, sk->buf + off); off += key_SU_encode(length, sk->buf + off); - memcpy (sk->buf + off, string, length); + memcpy (sk->buf + off, str, length); sk->buf_used = off + length; } @@ -1663,7 +1738,7 @@ static void extract_add_incomplete_field (RecWord *p) const char **map = 0; if (remain > 0) - map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain); + map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain, 0); while (map) { @@ -1675,7 +1750,7 @@ static void extract_add_incomplete_field (RecWord *p) { remain = p->length - (b - p->string); if (remain > 0) - map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain); + map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain, 0); else map = 0; } @@ -1690,7 +1765,7 @@ static void extract_add_incomplete_field (RecWord *p) buf[i++] = *(cp++); remain = p->length - (b - p->string); if (remain > 0) - map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain); + map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain, 0); else map = 0; } @@ -1707,17 +1782,24 @@ static void extract_add_complete_field (RecWord *p) char buf[IT_MAX_WORD+1]; const char **map = 0; int i = 0, remain = p->length; + int first; /* first position */ + +yaz_log(LOG_DEBUG, "Complete field, w='%s'", p->string); if (remain > 0) - map = zebra_maps_input (p->zebra_maps, p->reg_type, &b, remain); + map = zebra_maps_input (p->zebra_maps, p->reg_type, &b, remain, 1); while (remain > 0 && i < IT_MAX_WORD) { while (map && *map && **map == *CHR_SPACE) { remain = p->length - (b - p->string); + if (remain > 0) - map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain); + { + first = i ? 0 : 1; + map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain, first); + } else map = 0; } @@ -1730,14 +1812,24 @@ static void extract_add_complete_field (RecWord *p) { const char *cp = *map; - if (i >= IT_MAX_WORD) - break; - while (i < IT_MAX_WORD && *cp) - buf[i++] = *(cp++); + if (**map == *CHR_CUT) + { + i = 0; + } + else + { + if (i >= IT_MAX_WORD) + break; +yaz_log(LOG_DEBUG, "Adding string to index '%d'", *map); + while (i < IT_MAX_WORD && *cp) + buf[i++] = *(cp++); + } remain = p->length - (b - p->string); if (remain > 0) + { map = zebra_maps_input (p->zebra_maps, p->reg_type, &b, - remain); + remain, 0); + } else map = 0; } @@ -1750,9 +1842,9 @@ static void extract_add_complete_field (RecWord *p) void extract_token_add (RecWord *p) { WRBUF wrbuf; - #if 0 - yaz_log (LOG_LOG, "reg_type=%c attrSet=%d attrUse=%d seqno=%d s=%.*s", + yaz_log (LOG_LOG, "token_add " + "reg_type=%c attrSet=%d attrUse=%d seqno=%d s=%.*s", p->reg_type, p->attrSet, p->attrUse, p->seqno, p->length, p->string); #endif @@ -1808,8 +1900,13 @@ void encode_key_init (struct encode_info *i) i->prevseq=0; i->prevcmd=-1; i->keylen=0; +#if IT_KEY_NEW + i->encode_handle = iscz1_start(); +#endif } +#if IT_KEY_NEW +#else char *encode_key_int (int d, char *bp) { if (d <= 63) @@ -1834,6 +1931,8 @@ char *encode_key_int (int d, char *bp) } return bp; } +#endif + #define OLDENCODE 1 #ifdef OLDENCODE @@ -1844,11 +1943,19 @@ char *encode_key_int (int d, char *bp) void encode_key_write (char *k, struct encode_info *i, FILE *outf) { struct it_key key; - char *bp = i->buf; + char *bp = i->buf, *bp0; + const char *src = (char *) &key; + /* copy term to output buf */ while ((*bp++ = *k++)) ; - memcpy (&key, k+1, sizeof(struct it_key)); + /* and copy & align key so we can mangle */ + memcpy (&key, k+1, sizeof(struct it_key)); /* *k is insert/delete */ +#if IT_KEY_NEW + bp0 = bp++; + iscz1_encode(i->encode_handle, &bp, &src); + *bp0 = (*k * 128) + bp - bp0 - 1; /* length and insert/delete combined */ +#else bp = encode_key_int ( (key.sysno - i->sysno) * 2 + *k, bp); if (i->sysno != key.sysno) { @@ -1860,6 +1967,7 @@ void encode_key_write (char *k, struct encode_info *i, FILE *outf) bp = encode_key_int (key.seqno - i->seqno, bp); i->seqno = key.seqno; i->cmd = *k; +#endif if (fwrite (i->buf, bp - i->buf, 1, outf) != 1) { logf (LOG_FATAL|LOG_ERRNO, "fwrite"); @@ -1869,6 +1977,9 @@ void encode_key_write (char *k, struct encode_info *i, FILE *outf) void encode_key_flush (struct encode_info *i, FILE *outf) { /* dummy routine */ +#if IT_KEY_NEW + iscz1_stop(i->encode_handle); +#endif } #else