X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fextract.c;h=73fe1a1b46065f9d615abe4233a72f0f57e82c38;hb=ee4ada9656faa612678aee238151a8b6652ad549;hp=040aff0f2a4ebf0a28d7e1dbc71ce8ed0f2188a1;hpb=98e21f950e6d0babe88a6a1ccb03b90f0b3a6b1a;p=idzebra-moved-to-github.git diff --git a/index/extract.c b/index/extract.c index 040aff0..73fe1a1 100644 --- a/index/extract.c +++ b/index/extract.c @@ -1,10 +1,26 @@ -/* - * Copyright (C) 1994-2002, Index Data - * All rights reserved. - * Sebastian Hammer, Adam Dickmeiss - * - * $Id: extract.c,v 1.115 2002-04-04 20:50:36 adam Exp $ - */ +/* $Id: extract.c,v 1.123 2002-08-29 10:00:15 adam Exp $ + Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002 + Index Data Aps + +This file is part of the Zebra server. + +Zebra is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Zebra is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with Zebra; see the file LICENSE.zebra. If not, write to the +Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA +02111-1307, USA. +*/ + + #include #include #ifdef WIN32 @@ -24,27 +40,48 @@ #define PRINTF_OFF_T "%ld" #endif -static int records_inserted = 0; -static int records_updated = 0; -static int records_deleted = 0; -static int records_processed = 0; +#define USE_SHELLSORT 0 -static void logRecord (int showFlag) +#if USE_SHELLSORT +static void shellsort(void *ar, int r, size_t s, + int (*cmp)(const void *a, const void *b)) { - if (!showFlag) - ++records_processed; - if (showFlag || !(records_processed % 1000)) + char *a = ar; + char v[100]; + int h, i, j, k; + static const int incs[16] = { 1391376, 463792, 198768, 86961, 33936, + 13776, 4592, 1968, 861, 336, + 112, 48, 21, 7, 3, 1 }; + for ( k = 0; k < 16; k++) + for (h = incs[k], i = h; i < r; i++) + { + memcpy (v, a+s*i, s); + j = i; + while (j > h && (*cmp)(a + s*(j-h), v) > 0) + { + memcpy (a + s*j, a + s*(j-h), s); + j -= h; + } + memcpy (a+s*j, v, s); + } +} +#endif + +static void logRecord (ZebraHandle zh) +{ + ++zh->records_processed; + if (!(zh->records_processed % 1000)) { logf (LOG_LOG, "Records: %7d i/u/d %d/%d/%d", - records_processed, records_inserted, records_updated, - records_deleted); + zh->records_processed, zh->records_inserted, zh->records_updated, + zh->records_deleted); } } static void extract_init (struct recExtractCtrl *p, RecWord *w) { w->zebra_maps = p->zebra_maps; - w->seqnos = p->seqno; + w->seqno = 1; w->attrSet = VAL_BIB1; w->attrUse = 1016; w->reg_type = 'w'; @@ -377,19 +414,6 @@ struct recordLogInfo { struct recordGroup *rGroup; }; -static void recordLogPreamble (int level, const char *msg, void *info) -{ - struct recordLogInfo *p = (struct recordLogInfo *) info; - FILE *outf = yaz_log_file (); - - if (level & LOG_LOG) - return ; - fprintf (outf, "File %s, offset %d, type %s\n", - p->rGroup->recordType, p->recordOffset, p->fname); - log_event_start (NULL, NULL); -} - - static int recordExtract (ZebraHandle zh, SYSNO *sysno, const char *fname, struct recordGroup *rGroup, int deleteFlag, @@ -401,7 +425,6 @@ static int recordExtract (ZebraHandle zh, char *matchStr; SYSNO sysnotmp; Record rec; - struct recordLogInfo logInfo; off_t recordOffset = 0; if (fi->fd != -1) @@ -442,29 +465,41 @@ static int recordExtract (ZebraHandle zh, if (!rGroup->flagRw) printf ("File: %s " PRINTF_OFF_T "\n", fname, recordOffset); - - logInfo.fname = fname; - logInfo.recordOffset = recordOffset; - logInfo.rGroup = rGroup; - log_event_start (recordLogPreamble, &logInfo); + if (rGroup->flagRw) + { + char msg[512]; + sprintf (msg, "%s:" PRINTF_OFF_T , fname, recordOffset); + yaz_log_init_prefix2 (msg); + } r = (*recType->extract)(clientData, &extractCtrl); - log_event_start (NULL, NULL); - + yaz_log_init_prefix2 (0); if (r == RECCTRL_EXTRACT_EOF) return 0; - else if (r == RECCTRL_EXTRACT_ERROR) + else if (r == RECCTRL_EXTRACT_ERROR_GENERIC) { /* error occured during extraction ... */ if (rGroup->flagRw && - records_processed < rGroup->fileVerboseLimit) + zh->records_processed < rGroup->fileVerboseLimit) { logf (LOG_WARN, "fail %s %s " PRINTF_OFF_T, rGroup->recordType, fname, recordOffset); } return 0; } + else if (r == RECCTRL_EXTRACT_ERROR_NO_SUCH_FILTER) + { + /* error occured during extraction ... */ + if (rGroup->flagRw && + zh->records_processed < rGroup->fileVerboseLimit) + { + logf (LOG_WARN, "no filter for %s %s " + PRINTF_OFF_T, rGroup->recordType, + fname, recordOffset); + } + return 0; + } if (zh->reg->keys.buf_used == 0) { /* the extraction process returned no information - the record @@ -515,7 +550,7 @@ static int recordExtract (ZebraHandle zh, logf (LOG_WARN, "cannot delete record above (seems new)"); return 1; } - if (records_processed < rGroup->fileVerboseLimit) + if (zh->records_processed < rGroup->fileVerboseLimit) logf (LOG_LOG, "add %s %s " PRINTF_OFF_T, rGroup->recordType, fname, recordOffset); rec = rec_new (zh->reg->records); @@ -531,7 +566,7 @@ static int recordExtract (ZebraHandle zh, extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys); extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); - records_inserted++; + zh->records_inserted++; } else { @@ -546,11 +581,12 @@ static int recordExtract (ZebraHandle zh, if (recordAttr->runNumber == zebraExplain_runNumberIncrement (zh->reg->zei, 0)) { - logf (LOG_LOG, "skipped %s %s " PRINTF_OFF_T, rGroup->recordType, - fname, recordOffset); + yaz_log (LOG_LOG, "run number = %d", recordAttr->runNumber); + yaz_log (LOG_LOG, "skipped %s %s " PRINTF_OFF_T, + rGroup->recordType, fname, recordOffset); extract_flushSortKeys (zh, *sysno, -1, &zh->reg->sortKeys); rec_rm (&rec); - logRecord (0); + logRecord (zh); return 1; } delkeys.buf_used = rec->size[recInfo_delKeys]; @@ -568,16 +604,16 @@ static int recordExtract (ZebraHandle zh, } else { - if (records_processed < rGroup->fileVerboseLimit) + if (zh->records_processed < rGroup->fileVerboseLimit) logf (LOG_LOG, "delete %s %s " PRINTF_OFF_T, rGroup->recordType, fname, recordOffset); - records_deleted++; + zh->records_deleted++; if (matchStr) dict_delete (zh->reg->matchDict, matchStr); rec_del (zh->reg->records, &rec); } rec_rm (&rec); - logRecord (0); + logRecord (zh); return 1; } else @@ -591,11 +627,11 @@ static int recordExtract (ZebraHandle zh, } else { - if (records_processed < rGroup->fileVerboseLimit) + if (zh->records_processed < rGroup->fileVerboseLimit) logf (LOG_LOG, "update %s %s " PRINTF_OFF_T, rGroup->recordType, fname, recordOffset); extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys); - records_updated++; + zh->records_updated++; } } } @@ -680,7 +716,7 @@ static int recordExtract (ZebraHandle zh, /* commit this record */ rec_put (zh->reg->records, &rec); - logRecord (0); + logRecord (zh); return 1; } @@ -729,7 +765,7 @@ int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname, } if (!rGroup->recordType) { - if (records_processed < rGroup->fileVerboseLimit) + if (zh->records_processed < rGroup->fileVerboseLimit) logf (LOG_LOG, "? %s", fname); return 0; } @@ -920,19 +956,16 @@ int extract_rec_in_mem (ZebraHandle zh, const char *recordType, if (r == RECCTRL_EXTRACT_EOF) return 0; - else if (r == RECCTRL_EXTRACT_ERROR) + else if (r == RECCTRL_EXTRACT_ERROR_GENERIC) { /* error occured during extraction ... */ -#if 1 - yaz_log (LOG_WARN, "extract error"); -#else - if (rGroup->flagRw && - records_processed < rGroup->fileVerboseLimit) - { - logf (LOG_WARN, "fail %s %s %ld", rGroup->recordType, - fname, (long) recordOffset); - } -#endif + yaz_log (LOG_WARN, "extract error: generic"); + return 0; + } + else if (r == RECCTRL_EXTRACT_ERROR_NO_SUCH_FILTER) + { + /* error occured during extraction ... */ + yaz_log (LOG_WARN, "extract error: no such filter"); return 0; } if (zh->reg->keys.buf_used == 0) @@ -1278,8 +1311,8 @@ void extract_flushWriteKeys (ZebraHandle zh) (zh->reg->key_file_no)++; logf (LOG_LOG, "sorting section %d", (zh->reg->key_file_no)); #if !SORT_EXTRA - qsort (zh->reg->key_buf + zh->reg->ptr_top - ptr_i, ptr_i, sizeof(char*), - key_qsort_compare); + qsort (zh->reg->key_buf + zh->reg->ptr_top - ptr_i, ptr_i, + sizeof(char*), key_qsort_compare); extract_get_fname_tmp (zh, out_fname, zh->reg->key_file_no); if (!(outf = fopen (out_fname, "wb"))) @@ -1359,11 +1392,11 @@ void extract_add_index_string (RecWord *p, const char *string, unsigned short attrUse; int lead = 0; int diff = 0; - int *pseqno = &p->seqnos[p->reg_type]; + int *pseqno = &p->seqno; ZebraHandle zh = p->extractCtrl->handle; ZebraExplainInfo zei = zh->reg->zei; struct recKeys *keys = &zh->reg->keys; - + if (keys->buf_used+1024 > keys->buf_max) { char *b; @@ -1434,8 +1467,6 @@ void extract_add_index_string (RecWord *p, const char *string, dst += sizeof(*pseqno); } keys->buf_used = dst - keys->buf; - if (*pseqno) - (*pseqno)++; } static void extract_add_sort_string (RecWord *p, const char *string, @@ -1510,8 +1541,8 @@ static void extract_add_incomplete_field (RecWord *p) if (!i) return; extract_add_string (p, buf, i); + p->seqno++; } - (p->seqnos[p->reg_type])++; /* to separate this from next one */ } static void extract_add_complete_field (RecWord *p)