X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fextract.c;h=a14b7f50f4724fb781a5cfe248b8bf769ec669f0;hb=cc9f94a61cbd9dcc0df0cf7d0c7c41d2cec88189;hp=2f4bf846a6a72f8bebb39c6f3a1d049df9113de6;hpb=2b1851bd5565e3d21f9cf9a37661a584c063b75f;p=idzebra-moved-to-github.git diff --git a/index/extract.c b/index/extract.c index 2f4bf84..a14b7f5 100644 --- a/index/extract.c +++ b/index/extract.c @@ -3,7 +3,7 @@ * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * - * $Id: extract.c,v 1.114 2002-04-04 14:14:13 adam Exp $ + * $Id: extract.c,v 1.119 2002-05-07 11:05:19 adam Exp $ */ #include #include @@ -15,6 +15,7 @@ #include #include "index.h" +#include #include #if _FILE_OFFSET_BITS == 64 @@ -23,27 +24,45 @@ #define PRINTF_OFF_T "%ld" #endif -static int records_inserted = 0; -static int records_updated = 0; -static int records_deleted = 0; -static int records_processed = 0; +static void shellsort(void *ar, int r, size_t s, + int (*cmp)(const void *a, const void *b)) +{ + char *a = ar; + char v[100]; + int h, i, j, k; + static const int incs[16] = { 1391376, 463792, 198768, 86961, 33936, + 13776, 4592, 1968, 861, 336, + 112, 48, 21, 7, 3, 1 }; + for ( k = 0; k < 16; k++) + for (h = incs[k], i = h; i < r; i++) + { + memcpy (v, a+s*i, s); + j = i; + while (j > h && (*cmp)(a + s*(j-h), v) > 0) + { + memcpy (a + s*j, a + s*(j-h), s); + j -= h; + } + memcpy (a+s*j, v, s); + } +} -static void logRecord (int showFlag) + +static void logRecord (ZebraHandle zh) { - if (!showFlag) - ++records_processed; - if (showFlag || !(records_processed % 1000)) + ++zh->records_processed; + if (!(zh->records_processed % 1000)) { logf (LOG_LOG, "Records: %7d i/u/d %d/%d/%d", - records_processed, records_inserted, records_updated, - records_deleted); + zh->records_processed, zh->records_inserted, zh->records_updated, + zh->records_deleted); } } static void extract_init (struct recExtractCtrl *p, RecWord *w) { w->zebra_maps = p->zebra_maps; - w->seqnos = p->seqno; + w->seqno = 1; w->attrSet = VAL_BIB1; w->attrUse = 1016; w->reg_type = 'w'; @@ -457,7 +476,7 @@ static int recordExtract (ZebraHandle zh, { /* error occured during extraction ... */ if (rGroup->flagRw && - records_processed < rGroup->fileVerboseLimit) + zh->records_processed < rGroup->fileVerboseLimit) { logf (LOG_WARN, "fail %s %s " PRINTF_OFF_T, rGroup->recordType, fname, recordOffset); @@ -514,7 +533,7 @@ static int recordExtract (ZebraHandle zh, logf (LOG_WARN, "cannot delete record above (seems new)"); return 1; } - if (records_processed < rGroup->fileVerboseLimit) + if (zh->records_processed < rGroup->fileVerboseLimit) logf (LOG_LOG, "add %s %s " PRINTF_OFF_T, rGroup->recordType, fname, recordOffset); rec = rec_new (zh->reg->records); @@ -530,7 +549,7 @@ static int recordExtract (ZebraHandle zh, extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys); extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); - records_inserted++; + zh->records_inserted++; } else { @@ -545,11 +564,12 @@ static int recordExtract (ZebraHandle zh, if (recordAttr->runNumber == zebraExplain_runNumberIncrement (zh->reg->zei, 0)) { - logf (LOG_LOG, "skipped %s %s " PRINTF_OFF_T, rGroup->recordType, - fname, recordOffset); + yaz_log (LOG_LOG, "run number = %d", recordAttr->runNumber); + yaz_log (LOG_LOG, "skipped %s %s " PRINTF_OFF_T, + rGroup->recordType, fname, recordOffset); extract_flushSortKeys (zh, *sysno, -1, &zh->reg->sortKeys); rec_rm (&rec); - logRecord (0); + logRecord (zh); return 1; } delkeys.buf_used = rec->size[recInfo_delKeys]; @@ -567,16 +587,16 @@ static int recordExtract (ZebraHandle zh, } else { - if (records_processed < rGroup->fileVerboseLimit) + if (zh->records_processed < rGroup->fileVerboseLimit) logf (LOG_LOG, "delete %s %s " PRINTF_OFF_T, rGroup->recordType, fname, recordOffset); - records_deleted++; + zh->records_deleted++; if (matchStr) dict_delete (zh->reg->matchDict, matchStr); rec_del (zh->reg->records, &rec); } rec_rm (&rec); - logRecord (0); + logRecord (zh); return 1; } else @@ -590,11 +610,11 @@ static int recordExtract (ZebraHandle zh, } else { - if (records_processed < rGroup->fileVerboseLimit) + if (zh->records_processed < rGroup->fileVerboseLimit) logf (LOG_LOG, "update %s %s " PRINTF_OFF_T, rGroup->recordType, fname, recordOffset); extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys); - records_updated++; + zh->records_updated++; } } } @@ -679,7 +699,7 @@ static int recordExtract (ZebraHandle zh, /* commit this record */ rec_put (zh->reg->records, &rec); - logRecord (0); + logRecord (zh); return 1; } @@ -728,7 +748,7 @@ int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname, } if (!rGroup->recordType) { - if (records_processed < rGroup->fileVerboseLimit) + if (zh->records_processed < rGroup->fileVerboseLimit) logf (LOG_LOG, "? %s", fname); return 0; } @@ -814,9 +834,21 @@ int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname, fd = -1; else { - if ((fd = open (fname, O_BINARY|O_RDONLY)) == -1) + char full_rep[1024]; + + if (zh->path_reg && !yaz_is_abspath (fname)) { - logf (LOG_WARN|LOG_ERRNO, "open %s", fname); + strcpy (full_rep, zh->path_reg); + strcat (full_rep, "/"); + strcat (full_rep, fname); + } + else + strcpy (full_rep, fname); + + + if ((fd = open (full_rep, O_BINARY|O_RDONLY)) == -1) + { + logf (LOG_WARN|LOG_ERRNO, "open %s", full_rep); return 0; } } @@ -914,7 +946,7 @@ int extract_rec_in_mem (ZebraHandle zh, const char *recordType, yaz_log (LOG_WARN, "extract error"); #else if (rGroup->flagRw && - records_processed < rGroup->fileVerboseLimit) + zh->records_processed < rGroup->fileVerboseLimit) { logf (LOG_WARN, "fail %s %s %ld", rGroup->recordType, fname, (long) recordOffset); @@ -1265,8 +1297,8 @@ void extract_flushWriteKeys (ZebraHandle zh) (zh->reg->key_file_no)++; logf (LOG_LOG, "sorting section %d", (zh->reg->key_file_no)); #if !SORT_EXTRA - qsort (zh->reg->key_buf + zh->reg->ptr_top - ptr_i, ptr_i, sizeof(char*), - key_qsort_compare); + qsort (zh->reg->key_buf + zh->reg->ptr_top - ptr_i, ptr_i, + sizeof(char*), key_qsort_compare); extract_get_fname_tmp (zh, out_fname, zh->reg->key_file_no); if (!(outf = fopen (out_fname, "wb"))) @@ -1346,11 +1378,11 @@ void extract_add_index_string (RecWord *p, const char *string, unsigned short attrUse; int lead = 0; int diff = 0; - int *pseqno = &p->seqnos[p->reg_type]; + int *pseqno = &p->seqno; ZebraHandle zh = p->extractCtrl->handle; ZebraExplainInfo zei = zh->reg->zei; struct recKeys *keys = &zh->reg->keys; - + if (keys->buf_used+1024 > keys->buf_max) { char *b; @@ -1421,8 +1453,6 @@ void extract_add_index_string (RecWord *p, const char *string, dst += sizeof(*pseqno); } keys->buf_used = dst - keys->buf; - if (*pseqno) - (*pseqno)++; } static void extract_add_sort_string (RecWord *p, const char *string, @@ -1497,8 +1527,8 @@ static void extract_add_incomplete_field (RecWord *p) if (!i) return; extract_add_string (p, buf, i); + p->seqno++; } - (p->seqnos[p->reg_type])++; /* to separate this from next one */ } static void extract_add_complete_field (RecWord *p)