From: Adam Dickmeiss Date: Wed, 1 May 1996 13:46:35 +0000 (+0000) Subject: First work on multiple records in one file. X-Git-Tag: ZEBRA.1.0~475 X-Git-Url: http://git.indexdata.com/?p=idzebra-moved-to-github.git;a=commitdiff_plain;h=99e4ce02beb2ea748b9970ef35fe8834f04a1528 First work on multiple records in one file. New option, -offset, to the "unread" command in the filter module. --- diff --git a/index/Makefile b/index/Makefile index c177f9d..ab81dec 100644 --- a/index/Makefile +++ b/index/Makefile @@ -1,7 +1,7 @@ # Copyright (C) 1995, Index Data I/S # All rights reserved. # Sebastian Hammer, Adam Dickmeiss -# $Id: Makefile,v 1.32 1996-02-01 20:58:58 adam Exp $ +# $Id: Makefile,v 1.33 1996-05-01 13:46:35 adam Exp $ SHELL=/bin/sh RANLIB=ranlib @@ -9,7 +9,7 @@ RANLIB=ranlib YAZLIB=../../yaz/lib/libyaz.a #YAZLIB=-lyaz YAZINC=-I../../yaz/include -OSILIB=../../xtimosi/src/libmosi.a ../../yaz/lib/librfc.a +#OSILIB=../../xtimosi/src/libmosi.a ../../yaz/lib/librfc.a #OSILIB=../../xtimosi/src/libmosi.a -lrfc #NETLIB=-lnsl -lsocket diff --git a/index/extract.c b/index/extract.c index 45e6363..479e8bd 100644 --- a/index/extract.c +++ b/index/extract.c @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: extract.c,v $ - * Revision 1.53 1996-04-26 12:09:43 adam + * Revision 1.54 1996-05-01 13:46:35 adam + * First work on multiple records in one file. + * New option, -offset, to the "unread" command in the filter module. + * + * Revision 1.53 1996/04/26 12:09:43 adam * Added a few comments. * * Revision 1.52 1996/04/25 13:27:57 adam @@ -216,6 +220,19 @@ static int key_file_no; static int records_inserted = 0; static int records_updated = 0; static int records_deleted = 0; +static int records_processed = 0; + +static void logRecord (int showFlag) +{ + if (!showFlag) + ++records_processed; + if (showFlag || !(records_processed % 1000)) + { + logf (LOG_LOG, "Records: %7d i/u/d %d/%d/%d", + records_processed, records_inserted, records_updated, + records_deleted); + } +} void key_open (int mem) { @@ -351,9 +368,7 @@ int key_close (void) rec_close (&records); dict_close (matchDict); - logf (LOG_LOG, "Records inserted %6d", records_inserted); - logf (LOG_LOG, "Records updated %6d", records_updated); - logf (LOG_LOG, "Records deleted %6d", records_deleted); + logRecord (1); return key_file_no; } @@ -547,15 +562,12 @@ static void addRecordKeyAny (const RecWord *p) addRecordKey (p); } -#define FILE_READ_BUFSIZE 4096 struct file_read_info { - int file_noread; + off_t file_max; + off_t file_offset; + off_t file_moffset; + int file_more; int fd; -#if FILE_READ_BUFSIZE - char *file_buf; - int file_offset; - int file_bufsize; -#endif }; static struct file_read_info *file_read_start (int fd) @@ -563,82 +575,56 @@ static struct file_read_info *file_read_start (int fd) struct file_read_info *fi = xmalloc (sizeof(*fi)); fi->fd = fd; - fi->file_noread = 0; -#if FILE_READ_BUFSIZE - fi->file_offset = 0; - fi->file_buf = xmalloc (FILE_READ_BUFSIZE); - fi->file_bufsize = read (fd, fi->file_buf, FILE_READ_BUFSIZE); -#endif + fi->file_max = 0; + fi->file_moffset = 0; return fi; } static void file_read_stop (struct file_read_info *fi) { assert (fi); -#if FILE_READ_BUFSIZE - xfree (fi->file_buf); - fi->file_buf = NULL; -#endif xfree (fi); } +static off_t file_seek (void *handle, off_t offset) +{ + struct file_read_info *p = handle; + p->file_offset = offset; + return lseek (p->fd, offset, SEEK_SET); +} + static int file_read (void *handle, char *buf, size_t count) { struct file_read_info *p = handle; int fd = p->fd; -#if FILE_READ_BUFSIZE - int l = p->file_bufsize - p->file_offset; - - if (count > l) - { - int r; - if (l > 0) - memcpy (buf, p->file_buf + p->file_offset, l); - count = count-l; - if (count > FILE_READ_BUFSIZE) - { - if ((r = read (fd, buf + l, count)) == -1) - { - logf (LOG_FATAL|LOG_ERRNO, "read"); - exit (1); - } - p->file_bufsize = 0; - p->file_offset = 0; - p->file_noread += l+r; - return l+r; - } - p->file_bufsize = r = read (fd, p->file_buf, FILE_READ_BUFSIZE); - if (r == -1) - { - logf (LOG_FATAL|LOG_ERRNO, "read"); - exit (1); - } - else if (r <= count) - { - p->file_offset = r; - memcpy (buf + l, p->file_buf, r); - p->file_noread += l+r; - return l+r; - } - else - { - p->file_offset = count; - memcpy (buf + l, p->file_buf, count - l); - p->file_noread += count; - return count; - } - } - memcpy (buf, p->file_buf + p->file_offset, count); - p->file_offset += count; - p->file_noread += count; - return count; -#else int r; r = read (fd, buf, count); if (r > 0) - p->file_noread += r; + { + p->file_offset += r; + if (p->file_offset > p->file_max) + p->file_max = p->file_offset; + } return r; -#endif +} + +static void file_begin (void *handle) +{ + struct file_read_info *p = handle; + + p->file_offset = p->file_moffset; + if (p->file_moffset) + lseek (p->fd, p->file_moffset, SEEK_SET); + p->file_more = 0; +} + +static void file_end (void *handle, off_t offset) +{ + struct file_read_info *p = handle; + + assert (p->file_more == 0); + p->file_more = 1; + p->file_moffset = offset; } static int atois (const char **s) @@ -812,6 +798,7 @@ static int recordExtract (SYSNO *sysno, const char *fname, int r; char *matchStr; SYSNO sysnotmp; + off_t recordOffset = 0; Record rec; struct recordLogInfo logInfo; @@ -831,7 +818,12 @@ static int recordExtract (SYSNO *sysno, const char *fname, reckeys.buf_used = 0; reckeys.prevAttrUse = -1; reckeys.prevAttrSet = -1; + + recordOffset = fi->file_moffset; + extractCtrl.offset = recordOffset; extractCtrl.readf = file_read; + extractCtrl.seekf = file_seek; + extractCtrl.endf = file_end; r = (*recType->extract)(&extractCtrl); if (r) @@ -886,7 +878,8 @@ static int recordExtract (SYSNO *sysno, const char *fname, } logInfo.op = "add"; if (rGroup->fileVerboseFlag) - logf (LOG_LOG, "add %s %s", rGroup->recordType, fname); + logf (LOG_LOG, "add %s %s %ld", rGroup->recordType, + fname, (long) recordOffset); rec = rec_new (records); *sysno = rec->sysno; @@ -919,12 +912,14 @@ static int recordExtract (SYSNO *sysno, const char *fname, else { if (rGroup->fileVerboseFlag) - logf (LOG_LOG, "delete %s %s", rGroup->recordType, fname); + logf (LOG_LOG, "delete %s %s %ld", rGroup->recordType, + fname, (long) recordOffset); records_deleted++; if (matchStr) dict_delete (matchDict, matchStr); rec_del (records, &rec); } + logRecord (0); return 1; } else @@ -938,7 +933,8 @@ static int recordExtract (SYSNO *sysno, const char *fname, else { if (rGroup->fileVerboseFlag) - logf (LOG_LOG, "update %s %s", rGroup->recordType, fname); + logf (LOG_LOG, "update %s %s %ld", rGroup->recordType, + fname, (long) recordOffset); flushRecordKeys (*sysno, 1, &reckeys, rGroup->databaseName); records_updated++; } @@ -980,27 +976,20 @@ static int recordExtract (SYSNO *sysno, const char *fname, xfree (rec->info[recInfo_storeData]); if (rGroup->flagStoreData == 1) { - rec->size[recInfo_storeData] = fi->file_noread; - rec->info[recInfo_storeData] = xmalloc (fi->file_noread); -#if FILE_READ_BUFSIZE - if (fi->file_noread < FILE_READ_BUFSIZE) - memcpy (rec->info[recInfo_storeData], fi->file_buf, - fi->file_noread); - else -#endif + rec->size[recInfo_storeData] = fi->file_max; + rec->info[recInfo_storeData] = xmalloc (fi->file_max); + if (lseek (fi->fd, recordOffset, SEEK_SET) < 0) { - if (lseek (fi->fd, 0L, SEEK_SET) < 0) - { - logf (LOG_ERRNO|LOG_FATAL, "seek to 0 in %s", fname); - exit (1); - } - if (read (fi->fd, rec->info[recInfo_storeData], fi->file_noread) - < fi->file_noread) - { - logf (LOG_ERRNO|LOG_FATAL, "read %d bytes of %s", - fi->file_noread, fname); - exit (1); - } + logf (LOG_ERRNO|LOG_FATAL, "seek to %ld in %s", fname, + (long) recordOffset); + exit (1); + } + if (read (fi->fd, rec->info[recInfo_storeData], fi->file_max) + < fi->file_max) + { + logf (LOG_ERRNO|LOG_FATAL, "read %d bytes of %s", + fi->file_max, fname); + exit (1); } } else @@ -1013,8 +1002,16 @@ static int recordExtract (SYSNO *sysno, const char *fname, rec->info[recInfo_databaseName] = rec_strdup (rGroup->databaseName, &rec->size[recInfo_databaseName]); + /* update offset */ + xfree (rec->info[recInfo_offset]); + + rec->size[recInfo_offset] = sizeof(recordOffset); + rec->info[recInfo_offset] = xmalloc (sizeof(recordOffset)); + memcpy (rec->info[recInfo_offset], &recordOffset, sizeof(recordOffset)); + /* commit this record */ rec_put (records, &rec); + logRecord (0); return 1; } @@ -1141,7 +1138,12 @@ int fileExtract (SYSNO *sysno, const char *fname, } } fi = file_read_start (fd); - r = recordExtract (sysno, fname, rGroup, deleteFlag, fi, recType, subType); + do + { + file_begin (fi); + r = recordExtract (sysno, fname, rGroup, deleteFlag, fi, + recType, subType); + } while (r && !sysno && fi->file_more); log_event_start (NULL, NULL); file_read_stop (fi); if (fd != -1) diff --git a/index/recindex.h b/index/recindex.h index d95c9ef..d0538ca 100644 --- a/index/recindex.h +++ b/index/recindex.h @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: recindex.h,v $ - * Revision 1.8 1995-12-06 12:41:25 adam + * Revision 1.9 1996-05-01 13:46:36 adam + * First work on multiple records in one file. + * New option, -offset, to the "unread" command in the filter module. + * + * Revision 1.8 1995/12/06 12:41:25 adam * New command 'stat' for the index program. * Filenames can be read from stdin by specifying '-'. * Bug fix/enhancement of the transformation from terms to regular @@ -66,6 +70,7 @@ enum { recInfo_filename, recInfo_delKeys, recInfo_databaseName, - recInfo_storeData + recInfo_storeData, + recInfo_offset }; diff --git a/index/trav.c b/index/trav.c index 9ba756a..9658662 100644 --- a/index/trav.c +++ b/index/trav.c @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: trav.c,v $ - * Revision 1.24 1996-04-26 10:00:23 adam + * Revision 1.25 1996-05-01 13:46:37 adam + * First work on multiple records in one file. + * New option, -offset, to the "unread" command in the filter module. + * + * Revision 1.24 1996/04/26 10:00:23 adam * Added option -V to zebraidx to display version information. * Removed stupid warnings from file update. * @@ -143,15 +147,6 @@ static void repositoryExtractR (int deleteFlag, char *rep, } -static void stdinExtractR (int deleteFlag, struct recordGroup *rGroup) -{ - char tmppath[1024]; - - logf (LOG_LOG, "stdinExtractR"); - while (scanf ("%s", tmppath) == 1) - fileExtract (NULL, tmppath, rGroup, deleteFlag); -} - static void fileDeleteR (struct dirs_info *di, struct dirs_entry *dst, const char *base, char *src, struct recordGroup *rGroup) @@ -334,7 +329,6 @@ static void groupRes (struct recordGroup *rGroup) rGroup->recordId = res_get (common_resource, resStr); } - void repositoryShow (struct recordGroup *rGroup) { char src[1024]; @@ -419,6 +413,38 @@ static void fileUpdate (Dict dict, struct recordGroup *rGroup, } } + +static void repositoryExtract (int deleteFlag, struct recordGroup *rGroup, + const char *path) +{ + struct stat sbuf; + char src[1024]; + + assert (path); + strcpy (src, path); + + stat (src, &sbuf); + if (S_ISREG(sbuf.st_mode)) + fileExtract (NULL, src, rGroup, deleteFlag); + else if (S_ISDIR(sbuf.st_mode)) + repositoryExtractR (deleteFlag, src, rGroup); + else + logf (LOG_WARN, "Cannot handle file %s", src); +} + +static void repositoryExtractG (int deleteFlag, struct recordGroup *rGroup) +{ + if (*rGroup->path == '\0' || !strcmp(rGroup->path, "-")) + { + char src[1024]; + + while (scanf ("%s", src) == 1) + repositoryExtract (deleteFlag, rGroup, src); + } + else + repositoryExtract (deleteFlag, rGroup, rGroup->path); +} + void repositoryUpdate (struct recordGroup *rGroup) { groupRes (rGroup); @@ -442,27 +468,11 @@ void repositoryUpdate (struct recordGroup *rGroup) dict_close (dict); } else - { - char src[1024]; - - strcpy (src, rGroup->path); - if (*src == '\0' || !strcmp (src, "-")) - stdinExtractR (0, rGroup); - else - repositoryExtractR (0, src, rGroup); - } + repositoryExtractG (0, rGroup); } void repositoryDelete (struct recordGroup *rGroup) { - char src[256]; - - assert (rGroup->path); - groupRes (rGroup); - strcpy (src, rGroup->path); - if (*src == '\0' || !strcmp(src, "-")) - stdinExtractR (1, rGroup); - else - repositoryExtractR (1, src, rGroup); + repositoryExtractG (1, rGroup); } diff --git a/index/zserver.c b/index/zserver.c index 88a90a4..c3ea6b4 100644 --- a/index/zserver.c +++ b/index/zserver.c @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: zserver.c,v $ - * Revision 1.35 1996-03-26 16:01:14 adam + * Revision 1.36 1996-05-01 13:46:37 adam + * First work on multiple records in one file. + * New option, -offset, to the "unread" command in the filter module. + * + * Revision 1.35 1996/03/26 16:01:14 adam * New setting lockPath: directory of various lock files. * * Revision 1.34 1996/03/20 09:36:46 adam @@ -270,15 +274,27 @@ bend_searchresult *bend_search (void *handle, bend_searchrequest *q, int *fd) return &r; } +static int record_offset; + static int record_ext_read (void *fh, char *buf, size_t count) { return read (*((int*) fh), buf, count); } +static off_t record_ext_seek (void *fh, off_t offset) +{ + return lseek (*((int*) fh), offset + record_offset, SEEK_SET); +} + static int record_int_pos; static char *record_int_buf; static int record_int_len; +static off_t record_int_seek (void *fh, off_t offset) +{ + return (off_t) (record_int_pos = offset); +} + static int record_int_read (void *fh, char *buf, size_t count) { int l = record_int_len - record_int_pos; @@ -325,6 +341,7 @@ static int record_fetch (ZServerInfo *zi, int sysno, int score, ODR stream, if (rec->size[recInfo_storeData] > 0) { retrieveCtrl.readf = record_int_read; + retrieveCtrl.seekf = record_int_seek; record_int_len = rec->size[recInfo_storeData]; record_int_buf = rec->info[recInfo_storeData]; record_int_pos = 0; @@ -342,8 +359,15 @@ static int record_fetch (ZServerInfo *zi, int sysno, int score, ODR stream, rec_rm (&rec); return 0; /* or 14: System error in presenting records */ } + + memcpy (&record_offset, rec->info[recInfo_offset], + sizeof(record_offset)); + retrieveCtrl.fh = &fd; retrieveCtrl.readf = record_ext_read; + retrieveCtrl.seekf = record_ext_seek; + + record_ext_seek (retrieveCtrl.fh, 0); } retrieveCtrl.subType = subType; retrieveCtrl.localno = sysno;