X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fextract.c;h=285003d928f1862eef57e1cb129ced64c85df83f;hb=162fa86044b5aa303c03fa26f1ecf140b34060d5;hp=09c2d74461982260bcd781c6c7a787bcc1a4f6e3;hpb=366f5c2889c8bccb1f645aebf737b6082f200da5;p=idzebra-moved-to-github.git diff --git a/index/extract.c b/index/extract.c index 09c2d74..285003d 100644 --- a/index/extract.c +++ b/index/extract.c @@ -4,7 +4,17 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: extract.c,v $ - * Revision 1.32 1995-11-25 10:24:05 adam + * Revision 1.34 1995-11-28 09:09:38 adam + * Zebra config renamed. + * Use setting 'recordId' to identify record now. + * Bug fix in recindex.c: rec_release_blocks was invokeded even + * though the blocks were already released. + * File traversal properly deletes records when needed. + * + * Revision 1.33 1995/11/27 09:56:20 adam + * Record info elements better enumerated. Internal store of records. + * + * Revision 1.32 1995/11/25 10:24:05 adam * More record fields - they are enumerated now. * New options: flagStoreData flagStoreKey. * @@ -440,20 +450,25 @@ static void addRecordKeyAny (const RecWord *p) addRecordKey (p); } +#define FILE_READ_BUFSIZE 4096 + static char *file_buf; static int file_offset; static int file_bufsize; +static int file_noread; static void file_read_start (int fd) { file_offset = 0; - file_buf = xmalloc (4096); - file_bufsize = read (fd, file_buf, 4096); + file_buf = xmalloc (FILE_READ_BUFSIZE); + file_bufsize = read (fd, file_buf, FILE_READ_BUFSIZE); + file_noread = 0; } static void file_read_stop (int fd) { xfree (file_buf); + file_buf = NULL; } static int file_read (int fd, char *buf, size_t count) @@ -475,9 +490,10 @@ static int file_read (int fd, char *buf, size_t count) } file_bufsize = 0; file_offset = 0; + file_noread += r; return r; } - file_bufsize = r = read (fd, file_buf, 4096); + file_bufsize = r = read (fd, file_buf, FILE_READ_BUFSIZE); if (r == -1) { logf (LOG_FATAL|LOG_ERRNO, "read"); @@ -487,17 +503,20 @@ static int file_read (int fd, char *buf, size_t count) { file_offset = r; memcpy (buf + l, file_buf, r); + file_noread += (l+r); return l + r; } else { file_offset = count; memcpy (buf + l, file_buf, count - l); + file_noread += count; return count; } } memcpy (buf, file_buf + file_offset, count); file_offset += count; + file_noread += count; return count; } @@ -514,7 +533,6 @@ static int atois (const char **s) static char *fileMatchStr (struct recKeys *reckeys, struct recordGroup *rGroup, const char *fname, - const char *recordType, const char *spec) { static char dstBuf[2048]; @@ -602,7 +620,7 @@ static char *fileMatchStr (struct recKeys *reckeys, struct recordGroup *rGroup, else if (strcmp (special, "filename")) spec_src = fname; else if (strcmp (special, "type")) - spec_src = recordType; + spec_src = rGroup->recordType; else spec_src = NULL; if (spec_src) @@ -648,7 +666,6 @@ static char *fileMatchStr (struct recKeys *reckeys, struct recordGroup *rGroup, static int recordExtract (SYSNO *sysno, const char *fname, struct recordGroup *rGroup, int deleteFlag, int fd, - const char *file_type, RecType recType) { struct recExtractCtrl extractCtrl; @@ -657,24 +674,23 @@ static int recordExtract (SYSNO *sysno, const char *fname, SYSNO sysnotmp; Record rec; - extractCtrl.fd = fd; - - /* extract keys */ - extractCtrl.subType = ""; - extractCtrl.init = wordInit; - extractCtrl.add = addRecordKeyAny; - - reckeys.buf_used = 0; - file_read_start (extractCtrl.fd); - extractCtrl.readf = file_read; - r = (*recType->extract)(&extractCtrl); - file_read_stop (extractCtrl.fd); - close (extractCtrl.fd); - - if (r) + if (fd != -1) { - logf (LOG_WARN, "Couldn't extract file %s, code %d", fname, r); - return 0; + extractCtrl.fd = fd; + /* extract keys */ + extractCtrl.subType = ""; + extractCtrl.init = wordInit; + extractCtrl.add = addRecordKeyAny; + + reckeys.buf_used = 0; + extractCtrl.readf = file_read; + r = (*recType->extract)(&extractCtrl); + + if (r) + { + logf (LOG_WARN, "Couldn't extract file %s, code %d", fname, r); + return 0; + } } /* perform match if sysno not known and if match criteria is specified */ @@ -684,12 +700,12 @@ static int recordExtract (SYSNO *sysno, const char *fname, { sysnotmp = 0; sysno = &sysnotmp; - if (rGroup->fileMatch) + if (rGroup->recordId && *rGroup->recordId) { char *rinfo; - matchStr = fileMatchStr(&reckeys, rGroup, fname, file_type, - rGroup->fileMatch); + matchStr = fileMatchStr (&reckeys, rGroup, fname, + rGroup->recordId); if (matchStr) { rinfo = dict_lookup (matchDict, matchStr); @@ -712,7 +728,7 @@ static int recordExtract (SYSNO *sysno, const char *fname, logf (LOG_LOG, "? record %s", fname); return 1; } - logf (LOG_LOG, "add record %s", fname); + logf (LOG_LOG, "add %s record %s", rGroup->recordType, fname); rec = rec_new (records); *sysno = rec->sysno; @@ -739,7 +755,8 @@ static int recordExtract (SYSNO *sysno, const char *fname, fname); } else - logf (LOG_LOG, "delete record %s", fname); + logf (LOG_LOG, "delete %s record %s", rGroup->recordType, + fname); records_deleted++; rec_del (records, &rec); return 1; @@ -753,7 +770,8 @@ static int recordExtract (SYSNO *sysno, const char *fname, } else { - logf (LOG_LOG, "update record %s", fname); + logf (LOG_LOG, "update %s record %s", rGroup->recordType, + fname); flushRecordKeys (*sysno, 1, &reckeys, rGroup->databaseName); records_updated++; } @@ -761,7 +779,7 @@ static int recordExtract (SYSNO *sysno, const char *fname, } free (rec->info[recInfo_fileType]); rec->info[recInfo_fileType] = - rec_strdup (file_type, &rec->size[recInfo_fileType]); + rec_strdup (rGroup->recordType, &rec->size[recInfo_fileType]); free (rec->info[recInfo_filename]); rec->info[recInfo_filename] = @@ -770,7 +788,6 @@ static int recordExtract (SYSNO *sysno, const char *fname, free (rec->info[recInfo_delKeys]); if (reckeys.buf_used > 0 && rGroup->flagStoreKeys == 1) { - logf (LOG_LOG, "Storing keys..."); rec->info[recInfo_delKeys] = malloc (reckeys.buf_used); rec->size[recInfo_delKeys] = reckeys.buf_used; memcpy (rec->info[recInfo_delKeys], reckeys.buf, @@ -781,6 +798,35 @@ static int recordExtract (SYSNO *sysno, const char *fname, rec->info[recInfo_delKeys] = NULL; rec->size[recInfo_delKeys] = 0; } + + free (rec->info[recInfo_storeData]); + if (rGroup->flagStoreData == 1) + { + rec->size[recInfo_storeData] = file_noread; + rec->info[recInfo_storeData] = malloc (file_noread); + if (file_noread < FILE_READ_BUFSIZE) + memcpy (rec->info[recInfo_storeData], file_buf, file_noread); + else + { + if (lseek (fd, 0L, SEEK_SET) < 0) + { + logf (LOG_ERRNO|LOG_FATAL, "seek to 0 in %s", fname); + exit (1); + } + if (read (fd, rec->info[recInfo_storeData], file_noread) + < file_noread) + { + logf (LOG_ERRNO|LOG_FATAL, "read %d bytes of %s", + file_noread, fname); + exit (1); + } + } + } + else + { + rec->info[recInfo_storeData] = NULL; + rec->size[recInfo_storeData] = 0; + } free (rec->info[recInfo_databaseName]); rec->info[recInfo_databaseName] = rec_strdup (rGroup->databaseName, &rec->size[recInfo_databaseName]); @@ -789,16 +835,19 @@ static int recordExtract (SYSNO *sysno, const char *fname, return 1; } -int fileExtract (SYSNO *sysno, const char *fname, struct recordGroup *rGroup, - int deleteFlag) +int fileExtract (SYSNO *sysno, const char *fname, + const struct recordGroup *rGroupP, int deleteFlag) { int i, fd; char gprefix[128]; char ext[128]; char ext_res[128]; - const char *file_type; RecType recType; + struct recordGroup rGroupM; + struct recordGroup *rGroup = &rGroupM; + memcpy (rGroup, rGroupP, sizeof(*rGroupP)); + if (!rGroup->groupName || !*rGroup->groupName) *gprefix = '\0'; else @@ -819,22 +868,35 @@ int fileExtract (SYSNO *sysno, const char *fname, struct recordGroup *rGroup, break; } /* determine file type - depending on extension */ - sprintf (ext_res, "%sfileExtension.%s", gprefix, ext); - if (!(file_type = res_get (common_resource, ext_res))) + if (!rGroup->recordType) + { + sprintf (ext_res, "%srecordType.%s", gprefix, ext); + if (!(rGroup->recordType = res_get (common_resource, ext_res))) + { + sprintf (ext_res, "%srecordType", gprefix); + if (!(rGroup->recordType = res_get (common_resource, ext_res))) + { + logf (LOG_LOG, "? record %s", fname); + return 0; + } + } + } + if (!rGroup->recordType) + { + logf (LOG_LOG, "? record %s", fname); return 0; - if (!(recType = recType_byName (file_type))) + } + if (!(recType = recType_byName (rGroup->recordType))) + { + logf (LOG_WARN, "No such record type: %s", rGroup->recordType); return 0; + } /* determine match criteria */ - if (rGroup->fileMatch) + if (!rGroup->recordId) { - sprintf (ext_res, "%sfileMatch.%s", gprefix, ext); - rGroup->fileMatch = res_get (common_resource, ext_res); - if (!rGroup->fileMatch) - { - sprintf (ext_res, "%sfileMatch", gprefix); - rGroup->fileMatch = res_get (common_resource, ext_res); - } + sprintf (ext_res, "%srecordId.%s", gprefix, ext); + rGroup->recordId = res_get (common_resource, ext_res); } /* determine database name */ @@ -865,7 +927,6 @@ int fileExtract (SYSNO *sysno, const char *fname, struct recordGroup *rGroup, if (rGroup->flagStoreData == -1) rGroup->flagStoreData = 0; - if (rGroup->flagStoreKeys == -1) { const char *sval; @@ -882,16 +943,21 @@ int fileExtract (SYSNO *sysno, const char *fname, struct recordGroup *rGroup, if (rGroup->flagStoreKeys == -1) rGroup->flagStoreKeys = 0; - - /* open input file */ - if ((fd = open (fname, O_RDONLY)) == -1) + if (sysno && deleteFlag) + fd = -1; + else { - logf (LOG_WARN|LOG_ERRNO, "open %s", fname); - return 0; + if ((fd = open (fname, O_RDONLY)) == -1) + { + logf (LOG_WARN|LOG_ERRNO, "open %s", fname); + return 0; + } } - recordExtract (sysno, fname, rGroup, deleteFlag, fd, - file_type, recType); - close (fd); + file_read_start (fd); + recordExtract (sysno, fname, rGroup, deleteFlag, fd, recType); + file_read_stop (fd); + if (fd != -1) + close (fd); return 1; }