X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fextract.c;h=066262153574ee15179e9dbd6c725ea1441d5d57;hb=1cd8e47d53d94462ab3a96f01ab200222004b901;hp=d3a357d41f839a9f607a403b0fb600c260674af1;hpb=53f42a6fe9676c59ec2de267551157c989400611;p=idzebra-moved-to-github.git diff --git a/index/extract.c b/index/extract.c index d3a357d..0662621 100644 --- a/index/extract.c +++ b/index/extract.c @@ -4,7 +4,82 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: extract.c,v $ - * Revision 1.35 1995-11-28 14:26:21 adam + * Revision 1.55 1996-05-09 07:28:55 quinn + * Work towards phrases and multiple registers + * + * Revision 1.54 1996/05/01 13:46:35 adam + * First work on multiple records in one file. + * New option, -offset, to the "unread" command in the filter module. + * + * Revision 1.53 1996/04/26 12:09:43 adam + * Added a few comments. + * + * Revision 1.52 1996/04/25 13:27:57 adam + * Function recordExtract modified so that files with no keys (possibly empty) + * are ignored. + * + * Revision 1.51 1996/03/19 11:08:42 adam + * Bug fix: Log preamble wasn't always turned off after recordExtract. + * + * Revision 1.50 1996/02/12 18:45:36 adam + * New fileVerboseFlag in record group control. + * + * Revision 1.49 1996/02/05 12:29:57 adam + * Logging reduced a bit. + * The remaining running time is estimated during register merge. + * + * Revision 1.48 1996/02/01 20:53:26 adam + * The temporary per-record keys are compacted a little, and duplication + * of the per-records keys are avoided when they are saved in the record + * information buffer. + * + * Revision 1.47 1996/01/17 14:57:48 adam + * Prototype changed for reader functions in extract/retrieve. File + * is identified by 'void *' instead of 'int. + * + * Revision 1.46 1995/12/15 14:57:16 adam + * Bug fix. + * + * Revision 1.45 1995/12/15 12:37:41 adam + * In addRecordKeyAny: Writes key only when attrSet != -1. + * + * Revision 1.44 1995/12/12 16:00:54 adam + * System call sync(2) used after update/commit. + * Locking (based on fcntl) uses F_EXLCK and F_SHLCK instead of F_WRLCK + * and F_RDLCK. + * + * Revision 1.43 1995/12/11 09:12:46 adam + * The rec_get function returns NULL if record doesn't exist - will + * happen in the server if the result set records have been deleted since + * the creation of the set (i.e. the search). + * The server saves a result temporarily if it is 'volatile', i.e. the + * set is register dependent. + * + * Revision 1.42 1995/12/07 17:38:46 adam + * Work locking mechanisms for concurrent updates/commit. + * + * Revision 1.41 1995/12/06 16:06:42 adam + * Better diagnostics. Work on 'real' dictionary deletion. + * + * Revision 1.40 1995/12/05 16:57:40 adam + * More work on regular patterns. + * + * Revision 1.39 1995/12/05 13:20:18 adam + * Bug fix: file_read sometimes returned early EOF. + * + * Revision 1.38 1995/12/04 17:59:21 adam + * More work on regular expression conversion. + * + * Revision 1.37 1995/12/04 14:22:27 adam + * Extra arg to recType_byName. + * Started work on new regular expression parsed input to + * structured records. + * + * Revision 1.36 1995/11/30 08:34:29 adam + * Started work on commit facility. + * Changed a few malloc/free to xmalloc/xfree. + * + * Revision 1.35 1995/11/28 14:26:21 adam * Bug fix: recordId with constant wasn't right. * Bug fix: recordId dictionary entry wasn't deleted when needed. * @@ -148,8 +223,19 @@ static int key_file_no; static int records_inserted = 0; static int records_updated = 0; static int records_deleted = 0; +static int records_processed = 0; -#define MATCH_DICT "match" +static void logRecord (int showFlag) +{ + if (!showFlag) + ++records_processed; + if (showFlag || !(records_processed % 1000)) + { + logf (LOG_LOG, "Records: %7d i/u/d %d/%d/%d", + records_processed, records_inserted, records_updated, + records_deleted); + } +} void key_open (int mem) { @@ -162,9 +248,9 @@ void key_open (int mem) key_buf_used = 0; key_file_no = 0; - if (!(matchDict = dict_open (MATCH_DICT, 20, 1))) + if (!(matchDict = dict_open (GMATCH_DICT, 50, 1))) { - logf (LOG_FATAL, "dict_open fail of %s", MATCH_DICT); + logf (LOG_FATAL, "dict_open fail of %s", GMATCH_DICT); exit (1); } assert (!records); @@ -285,9 +371,7 @@ int key_close (void) rec_close (&records); dict_close (matchDict); - logf (LOG_LOG, "Records inserted %6d", records_inserted); - logf (LOG_LOG, "Records updated %6d", records_updated); - logf (LOG_LOG, "Records deleted %6d", records_deleted); + logRecord (1); return key_file_no; } @@ -302,6 +386,8 @@ struct recKeys { int buf_used; int buf_max; char *buf; + char prevAttrSet; + short prevAttrUse; } reckeys; static void addRecordKey (const RecWord *p) @@ -310,29 +396,46 @@ static void addRecordKey (const RecWord *p) char attrSet; short attrUse; size_t i; + int lead = 0; if (reckeys.buf_used+1024 > reckeys.buf_max) { char *b; - b = malloc (reckeys.buf_max += 65000); + b = xmalloc (reckeys.buf_max += 128000); if (reckeys.buf_used > 0) memcpy (b, reckeys.buf, reckeys.buf_used); - free (reckeys.buf); + xfree (reckeys.buf); reckeys.buf = b; } dst = reckeys.buf + reckeys.buf_used; + + attrSet = p->attrSet; + if (reckeys.buf_used > 0 && reckeys.prevAttrSet == attrSet) + lead |= 1; + else + reckeys.prevAttrSet = attrSet; + attrUse = p->attrUse; + if (reckeys.buf_used > 0 && reckeys.prevAttrUse == attrUse) + lead |= 2; + else + reckeys.prevAttrUse = attrUse; + switch (p->which) { - case Word_String: - attrSet = p->attrSet; - memcpy (dst, &attrSet, sizeof(attrSet)); - dst += sizeof(attrSet); + case Word_String: case Word_Phrase: + *dst++ = lead; - attrUse = p->attrUse; - memcpy (dst, &attrUse, sizeof(attrUse)); - dst += sizeof(attrUse); - + if (!(lead & 1)) + { + memcpy (dst, &attrSet, sizeof(attrSet)); + dst += sizeof(attrSet); + } + if (!(lead & 2)) + { + memcpy (dst, &attrUse, sizeof(attrUse)); + dst += sizeof(attrUse); + } for (i = 0; p->u.string[i]; i++) *dst++ = p->u.string[i]; *dst++ = '\0'; @@ -350,20 +453,27 @@ static void addRecordKey (const RecWord *p) static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys, const char *databaseName) { + char attrSet = -1; + short attrUse = -1; int off = 0; while (off < reckeys->buf_used) { const char *src = reckeys->buf + off; - char attrSet; - short attrUse; struct it_key key; - - memcpy (&attrSet, src, sizeof(attrSet)); - src += sizeof(attrSet); - - memcpy (&attrUse, src, sizeof(attrUse)); - src += sizeof(attrUse); + int lead; + + lead = *src++; + if (!(lead & 1)) + { + memcpy (&attrSet, src, sizeof(attrSet)); + src += sizeof(attrSet); + } + if (!(lead & 2)) + { + memcpy (&attrUse, src, sizeof(attrUse)); + src += sizeof(attrUse); + } if (key_buf_used + 1024 > (ptr_top-ptr_i)*sizeof(char*)) key_flush (); ++ptr_i; @@ -451,77 +561,73 @@ static void addRecordKeyAny (const RecWord *p) w.attrUse = 1016; addRecordKey (&w); } - addRecordKey (p); + if (p->attrSet != -1) + addRecordKey (p); } -#define FILE_READ_BUFSIZE 4096 - -static char *file_buf; -static int file_offset; -static int file_bufsize; -static int file_noread; +struct file_read_info { + off_t file_max; + off_t file_offset; + off_t file_moffset; + int file_more; + int fd; +}; -static void file_read_start (int fd) +static struct file_read_info *file_read_start (int fd) { - file_offset = 0; - file_buf = xmalloc (FILE_READ_BUFSIZE); - file_bufsize = read (fd, file_buf, FILE_READ_BUFSIZE); - file_noread = 0; + struct file_read_info *fi = xmalloc (sizeof(*fi)); + + fi->fd = fd; + fi->file_max = 0; + fi->file_moffset = 0; + return fi; } -static void file_read_stop (int fd) +static void file_read_stop (struct file_read_info *fi) { - xfree (file_buf); - file_buf = NULL; + assert (fi); + xfree (fi); } -static int file_read (int fd, char *buf, size_t count) +static off_t file_seek (void *handle, off_t offset) { - int l = file_bufsize - file_offset; + struct file_read_info *p = handle; + p->file_offset = offset; + return lseek (p->fd, offset, SEEK_SET); +} - if (count > l) +static int file_read (void *handle, char *buf, size_t count) +{ + struct file_read_info *p = handle; + int fd = p->fd; + int r; + r = read (fd, buf, count); + if (r > 0) { - int r; - if (l > 0) - memcpy (buf, file_buf + file_offset, l); - count = count-l; - if (count > file_bufsize) - { - if ((r = read (fd, buf + l, count)) == -1) - { - logf (LOG_FATAL|LOG_ERRNO, "read"); - exit (1); - } - file_bufsize = 0; - file_offset = 0; - file_noread += r; - return r; - } - file_bufsize = r = read (fd, file_buf, FILE_READ_BUFSIZE); - if (r == -1) - { - logf (LOG_FATAL|LOG_ERRNO, "read"); - exit (1); - } - else if (r <= count) - { - file_offset = r; - memcpy (buf + l, file_buf, r); - file_noread += (l+r); - return l + r; - } - else - { - file_offset = count; - memcpy (buf + l, file_buf, count - l); - file_noread += count; - return count; - } + p->file_offset += r; + if (p->file_offset > p->file_max) + p->file_max = p->file_offset; } - memcpy (buf, file_buf + file_offset, count); - file_offset += count; - file_noread += count; - return count; + return r; +} + +static void file_begin (void *handle) +{ + struct file_read_info *p = handle; + + p->file_offset = p->file_moffset; + if (p->file_moffset) + lseek (p->fd, p->file_moffset, SEEK_SET); + p->file_more = 0; +} + +static void file_end (void *handle, off_t offset) +{ + struct file_read_info *p = handle; + + assert (p->file_more == 0); + p->file_more = 1; + p->file_moffset = offset; } static int atois (const char **s) @@ -596,8 +702,8 @@ static char *fileMatchStr (struct recKeys *reckeys, struct recordGroup *rGroup, } if (first) { - logf (LOG_WARN, "Record in file %s didn't contain match" - " fields in (%d,%d)", fname, attrSet, attrUse); + logf (LOG_WARN, "Record didn't contain match" + " fields in (%d,%d)", attrSet, attrUse); return NULL; } } @@ -667,34 +773,75 @@ static char *fileMatchStr (struct recKeys *reckeys, struct recordGroup *rGroup, return dstBuf; } +struct recordLogInfo { + const char *fname; + char *op; + struct recordGroup *rGroup; +}; + +static void recordLogPreamble (int level, const char *msg, void *info) +{ + struct recordLogInfo *p = info; + FILE *outf = log_file (); + + if (level & LOG_LOG) + return ; + if (p->op) + fprintf (outf, "%s of ", p->op); + fprintf (outf, "%s type %s\n", p->rGroup->recordType, p->fname); + log_event_start (NULL, NULL); +} + static int recordExtract (SYSNO *sysno, const char *fname, struct recordGroup *rGroup, int deleteFlag, - int fd, - RecType recType) + struct file_read_info *fi, RecType recType, + char *subType) { struct recExtractCtrl extractCtrl; int r; char *matchStr; SYSNO sysnotmp; + off_t recordOffset = 0; Record rec; + struct recordLogInfo logInfo; - if (fd != -1) + logInfo.fname = fname; + logInfo.op = NULL; + logInfo.rGroup = rGroup; + log_event_start (recordLogPreamble, &logInfo); + + if (fi->fd != -1) { - extractCtrl.fd = fd; - /* extract keys */ - extractCtrl.subType = ""; + /* we are going to read from a file, so prepare the extraction */ + extractCtrl.fh = fi; + extractCtrl.subType = subType; extractCtrl.init = wordInit; extractCtrl.add = addRecordKeyAny; reckeys.buf_used = 0; + reckeys.prevAttrUse = -1; + reckeys.prevAttrSet = -1; + + recordOffset = fi->file_moffset; + extractCtrl.offset = recordOffset; extractCtrl.readf = file_read; + extractCtrl.seekf = file_seek; + extractCtrl.endf = file_end; r = (*recType->extract)(&extractCtrl); - + if (r) { + /* error occured during extraction ... */ logf (LOG_WARN, "Couldn't extract file %s, code %d", fname, r); return 0; } + if (reckeys.buf_used == 0) + { + /* the extraction process returned no information - the record + is probably empty */ + logf (LOG_WARN, "Empty file %s", fname); + return 0; + } } /* perform match if sysno not known and if match criteria is specified */ @@ -718,21 +865,24 @@ static int recordExtract (SYSNO *sysno, const char *fname, } else { - logf (LOG_WARN, "Record not inserted"); + logf (LOG_WARN, "Bad match criteria"); return 0; } } } - /* new record ? */ if (! *sysno) { + /* new record */ if (deleteFlag) { - logf (LOG_LOG, "? %s", fname); + logf (LOG_LOG, "Cannot delete new record"); return 1; } - logf (LOG_LOG, "add %s %s", rGroup->recordType, fname); + logInfo.op = "add"; + if (rGroup->fileVerboseFlag) + logf (LOG_LOG, "add %s %s+%ld", rGroup->recordType, + fname, (long) recordOffset); rec = rec_new (records); *sysno = rec->sysno; @@ -746,61 +896,78 @@ static int recordExtract (SYSNO *sysno, const char *fname, } else { + /* record already exists */ struct recKeys delkeys; rec = rec_get (records, *sysno); - + assert (rec); delkeys.buf_used = rec->size[recInfo_delKeys]; delkeys.buf = rec->info[recInfo_delKeys]; flushRecordKeys (*sysno, 0, &delkeys, rec->info[recInfo_databaseName]); if (deleteFlag) { + /* record going to be deleted */ + logInfo.op = "delete"; if (!delkeys.buf_used) { - logf (LOG_WARN, "cannot delete %s - no delete keys", fname); + logf (LOG_WARN, "cannot delete; storeKeys false"); } else { - SYSNO sysnoz = 0; - logf (LOG_LOG, "delete %s %s", rGroup->recordType, fname); + if (rGroup->fileVerboseFlag) + logf (LOG_LOG, "delete %s %s %ld", rGroup->recordType, + fname, (long) recordOffset); records_deleted++; if (matchStr) - dict_insert (matchDict, matchStr, sizeof(sysnoz), &sysnoz); + dict_delete (matchDict, matchStr); rec_del (records, &rec); } + logRecord (0); return 1; } else { + /* record going to be updated */ + logInfo.op = "update"; if (!delkeys.buf_used) { - logf (LOG_WARN, "cannot update %s - no delete keys", - fname); + logf (LOG_WARN, "cannot update; storeKeys false"); } else { - logf (LOG_LOG, "update %s %s", rGroup->recordType, - fname); + if (rGroup->fileVerboseFlag) + logf (LOG_LOG, "update %s %s %ld", rGroup->recordType, + fname, (long) recordOffset); flushRecordKeys (*sysno, 1, &reckeys, rGroup->databaseName); records_updated++; } } } - free (rec->info[recInfo_fileType]); + /* update file type */ + xfree (rec->info[recInfo_fileType]); rec->info[recInfo_fileType] = rec_strdup (rGroup->recordType, &rec->size[recInfo_fileType]); - free (rec->info[recInfo_filename]); + /* update filename */ + xfree (rec->info[recInfo_filename]); rec->info[recInfo_filename] = rec_strdup (fname, &rec->size[recInfo_filename]); - free (rec->info[recInfo_delKeys]); + /* update delete keys */ + xfree (rec->info[recInfo_delKeys]); if (reckeys.buf_used > 0 && rGroup->flagStoreKeys == 1) { - rec->info[recInfo_delKeys] = malloc (reckeys.buf_used); +#if 1 + rec->size[recInfo_delKeys] = reckeys.buf_used; + rec->info[recInfo_delKeys] = reckeys.buf; + reckeys.buf = NULL; + reckeys.buf_max = 0; +#else + rec->info[recInfo_delKeys] = xmalloc (reckeys.buf_used); rec->size[recInfo_delKeys] = reckeys.buf_used; memcpy (rec->info[recInfo_delKeys], reckeys.buf, rec->size[recInfo_delKeys]); +#endif } else { @@ -808,27 +975,24 @@ static int recordExtract (SYSNO *sysno, const char *fname, rec->size[recInfo_delKeys] = 0; } - free (rec->info[recInfo_storeData]); + /* update store data */ + xfree (rec->info[recInfo_storeData]); if (rGroup->flagStoreData == 1) { - rec->size[recInfo_storeData] = file_noread; - rec->info[recInfo_storeData] = malloc (file_noread); - if (file_noread < FILE_READ_BUFSIZE) - memcpy (rec->info[recInfo_storeData], file_buf, file_noread); - else + rec->size[recInfo_storeData] = fi->file_max; + rec->info[recInfo_storeData] = xmalloc (fi->file_max); + if (lseek (fi->fd, recordOffset, SEEK_SET) < 0) { - if (lseek (fd, 0L, SEEK_SET) < 0) - { - logf (LOG_ERRNO|LOG_FATAL, "seek to 0 in %s", fname); - exit (1); - } - if (read (fd, rec->info[recInfo_storeData], file_noread) - < file_noread) - { - logf (LOG_ERRNO|LOG_FATAL, "read %d bytes of %s", - file_noread, fname); - exit (1); - } + logf (LOG_ERRNO|LOG_FATAL, "seek to %ld in %s", fname, + (long) recordOffset); + exit (1); + } + if (read (fi->fd, rec->info[recInfo_storeData], fi->file_max) + < fi->file_max) + { + logf (LOG_ERRNO|LOG_FATAL, "read %d bytes of %s", + fi->file_max, fname); + exit (1); } } else @@ -836,24 +1000,36 @@ static int recordExtract (SYSNO *sysno, const char *fname, rec->info[recInfo_storeData] = NULL; rec->size[recInfo_storeData] = 0; } - free (rec->info[recInfo_databaseName]); + /* update database name */ + xfree (rec->info[recInfo_databaseName]); rec->info[recInfo_databaseName] = rec_strdup (rGroup->databaseName, &rec->size[recInfo_databaseName]); + /* update offset */ + xfree (rec->info[recInfo_offset]); + + rec->size[recInfo_offset] = sizeof(recordOffset); + rec->info[recInfo_offset] = xmalloc (sizeof(recordOffset)); + memcpy (rec->info[recInfo_offset], &recordOffset, sizeof(recordOffset)); + + /* commit this record */ rec_put (records, &rec); + logRecord (0); return 1; } int fileExtract (SYSNO *sysno, const char *fname, const struct recordGroup *rGroupP, int deleteFlag) { - int i, fd; + int r, i, fd; char gprefix[128]; char ext[128]; char ext_res[128]; + char subType[128]; RecType recType; struct recordGroup rGroupM; struct recordGroup *rGroup = &rGroupM; + struct file_read_info *fi; memcpy (rGroup, rGroupP, sizeof(*rGroupP)); @@ -885,17 +1061,19 @@ int fileExtract (SYSNO *sysno, const char *fname, sprintf (ext_res, "%srecordType", gprefix); if (!(rGroup->recordType = res_get (common_resource, ext_res))) { - logf (LOG_LOG, "? %s", fname); + if (rGroup->fileVerboseFlag) + logf (LOG_LOG, "? %s", fname); return 0; } } } if (!rGroup->recordType) { - logf (LOG_LOG, "? record %s", fname); + if (rGroup->fileVerboseFlag) + logf (LOG_LOG, "? record %s", fname); return 0; } - if (!(recType = recType_byName (rGroup->recordType))) + if (!(recType = recType_byName (rGroup->recordType, subType))) { logf (LOG_WARN, "No such record type: %s", rGroup->recordType); return 0; @@ -962,11 +1140,17 @@ int fileExtract (SYSNO *sysno, const char *fname, return 0; } } - file_read_start (fd); - recordExtract (sysno, fname, rGroup, deleteFlag, fd, recType); - file_read_stop (fd); + fi = file_read_start (fd); + do + { + file_begin (fi); + r = recordExtract (sysno, fname, rGroup, deleteFlag, fi, + recType, subType); + } while (r && !sysno && fi->file_more); + log_event_start (NULL, NULL); + file_read_stop (fi); if (fd != -1) close (fd); - return 1; + return r; }