X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fextract.c;h=6f7686c25ecd63fb43f0567f2da98c0bc67587e6;hb=5997f85a9dbeab93bae3492139ed20fea878372c;hp=d3a357d41f839a9f607a403b0fb600c260674af1;hpb=53f42a6fe9676c59ec2de267551157c989400611;p=idzebra-moved-to-github.git diff --git a/index/extract.c b/index/extract.c index d3a357d..6f7686c 100644 --- a/index/extract.c +++ b/index/extract.c @@ -4,7 +4,53 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: extract.c,v $ - * Revision 1.35 1995-11-28 14:26:21 adam + * Revision 1.47 1996-01-17 14:57:48 adam + * Prototype changed for reader functions in extract/retrieve. File + * is identified by 'void *' instead of 'int. + * + * Revision 1.46 1995/12/15 14:57:16 adam + * Bug fix. + * + * Revision 1.45 1995/12/15 12:37:41 adam + * In addRecordKeyAny: Writes key only when attrSet != -1. + * + * Revision 1.44 1995/12/12 16:00:54 adam + * System call sync(2) used after update/commit. + * Locking (based on fcntl) uses F_EXLCK and F_SHLCK instead of F_WRLCK + * and F_RDLCK. + * + * Revision 1.43 1995/12/11 09:12:46 adam + * The rec_get function returns NULL if record doesn't exist - will + * happen in the server if the result set records have been deleted since + * the creation of the set (i.e. the search). + * The server saves a result temporarily if it is 'volatile', i.e. the + * set is register dependent. + * + * Revision 1.42 1995/12/07 17:38:46 adam + * Work locking mechanisms for concurrent updates/commit. + * + * Revision 1.41 1995/12/06 16:06:42 adam + * Better diagnostics. Work on 'real' dictionary deletion. + * + * Revision 1.40 1995/12/05 16:57:40 adam + * More work on regular patterns. + * + * Revision 1.39 1995/12/05 13:20:18 adam + * Bug fix: file_read sometimes returned early EOF. + * + * Revision 1.38 1995/12/04 17:59:21 adam + * More work on regular expression conversion. + * + * Revision 1.37 1995/12/04 14:22:27 adam + * Extra arg to recType_byName. + * Started work on new regular expression parsed input to + * structured records. + * + * Revision 1.36 1995/11/30 08:34:29 adam + * Started work on commit facility. + * Changed a few malloc/free to xmalloc/xfree. + * + * Revision 1.35 1995/11/28 14:26:21 adam * Bug fix: recordId with constant wasn't right. * Bug fix: recordId dictionary entry wasn't deleted when needed. * @@ -149,8 +195,6 @@ static int records_inserted = 0; static int records_updated = 0; static int records_deleted = 0; -#define MATCH_DICT "match" - void key_open (int mem) { if (mem < 50000) @@ -162,9 +206,9 @@ void key_open (int mem) key_buf_used = 0; key_file_no = 0; - if (!(matchDict = dict_open (MATCH_DICT, 20, 1))) + if (!(matchDict = dict_open (GMATCH_DICT, 50, 1))) { - logf (LOG_FATAL, "dict_open fail of %s", MATCH_DICT); + logf (LOG_FATAL, "dict_open fail of %s", GMATCH_DICT); exit (1); } assert (!records); @@ -315,10 +359,10 @@ static void addRecordKey (const RecWord *p) { char *b; - b = malloc (reckeys.buf_max += 65000); + b = xmalloc (reckeys.buf_max += 65000); if (reckeys.buf_used > 0) memcpy (b, reckeys.buf, reckeys.buf_used); - free (reckeys.buf); + xfree (reckeys.buf); reckeys.buf = b; } dst = reckeys.buf + reckeys.buf_used; @@ -451,53 +495,71 @@ static void addRecordKeyAny (const RecWord *p) w.attrUse = 1016; addRecordKey (&w); } - addRecordKey (p); + if (p->attrSet != -1) + addRecordKey (p); } -#define FILE_READ_BUFSIZE 4096 - -static char *file_buf; -static int file_offset; -static int file_bufsize; -static int file_noread; +#define FILE_READ_BUFSIZE 4096 +struct file_read_info { + int file_noread; + int fd; +#if FILE_READ_BUFSIZE + char *file_buf; + int file_offset; + int file_bufsize; +#endif +}; -static void file_read_start (int fd) +static struct file_read_info *file_read_start (int fd) { - file_offset = 0; - file_buf = xmalloc (FILE_READ_BUFSIZE); - file_bufsize = read (fd, file_buf, FILE_READ_BUFSIZE); - file_noread = 0; + struct file_read_info *fi = xmalloc (sizeof(*fi)); + + fi->fd = fd; + fi->file_noread = 0; +#if FILE_READ_BUFSIZE + fi->file_offset = 0; + fi->file_buf = xmalloc (FILE_READ_BUFSIZE); + fi->file_bufsize = read (fd, fi->file_buf, FILE_READ_BUFSIZE); +#endif + return fi; } -static void file_read_stop (int fd) +static void file_read_stop (struct file_read_info *fi) { - xfree (file_buf); - file_buf = NULL; + assert (fi); +#if FILE_READ_BUFSIZE + xfree (fi->file_buf); + fi->file_buf = NULL; +#endif + xfree (fi); } -static int file_read (int fd, char *buf, size_t count) +static int file_read (void *handle, char *buf, size_t count) { - int l = file_bufsize - file_offset; + struct file_read_info *p = handle; + int fd = p->fd; +#if FILE_READ_BUFSIZE + int l = p->file_bufsize - p->file_offset; if (count > l) { int r; if (l > 0) - memcpy (buf, file_buf + file_offset, l); + memcpy (buf, p->file_buf + p->file_offset, l); count = count-l; - if (count > file_bufsize) + if (count > FILE_READ_BUFSIZE) { if ((r = read (fd, buf + l, count)) == -1) { logf (LOG_FATAL|LOG_ERRNO, "read"); exit (1); } - file_bufsize = 0; - file_offset = 0; - file_noread += r; - return r; + p->file_bufsize = 0; + p->file_offset = 0; + p->file_noread += l+r; + return l+r; } - file_bufsize = r = read (fd, file_buf, FILE_READ_BUFSIZE); + p->file_bufsize = r = read (fd, p->file_buf, FILE_READ_BUFSIZE); if (r == -1) { logf (LOG_FATAL|LOG_ERRNO, "read"); @@ -505,23 +567,30 @@ static int file_read (int fd, char *buf, size_t count) } else if (r <= count) { - file_offset = r; - memcpy (buf + l, file_buf, r); - file_noread += (l+r); - return l + r; + p->file_offset = r; + memcpy (buf + l, p->file_buf, r); + p->file_noread += l+r; + return l+r; } else { - file_offset = count; - memcpy (buf + l, file_buf, count - l); - file_noread += count; + p->file_offset = count; + memcpy (buf + l, p->file_buf, count - l); + p->file_noread += count; return count; } } - memcpy (buf, file_buf + file_offset, count); - file_offset += count; - file_noread += count; + memcpy (buf, p->file_buf + p->file_offset, count); + p->file_offset += count; + p->file_noread += count; return count; +#else + int r; + r = read (fd, buf, count); + if (r > 0) + p->file_noread += r; + return r; +#endif } static int atois (const char **s) @@ -669,8 +738,8 @@ static char *fileMatchStr (struct recKeys *reckeys, struct recordGroup *rGroup, static int recordExtract (SYSNO *sysno, const char *fname, struct recordGroup *rGroup, int deleteFlag, - int fd, - RecType recType) + struct file_read_info *fi, RecType recType, + char *subType) { struct recExtractCtrl extractCtrl; int r; @@ -678,11 +747,11 @@ static int recordExtract (SYSNO *sysno, const char *fname, SYSNO sysnotmp; Record rec; - if (fd != -1) + if (fi->fd != -1) { - extractCtrl.fd = fd; + extractCtrl.fh = fi; /* extract keys */ - extractCtrl.subType = ""; + extractCtrl.subType = subType; extractCtrl.init = wordInit; extractCtrl.add = addRecordKeyAny; @@ -749,7 +818,7 @@ static int recordExtract (SYSNO *sysno, const char *fname, struct recKeys delkeys; rec = rec_get (records, *sysno); - + assert (rec); delkeys.buf_used = rec->size[recInfo_delKeys]; delkeys.buf = rec->info[recInfo_delKeys]; flushRecordKeys (*sysno, 0, &delkeys, rec->info[recInfo_databaseName]); @@ -757,15 +826,15 @@ static int recordExtract (SYSNO *sysno, const char *fname, { if (!delkeys.buf_used) { - logf (LOG_WARN, "cannot delete %s - no delete keys", fname); + logf (LOG_WARN, "cannot delete %s: storeKeys false", + fname); } else { - SYSNO sysnoz = 0; logf (LOG_LOG, "delete %s %s", rGroup->recordType, fname); records_deleted++; if (matchStr) - dict_insert (matchDict, matchStr, sizeof(sysnoz), &sysnoz); + dict_delete (matchDict, matchStr); rec_del (records, &rec); } return 1; @@ -774,7 +843,7 @@ static int recordExtract (SYSNO *sysno, const char *fname, { if (!delkeys.buf_used) { - logf (LOG_WARN, "cannot update %s - no delete keys", + logf (LOG_WARN, "cannot update %s: storeKeys false", fname); } else @@ -786,18 +855,18 @@ static int recordExtract (SYSNO *sysno, const char *fname, } } } - free (rec->info[recInfo_fileType]); + xfree (rec->info[recInfo_fileType]); rec->info[recInfo_fileType] = rec_strdup (rGroup->recordType, &rec->size[recInfo_fileType]); - free (rec->info[recInfo_filename]); + xfree (rec->info[recInfo_filename]); rec->info[recInfo_filename] = rec_strdup (fname, &rec->size[recInfo_filename]); - free (rec->info[recInfo_delKeys]); + xfree (rec->info[recInfo_delKeys]); if (reckeys.buf_used > 0 && rGroup->flagStoreKeys == 1) { - rec->info[recInfo_delKeys] = malloc (reckeys.buf_used); + rec->info[recInfo_delKeys] = xmalloc (reckeys.buf_used); rec->size[recInfo_delKeys] = reckeys.buf_used; memcpy (rec->info[recInfo_delKeys], reckeys.buf, rec->size[recInfo_delKeys]); @@ -808,25 +877,28 @@ static int recordExtract (SYSNO *sysno, const char *fname, rec->size[recInfo_delKeys] = 0; } - free (rec->info[recInfo_storeData]); + xfree (rec->info[recInfo_storeData]); if (rGroup->flagStoreData == 1) { - rec->size[recInfo_storeData] = file_noread; - rec->info[recInfo_storeData] = malloc (file_noread); - if (file_noread < FILE_READ_BUFSIZE) - memcpy (rec->info[recInfo_storeData], file_buf, file_noread); + rec->size[recInfo_storeData] = fi->file_noread; + rec->info[recInfo_storeData] = xmalloc (fi->file_noread); +#if FILE_READ_BUFSIZE + if (fi->file_noread < FILE_READ_BUFSIZE) + memcpy (rec->info[recInfo_storeData], fi->file_buf, + fi->file_noread); else +#endif { - if (lseek (fd, 0L, SEEK_SET) < 0) + if (lseek (fi->fd, 0L, SEEK_SET) < 0) { logf (LOG_ERRNO|LOG_FATAL, "seek to 0 in %s", fname); exit (1); } - if (read (fd, rec->info[recInfo_storeData], file_noread) - < file_noread) + if (read (fi->fd, rec->info[recInfo_storeData], fi->file_noread) + < fi->file_noread) { logf (LOG_ERRNO|LOG_FATAL, "read %d bytes of %s", - file_noread, fname); + fi->file_noread, fname); exit (1); } } @@ -836,7 +908,7 @@ static int recordExtract (SYSNO *sysno, const char *fname, rec->info[recInfo_storeData] = NULL; rec->size[recInfo_storeData] = 0; } - free (rec->info[recInfo_databaseName]); + xfree (rec->info[recInfo_databaseName]); rec->info[recInfo_databaseName] = rec_strdup (rGroup->databaseName, &rec->size[recInfo_databaseName]); @@ -851,9 +923,11 @@ int fileExtract (SYSNO *sysno, const char *fname, char gprefix[128]; char ext[128]; char ext_res[128]; + char subType[128]; RecType recType; struct recordGroup rGroupM; struct recordGroup *rGroup = &rGroupM; + struct file_read_info *fi; memcpy (rGroup, rGroupP, sizeof(*rGroupP)); @@ -895,7 +969,7 @@ int fileExtract (SYSNO *sysno, const char *fname, logf (LOG_LOG, "? record %s", fname); return 0; } - if (!(recType = recType_byName (rGroup->recordType))) + if (!(recType = recType_byName (rGroup->recordType, subType))) { logf (LOG_WARN, "No such record type: %s", rGroup->recordType); return 0; @@ -962,9 +1036,9 @@ int fileExtract (SYSNO *sysno, const char *fname, return 0; } } - file_read_start (fd); - recordExtract (sysno, fname, rGroup, deleteFlag, fd, recType); - file_read_stop (fd); + fi = file_read_start (fd); + recordExtract (sysno, fname, rGroup, deleteFlag, fi, recType, subType); + file_read_stop (fi); if (fd != -1) close (fd); return 1;