X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fextract.c;h=479e8bdd6988c2b0ac5a3c04871d963828a49694;hb=99e4ce02beb2ea748b9970ef35fe8834f04a1528;hp=ca6ce2131cb3820409d31a91bf8da3cb41809613;hpb=6024b7c6a621b79c20953aeef15a17b96af92a8b;p=idzebra-moved-to-github.git

diff --git a/index/extract.c b/index/extract.c
index ca6ce21..479e8bd 100644
--- a/index/extract.c
+++ b/index/extract.c
@@ -4,7 +4,64 @@
  * Sebastian Hammer, Adam Dickmeiss
  *
  * $Log: extract.c,v $
- * Revision 1.39  1995-12-05 13:20:18  adam
+ * Revision 1.54  1996-05-01 13:46:35  adam
+ * First work on multiple records in one file.
+ * New option, -offset, to the "unread" command in the filter module.
+ *
+ * Revision 1.53  1996/04/26  12:09:43  adam
+ * Added a few comments.
+ *
+ * Revision 1.52  1996/04/25  13:27:57  adam
+ * Function recordExtract modified so that files with no keys (possibly empty)
+ * are ignored.
+ *
+ * Revision 1.51  1996/03/19  11:08:42  adam
+ * Bug fix: Log preamble wasn't always turned off after recordExtract.
+ *
+ * Revision 1.50  1996/02/12  18:45:36  adam
+ * New fileVerboseFlag in record group control.
+ *
+ * Revision 1.49  1996/02/05  12:29:57  adam
+ * Logging reduced a bit.
+ * The remaining running time is estimated during register merge.
+ *
+ * Revision 1.48  1996/02/01  20:53:26  adam
+ * The temporary per-record keys are compacted a little, and duplication
+ * of the per-records keys are avoided when they are saved in the record
+ * information buffer.
+ *
+ * Revision 1.47  1996/01/17  14:57:48  adam
+ * Prototype changed for reader functions in extract/retrieve. File
+ *  is identified by 'void *' instead of 'int.
+ *
+ * Revision 1.46  1995/12/15  14:57:16  adam
+ * Bug fix.
+ *
+ * Revision 1.45  1995/12/15  12:37:41  adam
+ * In addRecordKeyAny: Writes key only when attrSet != -1.
+ *
+ * Revision 1.44  1995/12/12  16:00:54  adam
+ * System call sync(2) used after update/commit.
+ * Locking (based on fcntl) uses F_EXLCK and F_SHLCK instead of F_WRLCK
+ * and F_RDLCK.
+ *
+ * Revision 1.43  1995/12/11  09:12:46  adam
+ * The rec_get function returns NULL if record doesn't exist - will
+ * happen in the server if the result set records have been deleted since
+ * the creation of the set (i.e. the search).
+ * The server saves a result temporarily if it is 'volatile', i.e. the
+ * set is register dependent.
+ *
+ * Revision 1.42  1995/12/07  17:38:46  adam
+ * Work locking mechanisms for concurrent updates/commit.
+ *
+ * Revision 1.41  1995/12/06  16:06:42  adam
+ * Better diagnostics. Work on 'real' dictionary deletion.
+ *
+ * Revision 1.40  1995/12/05  16:57:40  adam
+ * More work on regular patterns.
+ *
+ * Revision 1.39  1995/12/05  13:20:18  adam
  * Bug fix: file_read sometimes returned early EOF.
  *
  * Revision 1.38  1995/12/04  17:59:21  adam
@@ -163,8 +220,19 @@ static int key_file_no;
 static int records_inserted = 0;
 static int records_updated = 0;
 static int records_deleted = 0;
+static int records_processed = 0;
 
-#define MATCH_DICT "match"
+static void logRecord (int showFlag)
+{
+    if (!showFlag)
+        ++records_processed;
+    if (showFlag || !(records_processed % 1000))
+    {
+        logf (LOG_LOG, "Records: %7d i/u/d %d/%d/%d", 
+              records_processed, records_inserted, records_updated,
+              records_deleted);
+    }
+}
 
 void key_open (int mem)
 {
@@ -177,9 +245,9 @@ void key_open (int mem)
     key_buf_used = 0;
     key_file_no = 0;
 
-    if (!(matchDict = dict_open (MATCH_DICT, 20, 1)))
+    if (!(matchDict = dict_open (GMATCH_DICT, 50, 1)))
     {
-        logf (LOG_FATAL, "dict_open fail of %s", MATCH_DICT);
+        logf (LOG_FATAL, "dict_open fail of %s", GMATCH_DICT);
         exit (1);
     }
     assert (!records);
@@ -300,9 +368,7 @@ int key_close (void)
     rec_close (&records);
     dict_close (matchDict);
 
-    logf (LOG_LOG, "Records inserted %6d", records_inserted);
-    logf (LOG_LOG, "Records updated  %6d", records_updated);
-    logf (LOG_LOG, "Records deleted  %6d", records_deleted);
+    logRecord (1);
     return key_file_no;
 }
 
@@ -317,6 +383,8 @@ struct recKeys {
     int buf_used;
     int buf_max;
     char *buf;
+    char prevAttrSet;
+    short prevAttrUse;
 } reckeys;
 
 static void addRecordKey (const RecWord *p)
@@ -325,29 +393,46 @@ static void addRecordKey (const RecWord *p)
     char attrSet;
     short attrUse;
     size_t i;
+    int lead = 0;
 
     if (reckeys.buf_used+1024 > reckeys.buf_max)
     {
         char *b;
 
-        b = xmalloc (reckeys.buf_max += 65000);
+        b = xmalloc (reckeys.buf_max += 128000);
         if (reckeys.buf_used > 0)
             memcpy (b, reckeys.buf, reckeys.buf_used);
         xfree (reckeys.buf);
         reckeys.buf = b;
     }
     dst = reckeys.buf + reckeys.buf_used;
+
+    attrSet = p->attrSet;
+    if (reckeys.buf_used > 0 && reckeys.prevAttrSet == attrSet)
+        lead |= 1;
+    else
+        reckeys.prevAttrSet = attrSet;
+    attrUse = p->attrUse;
+    if (reckeys.buf_used > 0 && reckeys.prevAttrUse == attrUse)
+        lead |= 2;
+    else
+        reckeys.prevAttrUse = attrUse;
+
     switch (p->which)
     {
     case Word_String:
-        attrSet = p->attrSet;
-        memcpy (dst, &attrSet, sizeof(attrSet));
-        dst += sizeof(attrSet);
+        *dst++ = lead;
 
-        attrUse = p->attrUse;
-        memcpy (dst, &attrUse, sizeof(attrUse));
-        dst += sizeof(attrUse);
-        
+        if (!(lead & 1))
+        {
+            memcpy (dst, &attrSet, sizeof(attrSet));
+            dst += sizeof(attrSet);
+        }
+        if (!(lead & 2))
+        {
+            memcpy (dst, &attrUse, sizeof(attrUse));
+            dst += sizeof(attrUse);
+        }
         for (i = 0; p->u.string[i]; i++)
             *dst++ = p->u.string[i];
         *dst++ = '\0';
@@ -365,20 +450,27 @@ static void addRecordKey (const RecWord *p)
 static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys, 
                              const char *databaseName)
 {
+    char attrSet = -1;
+    short attrUse = -1;
     int off = 0;
     while (off < reckeys->buf_used)
     {
         const char *src = reckeys->buf + off;
-        char attrSet;
-        short attrUse;
         struct it_key key;
-        
-        memcpy (&attrSet, src, sizeof(attrSet));
-        src += sizeof(attrSet);
-
-        memcpy (&attrUse, src, sizeof(attrUse));
-        src += sizeof(attrUse);
+        int lead;
+    
+        lead = *src++;
 
+        if (!(lead & 1))
+        {
+            memcpy (&attrSet, src, sizeof(attrSet));
+            src += sizeof(attrSet);
+        }
+        if (!(lead & 2))
+        {
+            memcpy (&attrUse, src, sizeof(attrUse));
+            src += sizeof(attrUse);
+        }
         if (key_buf_used + 1024 > (ptr_top-ptr_i)*sizeof(char*))
             key_flush ();
         ++ptr_i;
@@ -466,92 +558,73 @@ static void addRecordKeyAny (const RecWord *p)
         w.attrUse = 1016;
         addRecordKey (&w);
     }
-    addRecordKey (p);
+    if (p->attrSet != -1)
+        addRecordKey (p);
 }
 
+struct file_read_info {
+    off_t file_max;
+    off_t file_offset;
+    off_t file_moffset;
+    int file_more;
+    int fd;
+};
 
-#define FILE_READ_BUFSIZE 4096
+static struct file_read_info *file_read_start (int fd)
+{
+    struct file_read_info *fi = xmalloc (sizeof(*fi));
 
-static int file_noread;
-#if FILE_READ_BUFSIZE
-static char *file_buf;
-static int file_offset;
-static int file_bufsize;
-#endif
+    fi->fd = fd;
+    fi->file_max = 0;
+    fi->file_moffset = 0;
+    return fi;
+}
 
-static void file_read_start (int fd)
+static void file_read_stop (struct file_read_info *fi)
 {
-    file_noread = 0;
-#if FILE_READ_BUFSIZE
-    file_offset = 0;
-    file_buf = xmalloc (FILE_READ_BUFSIZE);
-    file_bufsize = read (fd, file_buf, FILE_READ_BUFSIZE);
-#endif
+    assert (fi);
+    xfree (fi);
 }
 
-static void file_read_stop (int fd)
+static off_t file_seek (void *handle, off_t offset)
 {
-#if FILE_READ_BUFSIZE
-    xfree (file_buf);
-    file_buf = NULL;
-#endif
+    struct file_read_info *p = handle;
+    p->file_offset = offset;
+    return lseek (p->fd, offset, SEEK_SET);
 }
 
-static int file_read (int fd, char *buf, size_t count)
+static int file_read (void *handle, char *buf, size_t count)
 {
-#if FILE_READ_BUFSIZE
-    int l = file_bufsize - file_offset;
-
-    if (count > l)
-    {
-        int r;
-        if (l > 0)
-            memcpy (buf, file_buf + file_offset, l);
-        count = count-l;
-        if (count > FILE_READ_BUFSIZE)
-        {
-            if ((r = read (fd, buf + l, count)) == -1)
-            {
-                logf (LOG_FATAL|LOG_ERRNO, "read");
-                exit (1);
-            }
-            file_bufsize = 0;
-            file_offset = 0;
-            file_noread += l+r;
-            return l+r;
-        }
-        file_bufsize = r = read (fd, file_buf, FILE_READ_BUFSIZE);
-        if (r == -1)
-        {
-            logf (LOG_FATAL|LOG_ERRNO, "read");
-            exit (1);
-        }
-        else if (r <= count)
-        {
-            file_offset = r;
-            memcpy (buf + l, file_buf, r);
-            file_noread += (l+r);
-            return l + r;
-        }
-        else
-        {
-            file_offset = count;
-            memcpy (buf + l, file_buf, count - l);
-            file_noread += count;
-            return count;
-        }
-    }
-    memcpy (buf, file_buf + file_offset, count);
-    file_offset += count;
-    file_noread += count;
-    return count;
-#else
+    struct file_read_info *p = handle;
+    int fd = p->fd;
     int r;
     r = read (fd, buf, count);
     if (r > 0)
-        file_noread += r;
+    {
+        p->file_offset += r;
+        if (p->file_offset > p->file_max)
+            p->file_max = p->file_offset;
+    }
     return r;
-#endif
+}
+
+static void file_begin (void *handle)
+{
+    struct file_read_info *p = handle;
+
+    p->file_offset = p->file_moffset;
+    if (p->file_moffset)
+        lseek (p->fd, p->file_moffset, SEEK_SET);
+    p->file_more = 0;
+}
+
+static void file_end (void *handle, off_t offset)
+{
+    struct file_read_info *p = handle;
+
+    assert (p->file_more == 0);
+    p->file_more = 1;
+    p->file_moffset = offset;
 }
 
 static int atois (const char **s)
@@ -626,8 +699,8 @@ static char *fileMatchStr (struct recKeys *reckeys, struct recordGroup *rGroup,
                 }
             if (first)
             {
-                logf (LOG_WARN, "Record in file %s didn't contain match"
-                      " fields in (%d,%d)", fname, attrSet, attrUse);
+                logf (LOG_WARN, "Record didn't contain match"
+                      " fields in (%d,%d)", attrSet, attrUse);
                 return NULL;
             }
         }
@@ -697,33 +770,75 @@ static char *fileMatchStr (struct recKeys *reckeys, struct recordGroup *rGroup,
     return dstBuf;
 }
 
+struct recordLogInfo {
+    const char *fname;
+    char *op;
+    struct recordGroup *rGroup;
+};
+     
+static void recordLogPreamble (int level, const char *msg, void *info)
+{
+    struct recordLogInfo *p = info;
+    FILE *outf = log_file ();
+
+    if (level & LOG_LOG)
+        return ;
+    if (p->op) 
+        fprintf (outf, "%s of ", p->op);
+    fprintf (outf, "%s type %s\n", p->rGroup->recordType, p->fname);
+    log_event_start (NULL, NULL);
+}
+
 static int recordExtract (SYSNO *sysno, const char *fname,
                           struct recordGroup *rGroup, int deleteFlag,
-                          int fd, RecType recType, char *subType)
+                          struct file_read_info *fi, RecType recType,
+                          char *subType)
 {
     struct recExtractCtrl extractCtrl;
     int r;
     char *matchStr;
     SYSNO sysnotmp;
+    off_t recordOffset = 0;
     Record rec;
+    struct recordLogInfo logInfo;
 
-    if (fd != -1)
+    logInfo.fname = fname;
+    logInfo.op = NULL;
+    logInfo.rGroup = rGroup;
+    log_event_start (recordLogPreamble, &logInfo);
+    
+    if (fi->fd != -1)
     {
-        extractCtrl.fd = fd;
-        /* extract keys */
+        /* we are going to read from a file, so prepare the extraction */
+        extractCtrl.fh = fi;
         extractCtrl.subType = subType;
         extractCtrl.init = wordInit;
         extractCtrl.add = addRecordKeyAny;
 
         reckeys.buf_used = 0;
+        reckeys.prevAttrUse = -1;
+        reckeys.prevAttrSet = -1;
+
+        recordOffset = fi->file_moffset;
+        extractCtrl.offset = recordOffset;
         extractCtrl.readf = file_read;
+        extractCtrl.seekf = file_seek;
+        extractCtrl.endf = file_end;
         r = (*recType->extract)(&extractCtrl);
-  
+
         if (r)      
         {
+            /* error occured during extraction ... */
             logf (LOG_WARN, "Couldn't extract file %s, code %d", fname, r);
             return 0;
         }
+        if (reckeys.buf_used == 0)
+        {
+            /* the extraction process returned no information - the record
+               is probably empty */
+            logf (LOG_WARN, "Empty file %s", fname);
+            return 0;
+        }
     }
 
     /* perform match if sysno not known and if match criteria is specified */
@@ -747,21 +862,24 @@ static int recordExtract (SYSNO *sysno, const char *fname,
             }
             else
             {
-                logf (LOG_WARN, "Record not inserted");
+                logf (LOG_WARN, "Bad match criteria");
                 return 0;
             }
         }
     }
 
-    /* new record ? */
     if (! *sysno)
     {
+        /* new record */
         if (deleteFlag)
         {
-            logf (LOG_LOG, "? %s", fname);
+            logf (LOG_LOG, "Cannot delete new record");
             return 1;
         }
-        logf (LOG_LOG, "add %s %s", rGroup->recordType, fname);
+        logInfo.op = "add";
+        if (rGroup->fileVerboseFlag)
+            logf (LOG_LOG, "add %s %s %ld", rGroup->recordType,
+                  fname, (long) recordOffset);
         rec = rec_new (records);
         *sysno = rec->sysno;
 
@@ -775,61 +893,78 @@ static int recordExtract (SYSNO *sysno, const char *fname,
     }
     else
     {
+        /* record already exists */
         struct recKeys delkeys;
 
         rec = rec_get (records, *sysno);
-
+        assert (rec);
         delkeys.buf_used = rec->size[recInfo_delKeys];
 	delkeys.buf = rec->info[recInfo_delKeys];
         flushRecordKeys (*sysno, 0, &delkeys, rec->info[recInfo_databaseName]);
         if (deleteFlag)
         {
+            /* record going to be deleted */
+            logInfo.op = "delete";
             if (!delkeys.buf_used)
             {
-                logf (LOG_WARN, "cannot delete %s - no delete keys", fname);
+                logf (LOG_WARN, "cannot delete; storeKeys false");
             }
             else
             {
-                SYSNO sysnoz = 0;
-                logf (LOG_LOG, "delete %s %s", rGroup->recordType, fname);
+                if (rGroup->fileVerboseFlag)
+                    logf (LOG_LOG, "delete %s %s %ld", rGroup->recordType,
+                          fname, (long) recordOffset);
                 records_deleted++;
                 if (matchStr)
-                    dict_insert (matchDict, matchStr, sizeof(sysnoz), &sysnoz);
+                    dict_delete (matchDict, matchStr);
                 rec_del (records, &rec);
             }
+            logRecord (0);
             return 1;
         }
         else
         {
+            /* record going to be updated */
+            logInfo.op = "update";
             if (!delkeys.buf_used)
             {
-                logf (LOG_WARN, "cannot update %s - no delete keys",
-                      fname);
+                logf (LOG_WARN, "cannot update; storeKeys false");
             }
             else
             {
-                logf (LOG_LOG, "update %s %s", rGroup->recordType,
-                      fname);
+                if (rGroup->fileVerboseFlag)
+                    logf (LOG_LOG, "update %s %s %ld", rGroup->recordType,
+                          fname, (long) recordOffset);
                 flushRecordKeys (*sysno, 1, &reckeys, rGroup->databaseName); 
                 records_updated++;
             }
         }
     }
+    /* update file type */
     xfree (rec->info[recInfo_fileType]);
     rec->info[recInfo_fileType] =
         rec_strdup (rGroup->recordType, &rec->size[recInfo_fileType]);
 
+    /* update filename */
     xfree (rec->info[recInfo_filename]);
     rec->info[recInfo_filename] =
         rec_strdup (fname, &rec->size[recInfo_filename]);
 
+    /* update delete keys */
     xfree (rec->info[recInfo_delKeys]);
     if (reckeys.buf_used > 0 && rGroup->flagStoreKeys == 1)
     {
+#if 1
+        rec->size[recInfo_delKeys] = reckeys.buf_used;
+        rec->info[recInfo_delKeys] = reckeys.buf;
+        reckeys.buf = NULL;
+        reckeys.buf_max = 0;
+#else
         rec->info[recInfo_delKeys] = xmalloc (reckeys.buf_used);
         rec->size[recInfo_delKeys] = reckeys.buf_used;
         memcpy (rec->info[recInfo_delKeys], reckeys.buf,
                 rec->size[recInfo_delKeys]);
+#endif
     }
     else
     {
@@ -837,29 +972,24 @@ static int recordExtract (SYSNO *sysno, const char *fname,
         rec->size[recInfo_delKeys] = 0;
     }
 
+    /* update store data */
     xfree (rec->info[recInfo_storeData]);
     if (rGroup->flagStoreData == 1)
     {
-        rec->size[recInfo_storeData] = file_noread;
-        rec->info[recInfo_storeData] = xmalloc (file_noread);
-#if FILE_READ_BUFSIZE
-        if (file_noread < FILE_READ_BUFSIZE)
-	    memcpy (rec->info[recInfo_storeData], file_buf, file_noread);
-        else
-#endif
+        rec->size[recInfo_storeData] = fi->file_max;
+        rec->info[recInfo_storeData] = xmalloc (fi->file_max);
+        if (lseek (fi->fd, recordOffset, SEEK_SET) < 0)
         {
-            if (lseek (fd, 0L, SEEK_SET) < 0)
-            {
-                logf (LOG_ERRNO|LOG_FATAL, "seek to 0 in %s", fname);
-                exit (1);
-            }
-            if (read (fd, rec->info[recInfo_storeData], file_noread) 
-                < file_noread)
-            {
-                logf (LOG_ERRNO|LOG_FATAL, "read %d bytes of %s",
-                      file_noread, fname);
-                exit (1);
-            }
+            logf (LOG_ERRNO|LOG_FATAL, "seek to %ld in %s", fname,
+                  (long) recordOffset);
+            exit (1);
+        }
+        if (read (fi->fd, rec->info[recInfo_storeData], fi->file_max)
+            < fi->file_max)
+        {
+            logf (LOG_ERRNO|LOG_FATAL, "read %d bytes of %s",
+                  fi->file_max, fname);
+            exit (1);
         }
     }
     else
@@ -867,18 +997,28 @@ static int recordExtract (SYSNO *sysno, const char *fname,
         rec->info[recInfo_storeData] = NULL;
         rec->size[recInfo_storeData] = 0;
     }
+    /* update database name */
     xfree (rec->info[recInfo_databaseName]);
     rec->info[recInfo_databaseName] =
         rec_strdup (rGroup->databaseName, &rec->size[recInfo_databaseName]); 
 
+    /* update offset */
+    xfree (rec->info[recInfo_offset]);
+
+    rec->size[recInfo_offset] = sizeof(recordOffset);
+    rec->info[recInfo_offset] = xmalloc (sizeof(recordOffset));
+    memcpy (rec->info[recInfo_offset], &recordOffset, sizeof(recordOffset));
+    
+    /* commit this record */
     rec_put (records, &rec);
+    logRecord (0);
     return 1;
 }
 
 int fileExtract (SYSNO *sysno, const char *fname, 
                  const struct recordGroup *rGroupP, int deleteFlag)
 {
-    int i, fd;
+    int r, i, fd;
     char gprefix[128];
     char ext[128];
     char ext_res[128];
@@ -886,6 +1026,7 @@ int fileExtract (SYSNO *sysno, const char *fname,
     RecType recType;
     struct recordGroup rGroupM;
     struct recordGroup *rGroup = &rGroupM;
+    struct file_read_info *fi;
 
     memcpy (rGroup, rGroupP, sizeof(*rGroupP));
    
@@ -917,14 +1058,16 @@ int fileExtract (SYSNO *sysno, const char *fname,
             sprintf (ext_res, "%srecordType", gprefix);
             if (!(rGroup->recordType = res_get (common_resource, ext_res)))
             {
-                logf (LOG_LOG, "? %s", fname);
+                if (rGroup->fileVerboseFlag)
+                    logf (LOG_LOG, "? %s", fname);
                 return 0;
             }
         }
     }
     if (!rGroup->recordType)
     {
-        logf (LOG_LOG, "? record %s", fname);
+        if (rGroup->fileVerboseFlag)
+            logf (LOG_LOG, "? record %s", fname);
         return 0;
     }
     if (!(recType = recType_byName (rGroup->recordType, subType)))
@@ -994,11 +1137,17 @@ int fileExtract (SYSNO *sysno, const char *fname,
             return 0;
         }
     }
-    file_read_start (fd);
-    recordExtract (sysno, fname, rGroup, deleteFlag, fd, recType, subType);
-    file_read_stop (fd);
+    fi = file_read_start (fd);
+    do
+    {
+        file_begin (fi);
+        r = recordExtract (sysno, fname, rGroup, deleteFlag, fi,
+                           recType, subType);
+    } while (r && !sysno && fi->file_more);
+    log_event_start (NULL, NULL);
+    file_read_stop (fi);
     if (fd != -1)
         close (fd);
-    return 1;
+    return r;
 }