X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fextract.c;h=798354a384f7bb1a92cf26ef7386ec373ca6738e;hb=589b5761fb59634afc37909e62cb34ce56cbbb12;hp=ca6ce2131cb3820409d31a91bf8da3cb41809613;hpb=6024b7c6a621b79c20953aeef15a17b96af92a8b;p=idzebra-moved-to-github.git

diff --git a/index/extract.c b/index/extract.c
index ca6ce21..798354a 100644
--- a/index/extract.c
+++ b/index/extract.c
@@ -4,7 +4,47 @@
  * Sebastian Hammer, Adam Dickmeiss
  *
  * $Log: extract.c,v $
- * Revision 1.39  1995-12-05 13:20:18  adam
+ * Revision 1.49  1996-02-05 12:29:57  adam
+ * Logging reduced a bit.
+ * The remaining running time is estimated during register merge.
+ *
+ * Revision 1.48  1996/02/01  20:53:26  adam
+ * The temporary per-record keys are compacted a little, and duplication
+ * of the per-records keys are avoided when they are saved in the record
+ * information buffer.
+ *
+ * Revision 1.47  1996/01/17  14:57:48  adam
+ * Prototype changed for reader functions in extract/retrieve. File
+ *  is identified by 'void *' instead of 'int.
+ *
+ * Revision 1.46  1995/12/15  14:57:16  adam
+ * Bug fix.
+ *
+ * Revision 1.45  1995/12/15  12:37:41  adam
+ * In addRecordKeyAny: Writes key only when attrSet != -1.
+ *
+ * Revision 1.44  1995/12/12  16:00:54  adam
+ * System call sync(2) used after update/commit.
+ * Locking (based on fcntl) uses F_EXLCK and F_SHLCK instead of F_WRLCK
+ * and F_RDLCK.
+ *
+ * Revision 1.43  1995/12/11  09:12:46  adam
+ * The rec_get function returns NULL if record doesn't exist - will
+ * happen in the server if the result set records have been deleted since
+ * the creation of the set (i.e. the search).
+ * The server saves a result temporarily if it is 'volatile', i.e. the
+ * set is register dependent.
+ *
+ * Revision 1.42  1995/12/07  17:38:46  adam
+ * Work locking mechanisms for concurrent updates/commit.
+ *
+ * Revision 1.41  1995/12/06  16:06:42  adam
+ * Better diagnostics. Work on 'real' dictionary deletion.
+ *
+ * Revision 1.40  1995/12/05  16:57:40  adam
+ * More work on regular patterns.
+ *
+ * Revision 1.39  1995/12/05  13:20:18  adam
  * Bug fix: file_read sometimes returned early EOF.
  *
  * Revision 1.38  1995/12/04  17:59:21  adam
@@ -164,8 +204,6 @@ static int records_inserted = 0;
 static int records_updated = 0;
 static int records_deleted = 0;
 
-#define MATCH_DICT "match"
-
 void key_open (int mem)
 {
     if (mem < 50000)
@@ -177,9 +215,9 @@ void key_open (int mem)
     key_buf_used = 0;
     key_file_no = 0;
 
-    if (!(matchDict = dict_open (MATCH_DICT, 20, 1)))
+    if (!(matchDict = dict_open (GMATCH_DICT, 50, 1)))
     {
-        logf (LOG_FATAL, "dict_open fail of %s", MATCH_DICT);
+        logf (LOG_FATAL, "dict_open fail of %s", GMATCH_DICT);
         exit (1);
     }
     assert (!records);
@@ -317,6 +355,8 @@ struct recKeys {
     int buf_used;
     int buf_max;
     char *buf;
+    char prevAttrSet;
+    short prevAttrUse;
 } reckeys;
 
 static void addRecordKey (const RecWord *p)
@@ -325,29 +365,46 @@ static void addRecordKey (const RecWord *p)
     char attrSet;
     short attrUse;
     size_t i;
+    int lead = 0;
 
     if (reckeys.buf_used+1024 > reckeys.buf_max)
     {
         char *b;
 
-        b = xmalloc (reckeys.buf_max += 65000);
+        b = xmalloc (reckeys.buf_max += 128000);
         if (reckeys.buf_used > 0)
             memcpy (b, reckeys.buf, reckeys.buf_used);
         xfree (reckeys.buf);
         reckeys.buf = b;
     }
     dst = reckeys.buf + reckeys.buf_used;
+
+    attrSet = p->attrSet;
+    if (reckeys.buf_used > 0 && reckeys.prevAttrSet == attrSet)
+        lead |= 1;
+    else
+        reckeys.prevAttrSet = attrSet;
+    attrUse = p->attrUse;
+    if (reckeys.buf_used > 0 && reckeys.prevAttrUse == attrUse)
+        lead |= 2;
+    else
+        reckeys.prevAttrUse = attrUse;
+
     switch (p->which)
     {
     case Word_String:
-        attrSet = p->attrSet;
-        memcpy (dst, &attrSet, sizeof(attrSet));
-        dst += sizeof(attrSet);
+        *dst++ = lead;
 
-        attrUse = p->attrUse;
-        memcpy (dst, &attrUse, sizeof(attrUse));
-        dst += sizeof(attrUse);
-        
+        if (!(lead & 1))
+        {
+            memcpy (dst, &attrSet, sizeof(attrSet));
+            dst += sizeof(attrSet);
+        }
+        if (!(lead & 2))
+        {
+            memcpy (dst, &attrUse, sizeof(attrUse));
+            dst += sizeof(attrUse);
+        }
         for (i = 0; p->u.string[i]; i++)
             *dst++ = p->u.string[i];
         *dst++ = '\0';
@@ -365,20 +422,27 @@ static void addRecordKey (const RecWord *p)
 static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys, 
                              const char *databaseName)
 {
+    char attrSet = -1;
+    short attrUse = -1;
     int off = 0;
     while (off < reckeys->buf_used)
     {
         const char *src = reckeys->buf + off;
-        char attrSet;
-        short attrUse;
         struct it_key key;
-        
-        memcpy (&attrSet, src, sizeof(attrSet));
-        src += sizeof(attrSet);
-
-        memcpy (&attrUse, src, sizeof(attrUse));
-        src += sizeof(attrUse);
+        int lead;
+    
+        lead = *src++;
 
+        if (!(lead & 1))
+        {
+            memcpy (&attrSet, src, sizeof(attrSet));
+            src += sizeof(attrSet);
+        }
+        if (!(lead & 2))
+        {
+            memcpy (&attrUse, src, sizeof(attrUse));
+            src += sizeof(attrUse);
+        }
         if (key_buf_used + 1024 > (ptr_top-ptr_i)*sizeof(char*))
             key_flush ();
         ++ptr_i;
@@ -466,47 +530,57 @@ static void addRecordKeyAny (const RecWord *p)
         w.attrUse = 1016;
         addRecordKey (&w);
     }
-    addRecordKey (p);
+    if (p->attrSet != -1)
+        addRecordKey (p);
 }
 
-
 #define FILE_READ_BUFSIZE 4096
-
-static int file_noread;
+struct file_read_info {
+    int file_noread;
+    int fd;
 #if FILE_READ_BUFSIZE
-static char *file_buf;
-static int file_offset;
-static int file_bufsize;
+    char *file_buf;
+    int file_offset;
+    int file_bufsize;
 #endif
+};
 
-static void file_read_start (int fd)
+static struct file_read_info *file_read_start (int fd)
 {
-    file_noread = 0;
+    struct file_read_info *fi = xmalloc (sizeof(*fi));
+
+    fi->fd = fd;
+    fi->file_noread = 0;
 #if FILE_READ_BUFSIZE
-    file_offset = 0;
-    file_buf = xmalloc (FILE_READ_BUFSIZE);
-    file_bufsize = read (fd, file_buf, FILE_READ_BUFSIZE);
+    fi->file_offset = 0;
+    fi->file_buf = xmalloc (FILE_READ_BUFSIZE);
+    fi->file_bufsize = read (fd, fi->file_buf, FILE_READ_BUFSIZE);
 #endif
+    return fi;
 }
 
-static void file_read_stop (int fd)
+static void file_read_stop (struct file_read_info *fi)
 {
+    assert (fi);
 #if FILE_READ_BUFSIZE
-    xfree (file_buf);
-    file_buf = NULL;
+    xfree (fi->file_buf);
+    fi->file_buf = NULL;
 #endif
+    xfree (fi);
 }
 
-static int file_read (int fd, char *buf, size_t count)
+static int file_read (void *handle, char *buf, size_t count)
 {
+    struct file_read_info *p = handle;
+    int fd = p->fd;
 #if FILE_READ_BUFSIZE
-    int l = file_bufsize - file_offset;
+    int l = p->file_bufsize - p->file_offset;
 
     if (count > l)
     {
         int r;
         if (l > 0)
-            memcpy (buf, file_buf + file_offset, l);
+            memcpy (buf, p->file_buf + p->file_offset, l);
         count = count-l;
         if (count > FILE_READ_BUFSIZE)
         {
@@ -515,12 +589,12 @@ static int file_read (int fd, char *buf, size_t count)
                 logf (LOG_FATAL|LOG_ERRNO, "read");
                 exit (1);
             }
-            file_bufsize = 0;
-            file_offset = 0;
-            file_noread += l+r;
+            p->file_bufsize = 0;
+            p->file_offset = 0;
+            p->file_noread += l+r;
             return l+r;
         }
-        file_bufsize = r = read (fd, file_buf, FILE_READ_BUFSIZE);
+        p->file_bufsize = r = read (fd, p->file_buf, FILE_READ_BUFSIZE);
         if (r == -1)
         {
             logf (LOG_FATAL|LOG_ERRNO, "read");
@@ -528,28 +602,28 @@ static int file_read (int fd, char *buf, size_t count)
         }
         else if (r <= count)
         {
-            file_offset = r;
-            memcpy (buf + l, file_buf, r);
-            file_noread += (l+r);
-            return l + r;
+            p->file_offset = r;
+            memcpy (buf + l, p->file_buf, r);
+            p->file_noread += l+r;
+            return l+r;
         }
         else
         {
-            file_offset = count;
-            memcpy (buf + l, file_buf, count - l);
-            file_noread += count;
+            p->file_offset = count;
+            memcpy (buf + l, p->file_buf, count - l);
+            p->file_noread += count;
             return count;
         }
     }
-    memcpy (buf, file_buf + file_offset, count);
-    file_offset += count;
-    file_noread += count;
+    memcpy (buf, p->file_buf + p->file_offset, count);
+    p->file_offset += count;
+    p->file_noread += count;
     return count;
 #else
     int r;
     r = read (fd, buf, count);
     if (r > 0)
-        file_noread += r;
+        p->file_noread += r;
     return r;
 #endif
 }
@@ -626,8 +700,8 @@ static char *fileMatchStr (struct recKeys *reckeys, struct recordGroup *rGroup,
                 }
             if (first)
             {
-                logf (LOG_WARN, "Record in file %s didn't contain match"
-                      " fields in (%d,%d)", fname, attrSet, attrUse);
+                logf (LOG_WARN, "Record didn't contain match"
+                      " fields in (%d,%d)", attrSet, attrUse);
                 return NULL;
             }
         }
@@ -697,25 +771,53 @@ static char *fileMatchStr (struct recKeys *reckeys, struct recordGroup *rGroup,
     return dstBuf;
 }
 
+struct recordLogInfo {
+    const char *fname;
+    char *op;
+    struct recordGroup *rGroup;
+};
+     
+static void recordLogPreamble (int level, const char *msg, void *info)
+{
+    struct recordLogInfo *p = info;
+    FILE *outf = log_file ();
+
+    if (level & LOG_LOG)
+        return ;
+    if (p->op) 
+        fprintf (outf, "%s of ", p->op);
+    fprintf (outf, "%s type %s\n", p->rGroup->recordType, p->fname);
+    log_event_start (NULL, NULL);
+}
+
 static int recordExtract (SYSNO *sysno, const char *fname,
                           struct recordGroup *rGroup, int deleteFlag,
-                          int fd, RecType recType, char *subType)
+                          struct file_read_info *fi, RecType recType,
+                          char *subType)
 {
     struct recExtractCtrl extractCtrl;
     int r;
     char *matchStr;
     SYSNO sysnotmp;
     Record rec;
+    struct recordLogInfo logInfo;
 
-    if (fd != -1)
+    logInfo.fname = fname;
+    logInfo.op = NULL;
+    logInfo.rGroup = rGroup;
+    log_event_start (recordLogPreamble, &logInfo);
+    
+    if (fi->fd != -1)
     {
-        extractCtrl.fd = fd;
+        extractCtrl.fh = fi;
         /* extract keys */
         extractCtrl.subType = subType;
         extractCtrl.init = wordInit;
         extractCtrl.add = addRecordKeyAny;
 
         reckeys.buf_used = 0;
+        reckeys.prevAttrUse = -1;
+        reckeys.prevAttrSet = -1;
         extractCtrl.readf = file_read;
         r = (*recType->extract)(&extractCtrl);
   
@@ -747,7 +849,7 @@ static int recordExtract (SYSNO *sysno, const char *fname,
             }
             else
             {
-                logf (LOG_WARN, "Record not inserted");
+                logf (LOG_WARN, "Bad match criteria");
                 return 0;
             }
         }
@@ -758,10 +860,14 @@ static int recordExtract (SYSNO *sysno, const char *fname,
     {
         if (deleteFlag)
         {
-            logf (LOG_LOG, "? %s", fname);
+            logf (LOG_LOG, "Cannot delete new record");
             return 1;
         }
-        logf (LOG_LOG, "add %s %s", rGroup->recordType, fname);
+        logInfo.op = "add";
+#if 0
+        logf (LOG_LOG, "update %s %s", rGroup->recordType,
+              fname);
+#endif
         rec = rec_new (records);
         *sysno = rec->sysno;
 
@@ -778,38 +884,42 @@ static int recordExtract (SYSNO *sysno, const char *fname,
         struct recKeys delkeys;
 
         rec = rec_get (records, *sysno);
-
+        assert (rec);
         delkeys.buf_used = rec->size[recInfo_delKeys];
 	delkeys.buf = rec->info[recInfo_delKeys];
         flushRecordKeys (*sysno, 0, &delkeys, rec->info[recInfo_databaseName]);
         if (deleteFlag)
         {
+            logInfo.op = "delete";
             if (!delkeys.buf_used)
             {
-                logf (LOG_WARN, "cannot delete %s - no delete keys", fname);
+                logf (LOG_WARN, "cannot delete; storeKeys false");
             }
             else
             {
-                SYSNO sysnoz = 0;
+#if 0
                 logf (LOG_LOG, "delete %s %s", rGroup->recordType, fname);
+#endif
                 records_deleted++;
                 if (matchStr)
-                    dict_insert (matchDict, matchStr, sizeof(sysnoz), &sysnoz);
+                    dict_delete (matchDict, matchStr);
                 rec_del (records, &rec);
             }
             return 1;
         }
         else
         {
+            logInfo.op = "update";
             if (!delkeys.buf_used)
             {
-                logf (LOG_WARN, "cannot update %s - no delete keys",
-                      fname);
+                logf (LOG_WARN, "cannot update; storeKeys false");
             }
             else
             {
+#if 0
                 logf (LOG_LOG, "update %s %s", rGroup->recordType,
                       fname);
+#endif
                 flushRecordKeys (*sysno, 1, &reckeys, rGroup->databaseName); 
                 records_updated++;
             }
@@ -826,10 +936,17 @@ static int recordExtract (SYSNO *sysno, const char *fname,
     xfree (rec->info[recInfo_delKeys]);
     if (reckeys.buf_used > 0 && rGroup->flagStoreKeys == 1)
     {
+#if 1
+        rec->size[recInfo_delKeys] = reckeys.buf_used;
+        rec->info[recInfo_delKeys] = reckeys.buf;
+        reckeys.buf = NULL;
+        reckeys.buf_max = 0;
+#else
         rec->info[recInfo_delKeys] = xmalloc (reckeys.buf_used);
         rec->size[recInfo_delKeys] = reckeys.buf_used;
         memcpy (rec->info[recInfo_delKeys], reckeys.buf,
                 rec->size[recInfo_delKeys]);
+#endif
     }
     else
     {
@@ -840,24 +957,25 @@ static int recordExtract (SYSNO *sysno, const char *fname,
     xfree (rec->info[recInfo_storeData]);
     if (rGroup->flagStoreData == 1)
     {
-        rec->size[recInfo_storeData] = file_noread;
-        rec->info[recInfo_storeData] = xmalloc (file_noread);
+        rec->size[recInfo_storeData] = fi->file_noread;
+        rec->info[recInfo_storeData] = xmalloc (fi->file_noread);
 #if FILE_READ_BUFSIZE
-        if (file_noread < FILE_READ_BUFSIZE)
-	    memcpy (rec->info[recInfo_storeData], file_buf, file_noread);
+        if (fi->file_noread < FILE_READ_BUFSIZE)
+	    memcpy (rec->info[recInfo_storeData], fi->file_buf,
+                    fi->file_noread);
         else
 #endif
         {
-            if (lseek (fd, 0L, SEEK_SET) < 0)
+            if (lseek (fi->fd, 0L, SEEK_SET) < 0)
             {
                 logf (LOG_ERRNO|LOG_FATAL, "seek to 0 in %s", fname);
                 exit (1);
             }
-            if (read (fd, rec->info[recInfo_storeData], file_noread) 
-                < file_noread)
+            if (read (fi->fd, rec->info[recInfo_storeData], fi->file_noread) 
+                < fi->file_noread)
             {
                 logf (LOG_ERRNO|LOG_FATAL, "read %d bytes of %s",
-                      file_noread, fname);
+                      fi->file_noread, fname);
                 exit (1);
             }
         }
@@ -872,6 +990,7 @@ static int recordExtract (SYSNO *sysno, const char *fname,
         rec_strdup (rGroup->databaseName, &rec->size[recInfo_databaseName]); 
 
     rec_put (records, &rec);
+    log_event_start (NULL, NULL);
     return 1;
 }
 
@@ -886,6 +1005,7 @@ int fileExtract (SYSNO *sysno, const char *fname,
     RecType recType;
     struct recordGroup rGroupM;
     struct recordGroup *rGroup = &rGroupM;
+    struct file_read_info *fi;
 
     memcpy (rGroup, rGroupP, sizeof(*rGroupP));
    
@@ -917,14 +1037,18 @@ int fileExtract (SYSNO *sysno, const char *fname,
             sprintf (ext_res, "%srecordType", gprefix);
             if (!(rGroup->recordType = res_get (common_resource, ext_res)))
             {
+#if 0
                 logf (LOG_LOG, "? %s", fname);
+#endif
                 return 0;
             }
         }
     }
     if (!rGroup->recordType)
     {
+#if 0
         logf (LOG_LOG, "? record %s", fname);
+#endif
         return 0;
     }
     if (!(recType = recType_byName (rGroup->recordType, subType)))
@@ -994,9 +1118,9 @@ int fileExtract (SYSNO *sysno, const char *fname,
             return 0;
         }
     }
-    file_read_start (fd);
-    recordExtract (sysno, fname, rGroup, deleteFlag, fd, recType, subType);
-    file_read_stop (fd);
+    fi = file_read_start (fd);
+    recordExtract (sysno, fname, rGroup, deleteFlag, fi, recType, subType);
+    file_read_stop (fi);
     if (fd != -1)
         close (fd);
     return 1;