X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fextract.c;h=aa695502246c84e03bf02c969aeae1dcaf557b18;hb=0ca8a9894b43c838980d650dd265f7a3da4ed219;hp=066262153574ee15179e9dbd6c725ea1441d5d57;hpb=1cd8e47d53d94462ab3a96f01ab200222004b901;p=idzebra-moved-to-github.git

diff --git a/index/extract.c b/index/extract.c
index 0662621..aa69550 100644
--- a/index/extract.c
+++ b/index/extract.c
@@ -1,10 +1,76 @@
 /*
- * Copyright (C) 1994-1995, Index Data I/S 
+ * Copyright (C) 1994-1996, Index Data I/S 
  * All rights reserved.
  * Sebastian Hammer, Adam Dickmeiss
  *
  * $Log: extract.c,v $
- * Revision 1.55  1996-05-09 07:28:55  quinn
+ * Revision 1.73  1997-09-04 13:57:20  adam
+ * New file extract/retrieve method tellf (added).
+ * Added O_BINARY for open calls.
+ *
+ * Revision 1.72  1997/07/15 16:32:29  adam
+ * Bug fix: Match handler didn't terminate the resulting string!
+ *
+ * Revision 1.71  1997/07/15 16:28:41  adam
+ * Bug fix: storeData didn't work with files with multiple records.
+ * Bug fix: fixed memory management with records; not really well
+ *  thought through.
+ *
+ * Revision 1.70  1997/07/01 13:00:42  adam
+ * Bug fix in routine searchRecordKey: uninitialized variables.
+ *
+ * Revision 1.69  1997/04/29 09:26:03  adam
+ * Bug fix: generic recordId handling didn't work for compressed internal
+ * keys.
+ *
+ * Revision 1.68  1997/02/12 20:39:45  adam
+ * Implemented options -f <n> that limits the log to the first <n>
+ * records.
+ * Changed some log messages also.
+ *
+ * Revision 1.67  1996/11/15 15:02:14  adam
+ * Minor changes regarding logging.
+ *
+ * Revision 1.66  1996/11/14  09:52:21  adam
+ * Strings in record keys bound by IT_MAX_WORD.
+ *
+ * Revision 1.65  1996/11/14  08:57:56  adam
+ * Reduction of storeKeys area.
+ *
+ * Revision 1.64  1996/11/08 11:10:16  adam
+ * Buffers used during file match got bigger.
+ * Compressed ISAM support everywhere.
+ * Bug fixes regarding masking characters in queries.
+ * Redesigned Regexp-2 queries.
+ *
+ * Revision 1.63  1996/10/29 14:09:39  adam
+ * Use of cisam system - enabled if setting isamc is 1.
+ *
+ * Revision 1.62  1996/10/11 10:57:01  adam
+ * New module recctrl. Used to manage records (extract/retrieval).
+ * Several files have been moved to the recctrl sub directory.
+ *
+ * Revision 1.61  1996/06/06 12:08:37  quinn
+ * Added showRecord function
+ *
+ * Revision 1.60  1996/06/04  10:18:12  adam
+ * Search/scan uses character mapping module.
+ *
+ * Revision 1.59  1996/05/14  15:47:07  adam
+ * Cleanup of various buffer size entities.
+ *
+ * Revision 1.58  1996/05/14  06:16:38  adam
+ * Compact use/set bytes used in search service.
+ *
+ * Revision 1.57  1996/05/13 14:23:04  adam
+ * Work on compaction of set/use bytes in dictionary.
+ *
+ * Revision 1.56  1996/05/09  09:54:42  adam
+ * Server supports maps from one logical attributes to a list of physical
+ * attributes.
+ * The extraction process doesn't make space consuming 'any' keys.
+ *
+ * Revision 1.55  1996/05/09  07:28:55  quinn
  * Work towards phrases and multiple registers
  *
  * Revision 1.54  1996/05/01  13:46:35  adam
@@ -202,13 +268,11 @@
 #include <assert.h>
 #include <unistd.h>
 #include <fcntl.h>
-#include <ctype.h>
 
-#include <alexutil.h>
 #include <recctrl.h>
 #include "index.h"
 
-#include "recindex.h"
+#include "zinfo.h"
 
 static Dict matchDict;
 
@@ -225,6 +289,8 @@ static int records_updated = 0;
 static int records_deleted = 0;
 static int records_processed = 0;
 
+static ZebTargetInfo *zti = NULL;
+
 static void logRecord (int showFlag)
 {
     if (!showFlag)
@@ -239,6 +305,8 @@ static void logRecord (int showFlag)
 
 void key_open (int mem)
 {
+    if (!mem)
+        mem = atoi(res_get_def (common_resource, "memMax", "4"))*1024*1024;
     if (mem < 50000)
         mem = 50000;
     key_buf = xmalloc (mem);
@@ -255,12 +323,15 @@ void key_open (int mem)
     }
     assert (!records);
     records = rec_open (1);
+#if 1
+    zti = zebTargetInfo_open (records, 1);
+#endif
 }
 
 struct encode_info {
     int  sysno;
     int  seqno;
-    char buf[512];
+    char buf[768];
 };
 
 void encode_key_init (struct encode_info *i)
@@ -317,20 +388,45 @@ void encode_key_write (char *k, struct encode_info *i, FILE *outf)
     }
 }
 
+#define SORT_EXTRA 0
+
+#if SORT_EXTRA
+static int key_y_len;
+
+static int key_y_compare (const void *p1, const void *p2)
+{
+    int r;
+
+    if ((r = key_compare (*(char**) p1 + key_y_len + 1,
+                          *(char**) p2 + key_y_len + 1)))
+         return r;
+    return *(*(char**) p1 + key_y_len) - *(*(char**) p2 + key_y_len);
+}
+
+static int key_x_compare (const void *p1, const void *p2)
+{
+    return strcmp (*(char**) p1, *(char**) p2);
+}
+#endif
+
 void key_flush (void)
 {
     FILE *outf;
     char out_fname[200];
     char *prevcp, *cp;
     struct encode_info encode_info;
+#if SORT_EXTRA
+    int i;
+#endif
     
     if (ptr_i <= 0)
         return;
 
     key_file_no++;
     logf (LOG_LOG, "sorting section %d", key_file_no);
+#if !SORT_EXTRA
     qsort (key_buf + ptr_top-ptr_i, ptr_i, sizeof(char*), key_qsort_compare);
-    sprintf (out_fname, TEMP_FNAME, key_file_no);
+    getFnameTmp (out_fname, key_file_no);
 
     if (!(outf = fopen (out_fname, "w")))
     {
@@ -354,6 +450,42 @@ void key_flush (void)
         else
             encode_key_write (cp + strlen(cp), &encode_info, outf);
     }
+#else
+    qsort (key_buf + ptr_top-ptr_i, ptr_i, sizeof(char*), key_x_compare);
+    getFnameTmp (out_fname, key_file_no);
+
+    if (!(outf = fopen (out_fname, "w")))
+    {
+        logf (LOG_FATAL|LOG_ERRNO, "fopen (4) %s", out_fname);
+        exit (1);
+    }
+    logf (LOG_LOG, "writing section %d", key_file_no);
+    i = ptr_i;
+    prevcp =  key_buf[ptr_top-i];
+    while (1)
+        if (!--i || strcmp (prevcp, key_buf[ptr_top-i]))
+        {
+            key_y_len = strlen(prevcp)+1;
+#if 0
+            logf (LOG_LOG, "key_y_len: %2d %02x %02x %s",
+                      key_y_len, prevcp[0], prevcp[1], 2+prevcp);
+#endif
+            qsort (key_buf + ptr_top-ptr_i, ptr_i - i,
+                                   sizeof(char*), key_y_compare);
+            cp = key_buf[ptr_top-ptr_i];
+            --key_y_len;
+            encode_key_init (&encode_info);
+            encode_key_write (cp, &encode_info, outf);
+            while (--ptr_i > i)
+            {
+                cp = key_buf[ptr_top-ptr_i];
+                encode_key_write (cp+key_y_len, &encode_info, outf);
+            }
+            if (!i)
+                break;
+            prevcp = key_buf[ptr_top-ptr_i];
+        }
+#endif
     if (fclose (outf))
     {
         logf (LOG_FATAL|LOG_ERRNO, "fclose %s", out_fname);
@@ -368,6 +500,9 @@ int key_close (void)
 {
     key_flush ();
     xfree (key_buf);
+#if 1
+    zebTargetInfo_close (zti, 1);
+#endif
     rec_close (&records);
     dict_close (matchDict);
 
@@ -388,6 +523,7 @@ struct recKeys {
     char *buf;
     char prevAttrSet;
     short prevAttrUse;
+    int prevSeqNo;
 } reckeys;
 
 static void addRecordKey (const RecWord *p)
@@ -397,6 +533,7 @@ static void addRecordKey (const RecWord *p)
     short attrUse;
     size_t i;
     int lead = 0;
+    int diff = 0;
 
     if (reckeys.buf_used+1024 > reckeys.buf_max)
     {
@@ -420,32 +557,46 @@ static void addRecordKey (const RecWord *p)
         lead |= 2;
     else
         reckeys.prevAttrUse = attrUse;
+#if 1
+    diff = 1 + p->seqno - reckeys.prevSeqNo;
+    if (diff >= 1 && diff <= 15)
+        lead |= (diff << 2);
+    else
+        diff = 0;
+#endif
+    reckeys.prevSeqNo = p->seqno;
+
+    *dst++ = lead;
 
+    if (!(lead & 1))
+    {
+        memcpy (dst, &attrSet, sizeof(attrSet));
+        dst += sizeof(attrSet);
+    }
+    if (!(lead & 2))
+    {
+        memcpy (dst, &attrUse, sizeof(attrUse));
+        dst += sizeof(attrUse);
+    }
     switch (p->which)
     {
-    case Word_String: case Word_Phrase:
-        *dst++ = lead;
-
-        if (!(lead & 1))
-        {
-            memcpy (dst, &attrSet, sizeof(attrSet));
-            dst += sizeof(attrSet);
-        }
-        if (!(lead & 2))
-        {
-            memcpy (dst, &attrUse, sizeof(attrUse));
-            dst += sizeof(attrUse);
-        }
-        for (i = 0; p->u.string[i]; i++)
-            *dst++ = p->u.string[i];
-        *dst++ = '\0';
+        case Word_String:
+            *dst++ = 'w';
+            break;
+        case Word_Phrase:
+            *dst++ = 'p';
+            break;
+        case Word_Numeric:
+            *dst++ = 'n';
+    }
+    for (i = 0; p->u.string[i] && i < IT_MAX_WORD-3; i++)
+        *dst++ = p->u.string[i];
+    *dst++ = '\0';
 
+    if (!diff)
+    {
         memcpy (dst, &p->seqno, sizeof(p->seqno));
         dst += sizeof(p->seqno);
-
-        break;
-    default:
-        return;
     }
     reckeys.buf_used = dst - reckeys.buf;
 }
@@ -455,12 +606,19 @@ static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys,
 {
     char attrSet = -1;
     short attrUse = -1;
+    int seqno = 0;
     int off = 0;
+
+    if (zebTargetInfo_curDatabase (zti, databaseName))
+    {
+        if (zebTargetInfo_newDatabase (zti, databaseName))
+            abort ();
+    }
     while (off < reckeys->buf_used)
     {
         const char *src = reckeys->buf + off;
         struct it_key key;
-        int lead;
+        int lead, ch;
     
         lead = *src++;
 
@@ -478,17 +636,26 @@ static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys,
             key_flush ();
         ++ptr_i;
         key_buf[ptr_top-ptr_i] = (char*)key_buf + key_buf_used;
-        key_buf_used += index_word_prefix ((char*)key_buf + key_buf_used,
-                                           attrSet, attrUse, databaseName);
+
+        ch = zebTargetInfo_lookupSU (zti, attrSet, attrUse);
+        if (ch < 0)
+            ch = zebTargetInfo_addSU (zti, attrSet, attrUse);
+        assert (ch > 0);
+        ((char*) key_buf) [key_buf_used++] = ch;
         while (*src)
-            ((char*)key_buf) [key_buf_used++] = index_char_cvt (*src++);
+            ((char*)key_buf) [key_buf_used++] = *src++;
         src++;
         ((char*)key_buf) [key_buf_used++] = '\0';
-        
         ((char*) key_buf)[key_buf_used++] = cmd;
 
-        memcpy (&key.seqno, src, sizeof(key.seqno));
-        src += sizeof(key.seqno);
+        if (lead & 60)
+            seqno += ((lead>>2) & 15)-1;
+        else
+        {
+            memcpy (&seqno, src, sizeof(seqno));
+            src += sizeof(seqno);
+        }
+        key.seqno = seqno;
         key.sysno = sysno;
         memcpy ((char*)key_buf + key_buf_used, &key, sizeof(key));
         key_buf_used += sizeof(key);
@@ -504,34 +671,42 @@ static const char **searchRecordKey (struct recKeys *reckeys,
     int off = 0;
     int startSeq = -1;
     int i;
+    short attrUse;
+    char attrSet;
+    int seqno = 0;
 
     for (i = 0; i<32; i++)
         ws[i] = NULL;
     
     while (off < reckeys->buf_used)
     {
-        const char *src = reckeys->buf + off;
-        char attrSet;
-        short attrUse;
-        int seqno;
-        const char *wstart;
-        
-        memcpy (&attrSet, src, sizeof(attrSet));
-        src += sizeof(attrSet);
 
-        memcpy (&attrUse, src, sizeof(attrUse));
-        src += sizeof(attrUse);
+        const char *src = reckeys->buf + off;
+	const char *wstart;
+        int lead;
+    
+        lead = *src++;
 
+        if (!(lead & 1))
+        {
+            memcpy (&attrSet, src, sizeof(attrSet));
+            src += sizeof(attrSet);
+        }
+        if (!(lead & 2))
+        {
+            memcpy (&attrUse, src, sizeof(attrUse));
+            src += sizeof(attrUse);
+        }
         wstart = src;
         while (*src++)
             ;
-
-        memcpy (&seqno, src, sizeof(seqno));
-        src += sizeof(seqno);
-
-#if 0
-        logf (LOG_LOG, "(%d,%d) %d %s", attrSet, attrUse, seqno, wstart);
-#endif
+        if (lead & 60)
+            seqno += ((lead>>2) & 15)-1;
+        else
+        {
+            memcpy (&seqno, src, sizeof(seqno));
+            src += sizeof(seqno);
+        }
         if (attrUseS == attrUse && attrSetS == attrSet)
         {
             int woff;
@@ -550,21 +725,6 @@ static const char **searchRecordKey (struct recKeys *reckeys,
     return ws;
 }
 
-static void addRecordKeyAny (const RecWord *p)
-{
-    if (p->attrSet != 1 || p->attrUse != 1016)
-    {
-        RecWord w;
-
-        memcpy (&w, p, sizeof(w));
-        w.attrSet = 1;
-        w.attrUse = 1016;
-        addRecordKey (&w);
-    }
-    if (p->attrSet != -1)
-        addRecordKey (p);
-}
-
 struct file_read_info {
     off_t file_max;
     off_t file_offset;
@@ -596,6 +756,12 @@ static off_t file_seek (void *handle, off_t offset)
     return lseek (p->fd, offset, SEEK_SET);
 }
 
+static off_t file_tell (void *handle)
+{
+    struct file_read_info *p = handle;
+    return p->file_offset;
+}
+
 static int file_read (void *handle, char *buf, size_t count)
 {
     struct file_read_info *p = handle;
@@ -770,12 +936,13 @@ static char *fileMatchStr (struct recKeys *reckeys, struct recordGroup *rGroup,
               fname, rGroup->groupName ? rGroup->groupName : "none");
         return NULL;
     }
+    *dst = '\0';
     return dstBuf;
 }
 
 struct recordLogInfo {
     const char *fname;
-    char *op;
+    int recordOffset;
     struct recordGroup *rGroup;
 };
      
@@ -786,9 +953,8 @@ static void recordLogPreamble (int level, const char *msg, void *info)
 
     if (level & LOG_LOG)
         return ;
-    if (p->op) 
-        fprintf (outf, "%s of ", p->op);
-    fprintf (outf, "%s type %s\n", p->rGroup->recordType, p->fname);
+    fprintf (outf, "File %s, offset %d, type %s\n",
+             p->rGroup->recordType, p->recordOffset, p->fname);
     log_event_start (NULL, NULL);
 }
 
@@ -805,41 +971,58 @@ static int recordExtract (SYSNO *sysno, const char *fname,
     Record rec;
     struct recordLogInfo logInfo;
 
-    logInfo.fname = fname;
-    logInfo.op = NULL;
-    logInfo.rGroup = rGroup;
-    log_event_start (recordLogPreamble, &logInfo);
-    
     if (fi->fd != -1)
     {
         /* we are going to read from a file, so prepare the extraction */
         extractCtrl.fh = fi;
         extractCtrl.subType = subType;
         extractCtrl.init = wordInit;
-        extractCtrl.add = addRecordKeyAny;
+        extractCtrl.add = addRecordKey;
 
         reckeys.buf_used = 0;
         reckeys.prevAttrUse = -1;
         reckeys.prevAttrSet = -1;
+        reckeys.prevSeqNo = 0;
 
         recordOffset = fi->file_moffset;
         extractCtrl.offset = recordOffset;
         extractCtrl.readf = file_read;
         extractCtrl.seekf = file_seek;
+        extractCtrl.tellf = file_tell;
         extractCtrl.endf = file_end;
+        extractCtrl.map_chrs_input = map_chrs_input;
+        extractCtrl.flagShowRecords = rGroup->flagShowRecords;
+        if (rGroup->flagShowRecords)
+            printf ("File: %s %ld\n", fname, (long) recordOffset);
+
+        logInfo.fname = fname;
+        logInfo.recordOffset = recordOffset;
+        logInfo.rGroup = rGroup;
+        log_event_start (recordLogPreamble, &logInfo);
+
         r = (*recType->extract)(&extractCtrl);
 
+        log_event_start (NULL, NULL);
+
         if (r)      
         {
             /* error occured during extraction ... */
-            logf (LOG_WARN, "Couldn't extract file %s, code %d", fname, r);
+            if (!rGroup->flagShowRecords &&
+                    records_processed < rGroup->fileVerboseLimit)
+            {
+                logf (LOG_WARN, "fail %s %s %ld code = %d", rGroup->recordType,
+                      fname, (long) recordOffset, r);
+            }
             return 0;
         }
         if (reckeys.buf_used == 0)
         {
             /* the extraction process returned no information - the record
-               is probably empty */
-            logf (LOG_WARN, "Empty file %s", fname);
+               is probably empty - unless flagShowRecords is in use */
+            if (rGroup->flagShowRecords)
+                return 1;
+            logf (LOG_WARN, "No keys generated for file %s", fname);
+            logf (LOG_WARN, " The file is probably empty");
             return 0;
         }
     }
@@ -879,9 +1062,8 @@ static int recordExtract (SYSNO *sysno, const char *fname,
             logf (LOG_LOG, "Cannot delete new record");
             return 1;
         }
-        logInfo.op = "add";
-        if (rGroup->fileVerboseFlag)
-            logf (LOG_LOG, "add %s %s+%ld", rGroup->recordType,
+        if (records_processed < rGroup->fileVerboseLimit)
+            logf (LOG_LOG, "add %s %s %ld", rGroup->recordType,
                   fname, (long) recordOffset);
         rec = rec_new (records);
         *sysno = rec->sysno;
@@ -907,14 +1089,15 @@ static int recordExtract (SYSNO *sysno, const char *fname,
         if (deleteFlag)
         {
             /* record going to be deleted */
-            logInfo.op = "delete";
             if (!delkeys.buf_used)
             {
-                logf (LOG_WARN, "cannot delete; storeKeys false");
+                logf (LOG_LOG, "delete %s %s %ld", rGroup->recordType,
+                      fname, (long) recordOffset);
+                logf (LOG_WARN, "cannot delete file above, storeKeys false");
             }
             else
             {
-                if (rGroup->fileVerboseFlag)
+                if (records_processed < rGroup->fileVerboseLimit)
                     logf (LOG_LOG, "delete %s %s %ld", rGroup->recordType,
                           fname, (long) recordOffset);
                 records_deleted++;
@@ -928,14 +1111,15 @@ static int recordExtract (SYSNO *sysno, const char *fname,
         else
         {
             /* record going to be updated */
-            logInfo.op = "update";
             if (!delkeys.buf_used)
             {
-                logf (LOG_WARN, "cannot update; storeKeys false");
+                logf (LOG_LOG, "update %s %s %ld", rGroup->recordType,
+                      fname, (long) recordOffset);
+                logf (LOG_WARN, "cannot update file above, storeKeys false");
             }
             else
             {
-                if (rGroup->fileVerboseFlag)
+                if (records_processed < rGroup->fileVerboseLimit)
                     logf (LOG_LOG, "update %s %s %ld", rGroup->recordType,
                           fname, (long) recordOffset);
                 flushRecordKeys (*sysno, 1, &reckeys, rGroup->databaseName); 
@@ -979,16 +1163,18 @@ static int recordExtract (SYSNO *sysno, const char *fname,
     xfree (rec->info[recInfo_storeData]);
     if (rGroup->flagStoreData == 1)
     {
-        rec->size[recInfo_storeData] = fi->file_max;
-        rec->info[recInfo_storeData] = xmalloc (fi->file_max);
+        int size = fi->file_moffset - recordOffset; 
+        if (!size)
+            size = fi->file_max - recordOffset;
+        rec->size[recInfo_storeData] = size;
+        rec->info[recInfo_storeData] = xmalloc (size);
         if (lseek (fi->fd, recordOffset, SEEK_SET) < 0)
         {
             logf (LOG_ERRNO|LOG_FATAL, "seek to %ld in %s", fname,
                   (long) recordOffset);
             exit (1);
         }
-        if (read (fi->fd, rec->info[recInfo_storeData], fi->file_max)
-            < fi->file_max)
+        if (read (fi->fd, rec->info[recInfo_storeData], size) < size)
         {
             logf (LOG_ERRNO|LOG_FATAL, "read %d bytes of %s",
                   fi->file_max, fname);
@@ -1061,7 +1247,7 @@ int fileExtract (SYSNO *sysno, const char *fname,
             sprintf (ext_res, "%srecordType", gprefix);
             if (!(rGroup->recordType = res_get (common_resource, ext_res)))
             {
-                if (rGroup->fileVerboseFlag)
+                if (records_processed < rGroup->fileVerboseLimit)
                     logf (LOG_LOG, "? %s", fname);
                 return 0;
             }
@@ -1069,7 +1255,7 @@ int fileExtract (SYSNO *sysno, const char *fname,
     }
     if (!rGroup->recordType)
     {
-        if (rGroup->fileVerboseFlag)
+        if (records_processed < rGroup->fileVerboseLimit)
             logf (LOG_LOG, "? record %s", fname);
         return 0;
     }
@@ -1134,7 +1320,7 @@ int fileExtract (SYSNO *sysno, const char *fname,
         fd = -1;
     else
     {
-        if ((fd = open (fname, O_RDONLY)) == -1)
+        if ((fd = open (fname, O_BINARY|O_RDONLY)) == -1)
         {
             logf (LOG_WARN|LOG_ERRNO, "open %s", fname);
             return 0;
@@ -1147,7 +1333,6 @@ int fileExtract (SYSNO *sysno, const char *fname,
         r = recordExtract (sysno, fname, rGroup, deleteFlag, fi,
                            recType, subType);
     } while (r && !sysno && fi->file_more);
-    log_event_start (NULL, NULL);
     file_read_stop (fi);
     if (fd != -1)
         close (fd);