X-Git-Url: http://git.indexdata.com/?p=idzebra-moved-to-github.git;a=blobdiff_plain;f=index%2Fextract.c;h=de6aa9e776431ecbab32bc66729547e9b997c4f8;hp=37c0df1518d9d68190abe73b8d3bc88bc3b915a5;hb=47eeb5384a8fae8bcac8afb8a84782ca094683f9;hpb=5cc048d68e36e2d4d79ed7a7d7efec4266039351

diff --git a/index/extract.c b/index/extract.c
index 37c0df1..de6aa9e 100644
--- a/index/extract.c
+++ b/index/extract.c
@@ -1,4 +1,4 @@
-/* $Id: extract.c,v 1.175 2005-03-16 15:26:37 adam Exp $
+/* $Id: extract.c,v 1.186 2005-06-14 20:28:54 adam Exp $
    Copyright (C) 1995-2005
    Index Data ApS
 
@@ -25,7 +25,8 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
 #include <ctype.h>
 #ifdef WIN32
 #include <io.h>
-#else
+#endif
+#if HAVE_UNISTD_H
 #include <unistd.h>
 #endif
 #include <fcntl.h>
@@ -79,6 +80,8 @@ static void logRecord (ZebraHandle zh)
     }
 }
 
+static void extract_set_store_data_prepare(struct recExtractCtrl *p);
+
 static void extract_init (struct recExtractCtrl *p, RecWord *w)
 {
     w->zebra_maps = p->zebra_maps;
@@ -114,7 +117,7 @@ static const char **searchRecordKey (ZebraHandle zh,
 	int attrSet, attrUse;
 
 	iscz1_decode(decode_handle, &dst, &src);
-	assert(key.len < 4 && key.len > 2);
+	assert(key.len <= 4 && key.len > 2);
 
 	attrSet = (int) key.mem[0] >> 16;
 	attrUse = (int) key.mem[0] & 65535;
@@ -146,8 +149,6 @@ struct file_read_info {
     off_t file_moffset;     /* offset of rec/rec boundary */
     int file_more;
     int fd;
-    char *sdrbuf;
-    int sdrmax;
 };
 
 static struct file_read_info *file_read_start (int fd)
@@ -158,8 +159,8 @@ static struct file_read_info *file_read_start (int fd)
     fi->fd = fd;
     fi->file_max = 0;
     fi->file_moffset = 0;
-    fi->sdrbuf = 0;
-    fi->sdrmax = 0;
+    fi->file_offset = 0;
+    fi->file_more = 0;
     return fi;
 }
 
@@ -172,8 +173,6 @@ static off_t file_seek (void *handle, off_t offset)
 {
     struct file_read_info *p = (struct file_read_info *) handle;
     p->file_offset = offset;
-    if (p->sdrbuf)
-	return offset;
     return lseek (p->fd, offset, SEEK_SET);
 }
 
@@ -188,16 +187,7 @@ static int file_read (void *handle, char *buf, size_t count)
     struct file_read_info *p = (struct file_read_info *) handle;
     int fd = p->fd;
     int r;
-    if (p->sdrbuf)
-    {
-	r = count;
-	if (r > p->sdrmax - p->file_offset)
-	    r = p->sdrmax - p->file_offset;
-	if (r)
-	    memcpy (buf, p->sdrbuf + p->file_offset, r);
-    }
-    else
-	r = read (fd, buf, count);
+    r = read (fd, buf, count);
     if (r > 0)
     {
         p->file_offset += r;
@@ -207,23 +197,15 @@ static int file_read (void *handle, char *buf, size_t count)
     return r;
 }
 
-static void file_begin (void *handle)
-{
-    struct file_read_info *p = (struct file_read_info *) handle;
-
-    p->file_offset = p->file_moffset;
-    if (!p->sdrbuf && p->file_moffset)
-        lseek (p->fd, p->file_moffset, SEEK_SET);
-    p->file_more = 0;
-}
-
 static void file_end (void *handle, off_t offset)
 {
     struct file_read_info *p = (struct file_read_info *) handle;
 
-    assert (p->file_more == 0);
-    p->file_more = 1;
-    p->file_moffset = offset;
+    if (offset != p->file_moffset)
+    {
+	p->file_moffset = offset;
+	p->file_more = 1;
+    }
 }
 
 static char *fileMatchStr (ZebraHandle zh,
@@ -388,12 +370,28 @@ void create_rec_keys_codec(struct recKeys *keys)
     keys->buf_used = 0;
     iscz1_reset(keys->codec_handle);
 }
-     
+
+static void init_extractCtrl(ZebraHandle zh, struct recExtractCtrl *ctrl)
+{
+    int i;
+    for (i = 0; i<256; i++)
+    {
+	if (zebra_maps_is_positioned(zh->reg->zebra_maps, i))
+	    ctrl->seqno[i] = 1;
+	else
+	    ctrl->seqno[i] = 0;
+    }
+    ctrl->zebra_maps = zh->reg->zebra_maps;
+    ctrl->flagShowRecords = !zh->m_flag_rw;
+}
+
 static int file_extract_record(ZebraHandle zh,
 			       SYSNO *sysno, const char *fname,
 			       int deleteFlag,
 			       struct file_read_info *fi,
-			       int force_update)
+			       int force_update,
+			       RecType recType,
+			       void *recTypeClientData)
 {
     RecordAttr *recordAttr;
     int r;
@@ -401,17 +399,7 @@ static int file_extract_record(ZebraHandle zh,
     SYSNO sysnotmp;
     Record rec;
     off_t recordOffset = 0;
-    RecType recType;
-    void *clientData;
     
-    if (!(recType =
-	  recType_byName (zh->reg->recTypes, zh->res, zh->m_record_type,
-			  &clientData)))
-    {
-        yaz_log (YLOG_WARN, "No such record type: %s", zh->m_record_type);
-        return 0;
-    }
-
     /* announce database */
     if (zebraExplain_curDatabase (zh->reg->zei, zh->basenames[0]))
     {
@@ -425,13 +413,12 @@ static int file_extract_record(ZebraHandle zh,
 	struct recExtractCtrl extractCtrl;
 
         /* we are going to read from a file, so prepare the extraction */
-	int i;
-
 	create_rec_keys_codec(&zh->reg->keys);
 
 	zh->reg->sortKeys.buf_used = 0;
 	
 	recordOffset = fi->file_moffset;
+        extractCtrl.handle = zh;
 	extractCtrl.offset = fi->file_moffset;
 	extractCtrl.readf = file_read;
 	extractCtrl.seekf = file_seek;
@@ -443,16 +430,11 @@ static int file_extract_record(ZebraHandle zh,
 	extractCtrl.schemaAdd = extract_schema_add;
 	extractCtrl.dh = zh->reg->dh;
 	extractCtrl.match_criteria[0] = '\0';
-        extractCtrl.handle = zh;
-	for (i = 0; i<256; i++)
-	{
-	    if (zebra_maps_is_positioned(zh->reg->zebra_maps, i))
-		extractCtrl.seqno[i] = 1;
-	    else
-		extractCtrl.seqno[i] = 0;
-	}
-	extractCtrl.zebra_maps = zh->reg->zebra_maps;
-	extractCtrl.flagShowRecords = !zh->m_flag_rw;
+	extractCtrl.first_record = fi->file_offset ? 0 : 1;
+
+	extract_set_store_data_prepare(&extractCtrl);
+
+	init_extractCtrl(zh, &extractCtrl);
 
         if (!zh->m_flag_rw)
             printf ("File: %s " PRINTF_OFF_T "\n", fname, recordOffset);
@@ -463,7 +445,7 @@ static int file_extract_record(ZebraHandle zh,
             yaz_log_init_prefix2 (msg);
         }
 
-        r = (*recType->extract)(clientData, &extractCtrl);
+        r = (*recType->extract)(recTypeClientData, &extractCtrl);
 
         yaz_log_init_prefix2 (0);
 	if (r == RECCTRL_EXTRACT_EOF)
@@ -491,18 +473,6 @@ static int file_extract_record(ZebraHandle zh,
             }
             return 0;
         }
-        if (zh->reg->keys.buf_used == 0)
-        {
-            /* the extraction process returned no information - the record
-               is probably empty - unless flagShowRecords is in use */
-            if (!zh->m_flag_rw)
-                return 1;
-	    
-	    if (zh->records_processed < zh->m_file_verbose_limit)
-	        yaz_log (YLOG_WARN, "empty %s %s " PRINTF_OFF_T, zh->m_record_type,
-		    fname, recordOffset);
-            return 1;
-        }
         if (extractCtrl.match_criteria[0])
             matchStr = extractCtrl.match_criteria;
     }
@@ -534,6 +504,18 @@ static int file_extract_record(ZebraHandle zh,
 	    }
 	}
     }
+    if (! *sysno && zh->reg->keys.buf_used == 0)
+    {
+         /* the extraction process returned no information - the record
+            is probably empty - unless flagShowRecords is in use */
+         if (!zh->m_flag_rw)
+             return 1;
+  
+         if (zh->records_processed < zh->m_file_verbose_limit)
+	     yaz_log (YLOG_WARN, "empty %s %s " PRINTF_OFF_T, zh->m_record_type,
+	    fname, recordOffset);
+         return 1;
+    }
 
     if (! *sysno)
     {
@@ -685,7 +667,13 @@ static int file_extract_record(ZebraHandle zh,
 
     /* update store data */
     xfree (rec->info[recInfo_storeData]);
-    if (zh->m_store_data)
+    if (zh->store_data_buf)
+    {
+        rec->size[recInfo_storeData] = zh->store_data_size;
+        rec->info[recInfo_storeData] = zh->store_data_buf;
+	zh->store_data_buf = 0;
+    }
+    else if (zh->m_store_data)
     {
         rec->size[recInfo_storeData] = recordAttr->recordSize;
         rec->info[recInfo_storeData] = (char *)
@@ -732,6 +720,8 @@ int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname,
     char ext_res[128];
     struct file_read_info *fi;
     const char *original_record_type = 0;
+    RecType recType;
+    void *recTypeClientData;
 
     if (!zh->m_group || !*zh->m_group)
         *gprefix = '\0';
@@ -770,6 +760,21 @@ int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname,
         zh->m_record_id = res_get (zh->res, ext_res);
     }
 
+    if (!(recType =
+	  recType_byName (zh->reg->recTypes, zh->res, zh->m_record_type,
+			  &recTypeClientData)))
+    {
+        yaz_log(YLOG_WARN, "No such record type: %s", zh->m_record_type);
+        return 0;
+    }
+
+    switch(recType->version)
+    {
+    case 0:
+	break;
+    default:
+	yaz_log(YLOG_WARN, "Bad filter version: %s", zh->m_record_type);
+    }
     if (sysno && deleteFlag)
         fd = -1;
     else
@@ -796,9 +801,17 @@ int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname,
     fi = file_read_start (fd);
     do
     {
-        file_begin (fi);
-        r = file_extract_record (zh, sysno, fname, deleteFlag, fi, 1);
-    } while (r && !sysno && fi->file_more);
+	fi->file_moffset = fi->file_offset;
+	fi->file_more = 0;  /* file_end not called (yet) */
+        r = file_extract_record (zh, sysno, fname, deleteFlag, fi, 1,
+				 recType, recTypeClientData);
+	if (fi->file_more)
+	{   /* file_end has been called so reset offset .. */
+	    fi->file_offset = fi->file_moffset;
+	    lseek(fi->fd, fi->file_moffset, SEEK_SET);
+	}
+    }
+    while (r && !sysno);
     file_read_stop (fi);
     if (fd != -1)
         close (fd);
@@ -812,20 +825,20 @@ int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname,
   If not, and a record is provided, then sysno is got from there
   
  */
-int buffer_extract_record (ZebraHandle zh, 
-			   const char *buf, size_t buf_size,
-			   int delete_flag,
-			   int test_mode, 
-			   const char *recordType,
-			   SYSNO *sysno,
-			   const char *match_criteria,
-			   const char *fname,
-			   int force_update,
-			   int allow_update)
+ZEBRA_RES buffer_extract_record (ZebraHandle zh, 
+				 const char *buf, size_t buf_size,
+				 int delete_flag,
+				 int test_mode, 
+				 const char *recordType,
+				 SYSNO *sysno,
+				 const char *match_criteria,
+				 const char *fname,
+				 int force_update,
+				 int allow_update)
 {
     RecordAttr *recordAttr;
     struct recExtractCtrl extractCtrl;
-    int i, r;
+    int r;
     const char *matchStr = 0;
     RecType recType = NULL;
     void *clientData;
@@ -850,6 +863,7 @@ int buffer_extract_record (ZebraHandle zh,
     extractCtrl.seekf = zebra_record_int_seek;
     extractCtrl.tellf = zebra_record_int_tell;
     extractCtrl.endf = zebra_record_int_end;
+    extractCtrl.first_record = 1;
     extractCtrl.fh = &fc;
 
     create_rec_keys_codec(&zh->reg->keys);
@@ -860,17 +874,21 @@ int buffer_extract_record (ZebraHandle zh,
     {
         if (zebraExplain_newDatabase (zh->reg->zei, zh->basenames[0], 
 				      zh->m_explain_database))
-            return 0;
+            return ZEBRA_FAIL;
     }
     
-    if (recordType && *recordType) {
+    if (recordType && *recordType)
+    {
         yaz_log (YLOG_DEBUG, "Record type explicitly specified: %s", recordType);
         recType = recType_byName (zh->reg->recTypes, zh->res, recordType,
                                   &clientData);
-    } else {
-        if (!(zh->m_record_type)) {
+    } 
+    else
+    {
+        if (!(zh->m_record_type))
+	{
             yaz_log (YLOG_WARN, "No such record type defined");
-            return 0;
+            return ZEBRA_FAIL;
         }
         yaz_log (YLOG_DEBUG, "Get record type from rgroup: %s",zh->m_record_type);
         recType = recType_byName (zh->reg->recTypes, zh->res,
@@ -878,9 +896,10 @@ int buffer_extract_record (ZebraHandle zh,
         recordType = zh->m_record_type;
     }
     
-    if (!recType) {
+    if (!recType)
+    {
         yaz_log (YLOG_WARN, "No such record type: %s", zh->m_record_type);
-        return 0;
+        return ZEBRA_FAIL;
     }
     
     extractCtrl.init = extract_init;
@@ -888,42 +907,27 @@ int buffer_extract_record (ZebraHandle zh,
     extractCtrl.schemaAdd = extract_schema_add;
     extractCtrl.dh = zh->reg->dh;
     extractCtrl.handle = zh;
-    extractCtrl.zebra_maps = zh->reg->zebra_maps;
-    extractCtrl.flagShowRecords = 0;
     extractCtrl.match_criteria[0] = '\0';
-    for (i = 0; i<256; i++)
-    {
-	if (zebra_maps_is_positioned(zh->reg->zebra_maps, i))
-	    extractCtrl.seqno[i] = 1;
-	else
-	    extractCtrl.seqno[i] = 0;
-    }
+    
+    init_extractCtrl(zh, &extractCtrl);
+
+    extract_set_store_data_prepare(&extractCtrl);
 
     r = (*recType->extract)(clientData, &extractCtrl);
 
     if (r == RECCTRL_EXTRACT_EOF)
-	return 0;
+	return ZEBRA_FAIL;
     else if (r == RECCTRL_EXTRACT_ERROR_GENERIC)
     {
 	/* error occured during extraction ... */
 	yaz_log (YLOG_WARN, "extract error: generic");
-	return 0;
+	return ZEBRA_FAIL;
     }
     else if (r == RECCTRL_EXTRACT_ERROR_NO_SUCH_FILTER)
     {
 	/* error occured during extraction ... */
 	yaz_log (YLOG_WARN, "extract error: no such filter");
-	return 0;
-    }
-    if (zh->reg->keys.buf_used == 0)
-    {
-	/* the extraction process returned no information - the record
-	   is probably empty - unless flagShowRecords is in use */
-	if (test_mode)
-	    return 1;
-	yaz_log (YLOG_WARN, "No keys generated for record");
-	yaz_log (YLOG_WARN, " The file is probably empty");
-	return 1;
+	return ZEBRA_FAIL;
     }
     /* match criteria */
     matchStr = NULL;
@@ -942,7 +946,7 @@ int buffer_extract_record (ZebraHandle zh,
 		if (!matchStr)
                 {
                     yaz_log (YLOG_WARN, "Bad match criteria (recordID)");
-		    return 1;
+		    return ZEBRA_FAIL;
                 }
             }
         }
@@ -955,6 +959,13 @@ int buffer_extract_record (ZebraHandle zh,
 	    }
         }
     }
+    if (zh->reg->keys.buf_used == 0)
+    {
+	/* the extraction process returned no information - the record
+	   is probably empty - unless flagShowRecords is in use */
+	if (test_mode)
+	    return ZEBRA_OK;
+    }
 
     if (! *sysno)
     {
@@ -965,7 +976,7 @@ int buffer_extract_record (ZebraHandle zh,
 		yaz_log (YLOG_LOG, "delete %s %s %ld", recordType,
 			 pr_fname, (long) recordOffset);
             yaz_log (YLOG_WARN, "cannot delete record above (seems new)");
-            return 1;
+            return ZEBRA_FAIL;
         }
 	if (show_progress)
 	    yaz_log (YLOG_LOG, "add %s %s %ld", recordType, pr_fname,
@@ -998,7 +1009,7 @@ int buffer_extract_record (ZebraHandle zh,
 		yaz_log (YLOG_LOG, "skipped %s %s %ld", 
 			 recordType, pr_fname, (long) recordOffset);
 	    logRecord(zh);
-	    return -1;
+	    return ZEBRA_FAIL;
 	}
 
         rec = rec_get (zh->reg->records, *sysno);
@@ -1016,7 +1027,7 @@ int buffer_extract_record (ZebraHandle zh,
 		extract_flushSortKeys (zh, *sysno, -1, &zh->reg->sortKeys);
 		rec_rm (&rec);
 		logRecord(zh);
-		return -1;
+		return ZEBRA_FAIL;
 	    }
 	}
 
@@ -1053,7 +1064,7 @@ int buffer_extract_record (ZebraHandle zh,
             }
 	    rec_rm (&rec);
             logRecord(zh);
-            return 0;
+            return ZEBRA_OK;
         }
         else
         {
@@ -1130,7 +1141,15 @@ int buffer_extract_record (ZebraHandle zh,
 
     /* update store data */
     xfree (rec->info[recInfo_storeData]);
-    if (zh->m_store_data)
+
+    /* update store data */
+    if (zh->store_data_buf)
+    {
+        rec->size[recInfo_storeData] = zh->store_data_size;
+        rec->info[recInfo_storeData] = zh->store_data_buf;
+	zh->store_data_buf = 0;
+    }
+    else if (zh->m_store_data)
     {
         rec->size[recInfo_storeData] = recordAttr->recordSize;
         rec->info[recInfo_storeData] = (char *)
@@ -1153,14 +1172,13 @@ int buffer_extract_record (ZebraHandle zh,
     /* commit this record */
     rec_put (zh->reg->records, &rec);
     logRecord(zh);
-    return 0;
+    return ZEBRA_OK;
 }
 
 int explain_extract (void *handle, Record rec, data1_node *n)
 {
     ZebraHandle zh = (ZebraHandle) handle;
     struct recExtractCtrl extractCtrl;
-    int i;
 
     if (zebraExplain_curDatabase (zh->reg->zei,
 				  rec->info[recInfo_databaseName]))
@@ -1179,12 +1197,15 @@ int explain_extract (void *handle, Record rec, data1_node *n)
     extractCtrl.tokenAdd = extract_token_add;
     extractCtrl.schemaAdd = extract_schema_add;
     extractCtrl.dh = zh->reg->dh;
-    for (i = 0; i<256; i++)
-	extractCtrl.seqno[i] = 0;
-    extractCtrl.zebra_maps = zh->reg->zebra_maps;
+
+    init_extractCtrl(zh, &extractCtrl);
+
     extractCtrl.flagShowRecords = 0;
     extractCtrl.match_criteria[0] = '\0';
     extractCtrl.handle = handle;
+    extractCtrl.first_record = 1;
+    
+    extract_set_store_data_prepare(&extractCtrl);
 
     if (n)
 	grs_extract_tree(&extractCtrl, n);
@@ -1453,6 +1474,81 @@ void extract_add_it_key (ZebraHandle zh,
     keys->buf_used = dst - keys->buf;
 }
 
+ZEBRA_RES zebra_snippets_rec_keys(ZebraHandle zh, struct recKeys *reckeys,
+				  zebra_snippets *snippets)
+{
+   void *decode_handle = iscz1_start();
+    int off = 0;
+    int seqno = 0;
+    NMEM nmem = nmem_create();
+
+    yaz_log(YLOG_LOG, "zebra_rec_keys_snippets buf=%p sz=%d", reckeys->buf,
+	    reckeys->buf_used);
+    assert(reckeys->buf);
+    while (off < reckeys->buf_used)
+    {
+        const char *src = reckeys->buf + off;
+        struct it_key key;
+	char *dst = (char*) &key;
+	char dst_buf[IT_MAX_WORD];
+	char *dst_term = dst_buf;
+
+	iscz1_decode(decode_handle, &dst, &src);
+	assert(key.len <= 4 && key.len > 2);
+
+	seqno = (int) key.mem[key.len-1];
+
+	zebra_term_untrans_iconv(zh, nmem, src[0], &dst_term, src+1);
+	zebra_snippets_append(snippets, seqno, src[0], key.mem[0], dst_term);
+        while (*src++)
+	    ;
+        off = src - reckeys->buf;
+	nmem_reset(nmem);
+    }
+    nmem_destroy(nmem);
+    iscz1_stop(decode_handle);
+    return ZEBRA_OK;
+}
+
+void print_rec_keys(ZebraHandle zh, struct recKeys *reckeys)
+{
+    void *decode_handle = iscz1_start();
+    int off = 0;
+    int seqno = 0;
+    NMEM nmem = nmem_create();
+
+    yaz_log(YLOG_LOG, "print_rec_keys buf=%p sz=%d", reckeys->buf,
+	    reckeys->buf_used);
+    assert(reckeys->buf);
+    while (off < reckeys->buf_used)
+    {
+        const char *src = reckeys->buf + off;
+        struct it_key key;
+	char *dst = (char*) &key;
+	int attrSet, attrUse;
+	char dst_buf[IT_MAX_WORD];
+	char *dst_term = dst_buf;
+
+	iscz1_decode(decode_handle, &dst, &src);
+	assert(key.len <= 4 && key.len > 2);
+
+	attrSet = (int) key.mem[0] >> 16;
+	attrUse = (int) key.mem[0] & 65535;
+	seqno = (int) key.mem[key.len-1];
+
+	zebra_term_untrans_iconv(zh, nmem, src[0], &dst_term, src+1);
+	
+	yaz_log(YLOG_LOG, "ord=" ZINT_FORMAT " seqno=%d term=%s",
+		key.mem[0], seqno, dst_term); 
+        while (*src++)
+	    ;
+        off = src - reckeys->buf;
+	nmem_reset(nmem);
+    }
+    nmem_destroy(nmem);
+    iscz1_stop(decode_handle);
+}
+
 void extract_add_index_string (RecWord *p, const char *str, int length)
 {
     struct it_key key;
@@ -1667,9 +1763,34 @@ void extract_token_add (RecWord *p)
 	extract_add_incomplete_field(p);
 }
 
+static void extract_set_store_data_cb(struct recExtractCtrl *p,
+				      void *buf, size_t sz)
+{
+    ZebraHandle zh = (ZebraHandle) p->handle;
+
+    xfree(zh->store_data_buf);
+    zh->store_data_buf = 0;
+    zh->store_data_size = 0;
+    if (buf && sz)
+    {
+	zh->store_data_buf = xmalloc(sz);
+	zh->store_data_size = sz;
+	memcpy(zh->store_data_buf, buf, sz);
+    }
+}
+
+static void extract_set_store_data_prepare(struct recExtractCtrl *p)
+{
+    ZebraHandle zh = (ZebraHandle) p->handle;
+    xfree(zh->store_data_buf);
+    zh->store_data_buf = 0;
+    zh->store_data_size = 0;
+    p->setStoreData = extract_set_store_data_cb;
+}
+
 void extract_schema_add (struct recExtractCtrl *p, Odr_oid *oid)
 {
-    ZebraHandle zh = (ZebraHandle) (p->handle);
+    ZebraHandle zh = (ZebraHandle) p->handle;
     zebraExplain_addSchema (zh->reg->zei, oid);
 }