X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fextract.c;h=da261e2cdb17b610a0d63922eb2a55f0d9119c54;hb=8ddf739043beb068cac9287afa88ea92bf4d3ed6;hp=32aed3667db1a5ce7c01f792eaee843b3fba5767;hpb=ce3907338568fce46c5751e7e1091a5ad1c8e291;p=idzebra-moved-to-github.git

diff --git a/index/extract.c b/index/extract.c
index 32aed36..da261e2 100644
--- a/index/extract.c
+++ b/index/extract.c
@@ -1,10 +1,62 @@
 /*
- * Copyright (C) 1994-1998, Index Data I/S 
+ * Copyright (C) 1994-1999, Index Data 
  * All rights reserved.
  * Sebastian Hammer, Adam Dickmeiss
  *
  * $Log: extract.c,v $
- * Revision 1.80  1998-03-05 08:45:11  adam
+ * Revision 1.95  1999-05-21 12:00:17  adam
+ * Better diagnostics for extraction process.
+ *
+ * Revision 1.94  1999/05/20 12:57:18  adam
+ * Implemented TCL filter. Updated recctrl system.
+ *
+ * Revision 1.93  1999/05/15 14:36:38  adam
+ * Updated dictionary. Implemented "compression" of dictionary.
+ *
+ * Revision 1.92  1999/03/09 16:27:49  adam
+ * More work on SDRKit integration.
+ *
+ * Revision 1.91  1999/02/12 13:29:22  adam
+ * Implemented position-flag for registers.
+ *
+ * Revision 1.90  1999/02/02 14:50:52  adam
+ * Updated WIN32 code specific sections. Changed header.
+ *
+ * Revision 1.89  1998/10/28 10:54:38  adam
+ * SDRKit integration.
+ *
+ * Revision 1.88  1998/10/16 08:14:29  adam
+ * Updated record control system.
+ *
+ * Revision 1.87  1998/10/15 13:10:33  adam
+ * Fixed bug in Zebra that caused it to stop indexing when empty
+ * record was read.
+ *
+ * Revision 1.86  1998/10/13 20:33:53  adam
+ * Fixed one log message and change use ordinal to be an unsigned char.
+ *
+ * Revision 1.85  1998/09/22 10:03:41  adam
+ * Changed result sets to be persistent in the sense that they can
+ * be re-searched if needed.
+ * Fixed memory leak in rsm_or.
+ *
+ * Revision 1.84  1998/06/11 15:42:22  adam
+ * Changed the way use attributes are specified in the recordId
+ * specification.
+ *
+ * Revision 1.83  1998/06/08 14:43:10  adam
+ * Added suport for EXPLAIN Proxy servers - added settings databasePath
+ * and explainDatabase to facilitate this. Increased maximum number
+ * of databases and attributes in one register.
+ *
+ * Revision 1.82  1998/05/20 10:12:15  adam
+ * Implemented automatic EXPLAIN database maintenance.
+ * Modified Zebra to work with ASN.1 compiled version of YAZ.
+ *
+ * Revision 1.81  1998/03/11 11:19:04  adam
+ * Changed the way sequence numbers are generated.
+ *
+ * Revision 1.80  1998/03/05 08:45:11  adam
  * New result set model and modular ranking system. Moved towards
  * descent server API. System information stored as "SGML" records.
  *
@@ -292,7 +344,7 @@
  */
 #include <stdio.h>
 #include <assert.h>
-#ifdef WINDOWS
+#ifdef WIN32
 #include <io.h>
 #else
 #include <unistd.h>
@@ -306,6 +358,14 @@
 
 #include "zinfo.h"
 
+#ifndef ZEBRASDR
+#define ZEBRASDR 0
+#endif
+
+#if ZEBRASDR
+#include "zebrasdr.h"
+#endif
+
 static Dict matchDict;
 
 static Records records = NULL;
@@ -336,8 +396,13 @@ static void logRecord (int showFlag)
     }
 }
 
-int key_open (BFiles bfs, int mem, int rw, data1_handle dh)
+static int explain_extract (void *handle, Record drec, data1_node *n);
+
+int key_open (struct recordGroup *rGroup, int mem)
 {
+    BFiles bfs = rGroup->bfs;
+    int rw = rGroup->flagRw;
+    data1_handle dh = rGroup->dh;
     if (!mem)
         mem = atoi(res_get_def (common_resource, "memMax", "4"))*1024*1024;
     if (mem < 50000)
@@ -349,7 +414,7 @@ int key_open (BFiles bfs, int mem, int rw, data1_handle dh)
     key_buf_used = 0;
     key_file_no = 0;
 
-    if (!(matchDict = dict_open (bfs, GMATCH_DICT, 50, rw)))
+    if (!(matchDict = dict_open (bfs, GMATCH_DICT, 50, rw, 0)))
     {
         logf (LOG_FATAL, "dict_open fail of %s", GMATCH_DICT);
 	return -1;
@@ -361,7 +426,8 @@ int key_open (BFiles bfs, int mem, int rw, data1_handle dh)
 	dict_close (matchDict);
 	return -1;
     }
-    zti = zebraExplain_open (records, dh, rw);
+    zti = zebraExplain_open (records, dh, common_resource,
+			     rw, rGroup, explain_extract);
     if (!zti)
     {
 	rec_close (&records);
@@ -375,6 +441,7 @@ int key_open (BFiles bfs, int mem, int rw, data1_handle dh)
 struct encode_info {
     int  sysno;
     int  seqno;
+    int  cmd;
     char buf[768];
 };
 
@@ -382,6 +449,7 @@ void encode_key_init (struct encode_info *i)
 {
     i->sysno = 0;
     i->seqno = 0;
+    i->cmd = -1;
 }
 
 char *encode_key_int (int d, char *bp)
@@ -423,8 +491,11 @@ void encode_key_write (char *k, struct encode_info *i, FILE *outf)
         i->sysno = key.sysno;
         i->seqno = 0;
     }
+    else if (!i->seqno && !key.seqno && i->cmd == *k)
+	return;
     bp = encode_key_int (key.seqno - i->seqno, bp);
     i->seqno = key.seqno;
+    i->cmd = *k;
     if (fwrite (i->buf, bp - i->buf, 1, outf) != 1)
     {
         logf (LOG_FATAL|LOG_ERRNO, "fwrite");
@@ -540,13 +611,14 @@ void key_flush (void)
     key_buf_used = 0;
 }
 
-int key_close (int rw)
+int key_close (struct recordGroup *rGroup)
 {
-    key_flush ();
-    xfree (key_buf);
+    int rw = rGroup->flagRw;
     if (rw)
 	zebraExplain_runNumberIncrement (zti, 1);
-    zebraExplain_close (zti, rw);
+    zebraExplain_close (zti, rw, 0);
+    key_flush ();
+    xfree (key_buf);
     rec_close (&records);
     dict_close (matchDict);
     sortIdx_close (sortIdx);
@@ -558,7 +630,8 @@ int key_close (int rw)
 static void wordInit (struct recExtractCtrl *p, RecWord *w)
 {
     w->zebra_maps = p->zebra_maps;
-    w->attrSet = 1;
+    w->seqnos = p->seqno;
+    w->attrSet = VAL_BIB1;
     w->attrUse = 1016;
     w->reg_type = 'w';
 }
@@ -583,10 +656,11 @@ static struct recKeys {
 static void addIndexString (RecWord *p, const char *string, int length)
 {
     char *dst;
-    char attrSet;
-    short attrUse;
+    unsigned char attrSet;
+    unsigned short attrUse;
     int lead = 0;
     int diff = 0;
+    int *pseqno = &p->seqnos[p->reg_type];
 
     if (reckeys.buf_used+1024 > reckeys.buf_max)
     {
@@ -611,14 +685,14 @@ static void addIndexString (RecWord *p, const char *string, int length)
     else
         reckeys.prevAttrUse = attrUse;
 #if 1
-    diff = 1 + p->seqno - reckeys.prevSeqNo;
+    diff = 1 + *pseqno - reckeys.prevSeqNo;
     if (diff >= 1 && diff <= 15)
         lead |= (diff << 2);
     else
         diff = 0;
 #endif
-    reckeys.prevSeqNo = p->seqno;
-
+    reckeys.prevSeqNo = *pseqno;
+    
     *dst++ = lead;
 
     if (!(lead & 1))
@@ -638,11 +712,12 @@ static void addIndexString (RecWord *p, const char *string, int length)
 
     if (!diff)
     {
-        memcpy (dst, &p->seqno, sizeof(p->seqno));
-        dst += sizeof(p->seqno);
+        memcpy (dst, pseqno, sizeof(*pseqno));
+        dst += sizeof(*pseqno);
     }
     reckeys.buf_used = dst - reckeys.buf;
-    (p->seqno)++;
+    if (*pseqno)
+	(*pseqno)++;
 }
 
 static void addSortString (RecWord *p, const char *string, int length)
@@ -657,9 +732,10 @@ static void addSortString (RecWord *p, const char *string, int length)
     sk->next = sortKeys;
     sortKeys = sk;
 
-    sk->string = xmalloc (p->length);
-    sk->length = p->length;
-    memcpy (sk->string, p->string, p->length);
+    sk->string = xmalloc (length);
+    sk->length = length;
+    memcpy (sk->string, string, length);
+
     sk->attrSet = p->attrSet;
     sk->attrUse = p->attrUse;
 }
@@ -788,19 +864,13 @@ static void flushSortKeys (SYSNO sysno, int cmd)
     sortKeys = NULL;
 }
 
-static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys, 
-                             const char *databaseName)
+static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys)
 {
-    char attrSet = -1;
-    short attrUse = -1;
+    unsigned char attrSet = (unsigned char) -1;
+    unsigned short attrUse = (unsigned short) -1;
     int seqno = 0;
     int off = 0;
 
-    if (zebraExplain_curDatabase (zti, databaseName))
-    {
-        if (zebraExplain_newDatabase (zti, databaseName))
-            abort ();
-    }
     zebraExplain_recordCountIncrement (zti, cmd ? 1 : -1);
     while (off < reckeys->buf_used)
     {
@@ -829,7 +899,8 @@ static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys,
         if (ch < 0)
             ch = zebraExplain_addSU (zti, attrSet, attrUse);
         assert (ch > 0);
-        ((char*) key_buf) [key_buf_used++] = ch;
+	key_buf_used += key_SU_code (ch, ((char*)key_buf) + key_buf_used);
+
         while (*src)
             ((char*)key_buf) [key_buf_used++] = *src++;
         src++;
@@ -914,11 +985,13 @@ static const char **searchRecordKey (struct recKeys *reckeys,
 }
 
 struct file_read_info {
-    off_t file_max;
-    off_t file_offset;
-    off_t file_moffset;
+    off_t file_max;	    /* maximum offset so far */
+    off_t file_offset;      /* current offset */
+    off_t file_moffset;     /* offset of rec/rec boundary */
     int file_more;
     int fd;
+    char *sdrbuf;
+    int sdrmax;
 };
 
 static struct file_read_info *file_read_start (int fd)
@@ -928,12 +1001,13 @@ static struct file_read_info *file_read_start (int fd)
     fi->fd = fd;
     fi->file_max = 0;
     fi->file_moffset = 0;
+    fi->sdrbuf = 0;
+    fi->sdrmax = 0;
     return fi;
 }
 
 static void file_read_stop (struct file_read_info *fi)
 {
-    assert (fi);
     xfree (fi);
 }
 
@@ -941,6 +1015,8 @@ static off_t file_seek (void *handle, off_t offset)
 {
     struct file_read_info *p = handle;
     p->file_offset = offset;
+    if (p->sdrbuf)
+	return offset;
     return lseek (p->fd, offset, SEEK_SET);
 }
 
@@ -955,7 +1031,16 @@ static int file_read (void *handle, char *buf, size_t count)
     struct file_read_info *p = handle;
     int fd = p->fd;
     int r;
-    r = read (fd, buf, count);
+    if (p->sdrbuf)
+    {
+	r = count;
+	if (r > p->sdrmax - p->file_offset)
+	    r = p->sdrmax - p->file_offset;
+	if (r)
+	    memcpy (buf, p->sdrbuf + p->file_offset, r);
+    }
+    else
+	r = read (fd, buf, count);
     if (r > 0)
     {
         p->file_offset += r;
@@ -970,7 +1055,7 @@ static void file_begin (void *handle)
     struct file_read_info *p = handle;
 
     p->file_offset = p->file_moffset;
-    if (p->file_moffset)
+    if (!p->sdrbuf && p->file_moffset)
         lseek (p->fd, p->file_moffset, SEEK_SET);
     p->file_more = 0;
 }
@@ -984,26 +1069,13 @@ static void file_end (void *handle, off_t offset)
     p->file_moffset = offset;
 }
 
-static int atois (const char **s)
-{
-    int val = 0, c;
-    while ( (c=**s) >= '0' && c <= '9')
-    {
-        val = val*10 + c - '0';
-        ++(*s);
-    }
-    return val;
-}
-
 static char *fileMatchStr (struct recKeys *reckeys, struct recordGroup *rGroup,
-                           const char *fname,
-                           const char *spec)
+                           const char *fname, const char *spec)
 {
     static char dstBuf[2048];
     char *dst = dstBuf;
     const char *s = spec;
     static const char **w;
-    int i;
 
     while (1)
     {
@@ -1013,21 +1085,39 @@ static char *fileMatchStr (struct recKeys *reckeys, struct recordGroup *rGroup,
             break;
         if (*s == '(')
         {
+	    char attset_str[64], attname_str[64];
+	    data1_attset *attset;
+	    int i;
             char matchFlag[32];
-            int attrSet, attrUse;
+            int attSet = 1, attUse = 1;
             int first = 1;
 
             s++;
-            attrSet = atois (&s);
-            if (*s != ',')
-            {
-                logf (LOG_WARN, "Missing , in match criteria %s in group %s",
-                      spec, rGroup->groupName ? rGroup->groupName : "none");
-                return NULL;
-            }
-            s++;
-            attrUse = atois (&s);
-            w = searchRecordKey (reckeys, attrSet, attrUse);
+	    for (i = 0; *s && *s != ',' && *s != ')'; s++)
+		if (i < 63)
+		    attset_str[i++] = *s;
+	    attset_str[i] = '\0';
+
+	    if (*s == ',')
+	    {
+		s++;
+		for (i = 0; *s && *s != ')'; s++)
+		    if (i < 63)
+			attname_str[i++] = *s;
+		attname_str[i] = '\0';
+	    }
+	    
+	    if ((attset = data1_get_attset (rGroup->dh, attset_str)))
+	    {
+		data1_att *att;
+		attSet = attset->reference;
+		att = data1_getattbyname(rGroup->dh, attset, attname_str);
+		if (att)
+		    attUse = att->value;
+		else
+		    attUse = atoi (attname_str);
+	    }
+            w = searchRecordKey (reckeys, attSet, attUse);
             assert (w);
 
             if (*s == ')')
@@ -1057,7 +1147,7 @@ static char *fileMatchStr (struct recKeys *reckeys, struct recordGroup *rGroup,
             if (first)
             {
                 logf (LOG_WARN, "Record didn't contain match"
-                      " fields in (%d,%d)", attrSet, attrUse);
+                      " fields in (%s,%s)", attset_str, attname_str);
                 return NULL;
             }
         }
@@ -1146,42 +1236,58 @@ static void recordLogPreamble (int level, const char *msg, void *info)
     log_event_start (NULL, NULL);
 }
 
+void addSchema (struct recExtractCtrl *p, Odr_oid *oid)
+{
+    zebraExplain_addSchema (zti, oid);
+}
+
 static int recordExtract (SYSNO *sysno, const char *fname,
                           struct recordGroup *rGroup, int deleteFlag,
-                          struct file_read_info *fi, RecType recType,
-                          char *subType)
+                          struct file_read_info *fi,
+			  RecType recType, char *subType, void *clientData)
 {
-    struct recExtractCtrl extractCtrl;
     RecordAttr *recordAttr;
     int r;
     char *matchStr;
     SYSNO sysnotmp;
-    off_t recordOffset = 0;
     Record rec;
     struct recordLogInfo logInfo;
+    off_t recordOffset = 0;
 
     if (fi->fd != -1)
     {
+	struct recExtractCtrl extractCtrl;
+
         /* we are going to read from a file, so prepare the extraction */
-        extractCtrl.fh = fi;
-        extractCtrl.subType = subType;
-        extractCtrl.init = wordInit;
-        extractCtrl.add = addRecordKey;
-	extractCtrl.dh = rGroup->dh;
+	int i;
 
-        reckeys.buf_used = 0;
-        reckeys.prevAttrUse = -1;
-        reckeys.prevAttrSet = -1;
-        reckeys.prevSeqNo = 0;
-
-        recordOffset = fi->file_moffset;
-        extractCtrl.offset = recordOffset;
-        extractCtrl.readf = file_read;
-        extractCtrl.seekf = file_seek;
-        extractCtrl.tellf = file_tell;
-        extractCtrl.endf = file_end;
+	reckeys.buf_used = 0;
+	reckeys.prevAttrUse = -1;
+	reckeys.prevAttrSet = -1;
+	reckeys.prevSeqNo = 0;
+	
+	recordOffset = fi->file_moffset;
+	extractCtrl.offset = fi->file_moffset;
+	extractCtrl.readf = file_read;
+	extractCtrl.seekf = file_seek;
+	extractCtrl.tellf = file_tell;
+	extractCtrl.endf = file_end;
+	extractCtrl.fh = fi;
+	extractCtrl.subType = subType;
+	extractCtrl.init = wordInit;
+	extractCtrl.addWord = addRecordKey;
+	extractCtrl.addSchema = addSchema;
+	extractCtrl.dh = rGroup->dh;
+	for (i = 0; i<256; i++)
+	{
+	    if (zebra_maps_is_positioned(rGroup->zebra_maps, i))
+		extractCtrl.seqno[i] = 1;
+	    else
+		extractCtrl.seqno[i] = 0;
+	}
 	extractCtrl.zebra_maps = rGroup->zebra_maps;
-        extractCtrl.flagShowRecords = !rGroup->flagRw;
+	extractCtrl.flagShowRecords = !rGroup->flagRw;
+
         if (!rGroup->flagRw)
             printf ("File: %s %ld\n", fname, (long) recordOffset);
 
@@ -1190,18 +1296,20 @@ static int recordExtract (SYSNO *sysno, const char *fname,
         logInfo.rGroup = rGroup;
         log_event_start (recordLogPreamble, &logInfo);
 
-        r = (*recType->extract)(&extractCtrl);
+        r = (*recType->extract)(clientData, &extractCtrl);
 
         log_event_start (NULL, NULL);
 
-        if (r)      
-        {
+	if (r == RECCTRL_EXTRACT_EOF)
+	    return 0;
+	else if (r == RECCTRL_EXTRACT_ERROR)
+	{
             /* error occured during extraction ... */
             if (rGroup->flagRw &&
 		records_processed < rGroup->fileVerboseLimit)
             {
-                logf (LOG_WARN, "fail %s %s %ld code = %d", rGroup->recordType,
-                      fname, (long) recordOffset, r);
+                logf (LOG_WARN, "fail %s %s %ld", rGroup->recordType,
+                      fname, (long) recordOffset);
             }
             return 0;
         }
@@ -1213,7 +1321,7 @@ static int recordExtract (SYSNO *sysno, const char *fname,
                 return 1;
             logf (LOG_WARN, "No keys generated for file %s", fname);
             logf (LOG_WARN, " The file is probably empty");
-            return 0;
+            return 1;
         }
     }
 
@@ -1267,7 +1375,7 @@ static int recordExtract (SYSNO *sysno, const char *fname,
         {
             dict_insert (matchDict, matchStr, sizeof(*sysno), sysno);
         }
-        flushRecordKeys (*sysno, 1, &reckeys, rGroup->databaseName);
+        flushRecordKeys (*sysno, 1, &reckeys);
 	flushSortKeys (*sysno, 1);
 
         records_inserted++;
@@ -1293,7 +1401,7 @@ static int recordExtract (SYSNO *sysno, const char *fname,
         delkeys.buf_used = rec->size[recInfo_delKeys];
 	delkeys.buf = rec->info[recInfo_delKeys];
 	flushSortKeys (*sysno, 0);
-        flushRecordKeys (*sysno, 0, &delkeys, rec->info[recInfo_databaseName]);
+        flushRecordKeys (*sysno, 0, &delkeys);
         if (deleteFlag)
         {
             /* record going to be deleted */
@@ -1331,7 +1439,7 @@ static int recordExtract (SYSNO *sysno, const char *fname,
                 if (records_processed < rGroup->fileVerboseLimit)
                     logf (LOG_LOG, "update %s %s %ld", rGroup->recordType,
                           fname, (long) recordOffset);
-                flushRecordKeys (*sysno, 1, &reckeys, rGroup->databaseName); 
+                flushRecordKeys (*sysno, 1, &reckeys);
                 records_updated++;
             }
         }
@@ -1386,8 +1494,8 @@ static int recordExtract (SYSNO *sysno, const char *fname,
         rec->info[recInfo_storeData] = xmalloc (recordAttr->recordSize);
         if (lseek (fi->fd, recordOffset, SEEK_SET) < 0)
         {
-            logf (LOG_ERRNO|LOG_FATAL, "seek to %ld in %s", fname,
-                  (long) recordOffset);
+            logf (LOG_ERRNO|LOG_FATAL, "seek to %ld in %s",
+                  (long) recordOffset, fname);
             exit (1);
         }
         if (read (fi->fd, rec->info[recInfo_storeData], recordAttr->recordSize)
@@ -1429,6 +1537,7 @@ int fileExtract (SYSNO *sysno, const char *fname,
     struct recordGroup rGroupM;
     struct recordGroup *rGroup = &rGroupM;
     struct file_read_info *fi;
+    void *clientData;
 
     memcpy (rGroup, rGroupP, sizeof(*rGroupP));
    
@@ -1440,12 +1549,10 @@ int fileExtract (SYSNO *sysno, const char *fname,
     logf (LOG_DEBUG, "fileExtract %s", fname);
 
     /* determine file extension */
+    *ext = '\0';
     for (i = strlen(fname); --i >= 0; )
         if (fname[i] == '/')
-        {
-            strcpy (ext, "");
             break;
-        }
         else if (fname[i] == '.')
         {
             strcpy (ext, fname+i+1);
@@ -1458,21 +1565,20 @@ int fileExtract (SYSNO *sysno, const char *fname,
         if (!(rGroup->recordType = res_get (common_resource, ext_res)))
         {
             sprintf (ext_res, "%srecordType", gprefix);
-            if (!(rGroup->recordType = res_get (common_resource, ext_res)))
-            {
-                if (records_processed < rGroup->fileVerboseLimit)
-                    logf (LOG_LOG, "? %s", fname);
-                return 0;
-            }
+            rGroup->recordType = res_get (common_resource, ext_res);
         }
     }
     if (!rGroup->recordType)
     {
         if (records_processed < rGroup->fileVerboseLimit)
-            logf (LOG_LOG, "? record %s", fname);
+            logf (LOG_LOG, "? %s", fname);
         return 0;
     }
-    if (!(recType = recType_byName (rGroup->recordType, subType)))
+    if (!*rGroup->recordType)
+	return 0;
+    if (!(recType =
+	  recType_byName (rGroup->recTypes, rGroup->recordType, subType,
+			  &clientData)))
     {
         logf (LOG_WARN, "No such record type: %s", rGroup->recordType);
         return 0;
@@ -1498,6 +1604,20 @@ int fileExtract (SYSNO *sysno, const char *fname,
     if (!rGroup->databaseName)
         rGroup->databaseName = "Default";
 
+    /* determine if explain database */
+    
+    sprintf (ext_res, "%sexplainDatabase", gprefix);
+    rGroup->explainDatabase =
+	atoi (res_get_def (common_resource, ext_res, "0"));
+
+    /* announce database */
+    if (zebraExplain_curDatabase (zti, rGroup->databaseName))
+    {
+        if (zebraExplain_newDatabase (zti, rGroup->databaseName,
+				      rGroup->explainDatabase))
+            abort ();
+    }
+
     if (rGroup->flagStoreData == -1)
     {
         const char *sval;
@@ -1529,6 +1649,59 @@ int fileExtract (SYSNO *sysno, const char *fname,
     if (rGroup->flagStoreKeys == -1)
         rGroup->flagStoreKeys = 0;
 
+#if ZEBRASDR
+    if (rGroup->useSDR)
+    {
+	ZebraSdrHandle h;
+	char xname[128], *xp;
+
+	strncpy (xname, fname, 127);
+	if (!(xp = strchr (xname, '.')))
+	    return 0;
+	*xp = '\0';
+	if (strcmp (xp+1, "sdr.bits"))
+	    return 0;
+
+        h = zebraSdr_open (xname);
+	if (!h)
+	{
+	    logf (LOG_WARN, "sdr open %s", xname);
+	    return 0;
+	}
+	for (;;)
+	{
+    	    unsigned char *buf;
+	    char sdr_name[128];
+	    int r, segmentno;
+
+	    segmentno = zebraSdr_segment (h, 0);
+	    sprintf (sdr_name, "%%%s.%d", xname, segmentno);
+
+#if 0
+	    if (segmentno > 20)
+		break;
+#endif
+	    r = zebraSdr_read (h, &buf);
+
+    	    if (!r)
+		break;
+
+            fi = file_read_start (0);
+	    fi->sdrbuf = buf;
+	    fi->sdrmax = r;
+	    do
+	    {
+		file_begin (fi);
+		r = recordExtract (sysno, sdr_name, rGroup, deleteFlag, fi,
+                           recType, subType);
+	    } while (r && !sysno && fi->file_more);
+	    file_read_stop (fi);
+	    free (buf);
+	}
+	zebraSdr_close (h);
+	return 1;
+    }
+#endif
     if (sysno && deleteFlag)
         fd = -1;
     else
@@ -1544,7 +1717,7 @@ int fileExtract (SYSNO *sysno, const char *fname,
     {
         file_begin (fi);
         r = recordExtract (sysno, fname, rGroup, deleteFlag, fi,
-                           recType, subType);
+                           recType, subType, clientData);
     } while (r && !sysno && fi->file_more);
     file_read_stop (fi);
     if (fd != -1)
@@ -1552,3 +1725,53 @@ int fileExtract (SYSNO *sysno, const char *fname,
     return r;
 }
 
+static int explain_extract (void *handle, Record rec, data1_node *n)
+{
+    struct recordGroup *rGroup = (struct recordGroup*) handle;
+    struct recExtractCtrl extractCtrl;
+    int i;
+
+    if (zebraExplain_curDatabase (zti, rec->info[recInfo_databaseName]))
+    {
+        if (zebraExplain_newDatabase (zti, rec->info[recInfo_databaseName], 0))
+            abort ();
+    }
+
+    reckeys.buf_used = 0;
+    reckeys.prevAttrUse = -1;
+    reckeys.prevAttrSet = -1;
+    reckeys.prevSeqNo = 0;
+    
+    extractCtrl.init = wordInit;
+    extractCtrl.addWord = addRecordKey;
+    extractCtrl.addSchema = addSchema;
+    extractCtrl.dh = rGroup->dh;
+    for (i = 0; i<256; i++)
+	extractCtrl.seqno[i] = 0;
+    extractCtrl.zebra_maps = rGroup->zebra_maps;
+    extractCtrl.flagShowRecords = !rGroup->flagRw;
+    
+    grs_extract_tree(&extractCtrl, n);
+
+    logf (LOG_DEBUG, "flush explain record, sysno=%d", rec->sysno);
+
+    if (rec->size[recInfo_delKeys])
+    {
+	struct recKeys delkeys;
+
+	delkeys.buf_used = rec->size[recInfo_delKeys];
+	delkeys.buf = rec->info[recInfo_delKeys];
+	flushSortKeys (rec->sysno, 0);
+	flushRecordKeys (rec->sysno, 0, &delkeys);
+    }
+    flushRecordKeys (rec->sysno, 1, &reckeys);
+    flushSortKeys (rec->sysno, 1);
+
+    xfree (rec->info[recInfo_delKeys]);
+    rec->size[recInfo_delKeys] = reckeys.buf_used;
+    rec->info[recInfo_delKeys] = reckeys.buf;
+    reckeys.buf = NULL;
+    reckeys.buf_max = 0;
+
+    return 0;
+}