Sort Keys are stored using the reckeys_t structure. sort files
[idzebra-moved-to-github.git] / index / extract.c
index f1506c5..86f868b 100644 (file)
@@ -1,5 +1,5 @@
-/* $Id: extract.c,v 1.206 2006-03-30 09:52:15 adam Exp $
-   Copyright (C) 1995-2005
+/* $Id: extract.c,v 1.214 2006-05-18 12:03:05 adam Exp $
+   Copyright (C) 1995-2006
    Index Data ApS
 
 This file is part of the Zebra server.
@@ -36,12 +36,19 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
 #include <direntz.h>
 #include <charmap.h>
 
-#if _FILE_OFFSET_BITS == 64
-#define PRINTF_OFF_T "%Ld"
+#ifdef WIN32
+#define PRINTF_OFF_T "%I64d"
+#else
+/* !WIN32 */
+#if SIZEOF_OFF_T == SIZEOF_LONG_LONG
+#define PRINTF_OFF_T "%lld"
 #else
 #define PRINTF_OFF_T "%ld"
 #endif
 
+#endif
+
+
 #define USE_SHELLSORT 0
 
 #if USE_SHELLSORT
@@ -81,6 +88,8 @@ static void logRecord (ZebraHandle zh)
     }
 }
 
+static void extract_add_index_string (RecWord *p, const char *str, int length);
+
 static void extract_set_store_data_prepare(struct recExtractCtrl *p);
 
 static void extract_init (struct recExtractCtrl *p, RecWord *w)
@@ -379,14 +388,25 @@ static void init_extractCtrl(ZebraHandle zh, struct recExtractCtrl *ctrl)
     ctrl->flagShowRecords = !zh->m_flag_rw;
 }
 
-static int file_extract_record(ZebraHandle zh,
-                              SYSNO *sysno, const char *fname,
-                              int deleteFlag,
-                              struct file_read_info *fi,
-                              int force_update,
-                              RecType recType,
-                              void *recTypeClientData)
+static void all_matches_add(struct recExtractCtrl *ctrl)
 {
+    RecWord word;
+    extract_init(ctrl, &word);
+    word.index_name = "allrecords";
+    word.index_type = 'w';
+    word.seqno = 1;
+    extract_add_index_string (&word, "", 0);
+}
+
+static ZEBRA_RES file_extract_record(ZebraHandle zh,
+                                    SYSNO *sysno, const char *fname,
+                                    int deleteFlag,
+                                    struct file_read_info *fi,
+                                    int force_update,
+                                    RecType recType,
+                                    void *recTypeClientData)
+{
+    const char *match_str_to_print = "";
     RecordAttr *recordAttr;
     int r;
     const char *matchStr = 0;
@@ -400,7 +420,7 @@ static int file_extract_record(ZebraHandle zh,
     {
         if (zebraExplain_newDatabase (zh->reg->zei, zh->basenames[0],
                                      zh->m_explain_database))
-           return 0;
+           return ZEBRA_FAIL;
     }
 
     if (fi->fd != -1)
@@ -408,11 +428,7 @@ static int file_extract_record(ZebraHandle zh,
         /* we are going to read from a file, so prepare the extraction */
        zebra_rec_keys_reset(zh->reg->keys);
 
-#if NATTR
        zebra_rec_keys_reset(zh->reg->sortKeys);
-#else
-       zh->reg->sortKeys.buf_used = 0;
-#endif
        recordOffset = fi->file_moffset;
         extractCtrl.handle = zh;
        extractCtrl.offset = fi->file_moffset;
@@ -447,7 +463,7 @@ static int file_extract_record(ZebraHandle zh,
 
         yaz_log_init_prefix2 (0);
        if (r == RECCTRL_EXTRACT_EOF)
-           return 0;
+           return ZEBRA_FAIL;
        else if (r == RECCTRL_EXTRACT_ERROR_GENERIC)
        {
             /* error occured during extraction ... */
@@ -457,7 +473,7 @@ static int file_extract_record(ZebraHandle zh,
                 yaz_log (YLOG_WARN, "fail %s %s " PRINTF_OFF_T, zh->m_record_type,
                       fname, recordOffset);
             }
-            return 0;
+            return ZEBRA_FAIL;
         }
        else if (r == RECCTRL_EXTRACT_ERROR_NO_SUCH_FILTER)
        {
@@ -469,13 +485,20 @@ static int file_extract_record(ZebraHandle zh,
                       PRINTF_OFF_T, zh->m_record_type,
                       fname, recordOffset);
             }
-            return 0;
+            return ZEBRA_FAIL;
         }
+        all_matches_add(&extractCtrl);
         if (extractCtrl.match_criteria[0])
-            matchStr = extractCtrl.match_criteria;     
+            matchStr = extractCtrl.match_criteria;
     }
 
-    /* perform match if sysno not known and if match criteria is specified */
+    /* if matchStr is set now - we assume it's printable .
+       For internal matchStr (see below) we don't print */
+    if (matchStr)
+        match_str_to_print = matchStr;
+
+    /* perform internal match if sysno not known and if match criteria is
+       specified already */
     if (!sysno) 
     {
         sysnotmp = 0;
@@ -488,7 +511,12 @@ static int file_extract_record(ZebraHandle zh,
            if (!matchStr)
            {
                yaz_log(YLOG_WARN, "Bad match criteria");
-               return 0;
+
+                if (zebra_rec_keys_empty(zh->reg->keys))
+                {
+                    yaz_log(YLOG_WARN, "And no index keys");
+                }
+               return ZEBRA_FAIL;
            }
        }
        if (matchStr)
@@ -508,12 +536,12 @@ static int file_extract_record(ZebraHandle zh,
          /* the extraction process returned no information - the record
             is probably empty - unless flagShowRecords is in use */
          if (!zh->m_flag_rw)
-             return 1;
+             return ZEBRA_OK;
   
          if (zh->records_processed < zh->m_file_verbose_limit)
             yaz_log (YLOG_WARN, "empty %s %s " PRINTF_OFF_T, zh->m_record_type,
            fname, recordOffset);
-         return 1;
+         return ZEBRA_OK;
     }
 
     if (! *sysno)
@@ -524,15 +552,20 @@ static int file_extract_record(ZebraHandle zh,
            yaz_log (YLOG_LOG, "delete %s %s " PRINTF_OFF_T, zh->m_record_type,
                  fname, recordOffset);
             yaz_log (YLOG_WARN, "cannot delete record above (seems new)");
-            return 1;
+            return ZEBRA_OK;
         }
-        if (zh->records_processed < zh->m_file_verbose_limit)
-            yaz_log (YLOG_LOG, "add %s %s " PRINTF_OFF_T, zh->m_record_type,
-                  fname, recordOffset);
-        rec = rec_new (zh->reg->records);
 
+        rec = rec_new (zh->reg->records);
+        
         *sysno = rec->sysno;
-
+        
+        if (zh->records_processed < zh->m_file_verbose_limit)
+        {
+            yaz_log(YLOG_LOG, "add %s %s " PRINTF_OFF_T 
+                    " " ZINT_FORMAT " %s" ,
+                    zh->m_record_type,
+                    fname, recordOffset, *sysno, match_str_to_print);
+        }
        recordAttr = rec_init_attr (zh->reg->zei, rec);
        recordAttr->staticrank = extractCtrl.staticrank;
 
@@ -542,11 +575,9 @@ static int file_extract_record(ZebraHandle zh,
             dict_insert_ord(zh->reg->matchDict, db_ord, matchStr,
                            sizeof(*sysno), sysno);
         }
-#if NATTR
+
+
        extract_flushSortKeys (zh, *sysno, 1, zh->reg->sortKeys);
-#else
-       extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
-#endif
         extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys,
                                 recordAttr->staticrank);
         zh->records_inserted++;
@@ -556,11 +587,7 @@ static int file_extract_record(ZebraHandle zh,
         /* record already exists */
        zebra_rec_keys_t delkeys = zebra_rec_keys_open();
 
-#if NATTR
        zebra_rec_keys_t sortKeys = zebra_rec_keys_open();
-#else
-        struct sortKeys sortKeys;
-#endif
 
         rec = rec_get (zh->reg->records, *sysno);
         assert (rec);
@@ -572,18 +599,11 @@ static int file_extract_record(ZebraHandle zh,
                               rec->size[recInfo_delKeys],
                               0);
 
-#if NATTR
        zebra_rec_keys_set_buf(sortKeys,
                               rec->info[recInfo_sortKeys],
                               rec->size[recInfo_sortKeys],
                               0);
        extract_flushSortKeys (zh, *sysno, 0, sortKeys);
-#else
-        sortKeys.buf_used = rec->size[recInfo_sortKeys];
-        sortKeys.buf = rec->info[recInfo_sortKeys];
-       extract_flushSortKeys (zh, *sysno, 0, &sortKeys);
-#endif
-
         extract_flushRecordKeys (zh, *sysno, 0, delkeys,
                                 recordAttr->staticrank); /* old values */  
         if (deleteFlag)
@@ -591,15 +611,20 @@ static int file_extract_record(ZebraHandle zh,
             /* record going to be deleted */
             if (zebra_rec_keys_empty(delkeys))
             {
-                yaz_log (YLOG_LOG, "delete %s %s " PRINTF_OFF_T,
-                        zh->m_record_type, fname, recordOffset);
+                yaz_log (YLOG_LOG, "delete %s %s " PRINTF_OFF_T 
+                         " " ZINT_FORMAT,
+                        zh->m_record_type, fname, recordOffset, *sysno);
                 yaz_log (YLOG_WARN, "cannot delete file above, storeKeys false (1)");
             }
             else
             {
                 if (zh->records_processed < zh->m_file_verbose_limit)
-                    yaz_log (YLOG_LOG, "delete %s %s " PRINTF_OFF_T,
-                            zh->m_record_type, fname, recordOffset);
+                {
+                    yaz_log(YLOG_LOG, "delete %s %s " PRINTF_OFF_T 
+                            " " ZINT_FORMAT " %s" ,
+                            zh->m_record_type,
+                            fname, recordOffset, *sysno, match_str_to_print);
+                }
                 zh->records_deleted++;
                 if (matchStr)
                {
@@ -610,28 +635,26 @@ static int file_extract_record(ZebraHandle zh,
             }
            rec_rm (&rec);
             logRecord (zh);
-            return 1;
+            return ZEBRA_OK;
         }
         else
         {
            /* flush new keys for sort&search etc */
             if (zh->records_processed < zh->m_file_verbose_limit)
-                yaz_log (YLOG_LOG, "update %s %s " PRINTF_OFF_T,
-                      zh->m_record_type, fname, recordOffset);
+            {
+                yaz_log(YLOG_LOG, "update %s %s " PRINTF_OFF_T 
+                        " " ZINT_FORMAT " %s" ,
+                        zh->m_record_type,
+                        fname, recordOffset, *sysno, match_str_to_print);
+            }
            recordAttr->staticrank = extractCtrl.staticrank;
-#if NATTR
             extract_flushSortKeys (zh, *sysno, 1, zh->reg->sortKeys);
-#else
-            extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
-#endif
             extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys,
                                         recordAttr->staticrank);
             zh->records_updated++;
         }
        zebra_rec_keys_close(delkeys);
-#if NATTR
        zebra_rec_keys_close(sortKeys);
-#endif
     }
     /* update file type */
     xfree (rec->info[recInfo_fileType]);
@@ -660,16 +683,9 @@ static int file_extract_record(ZebraHandle zh,
     /* update sort keys */
     xfree (rec->info[recInfo_sortKeys]);
 
-#if NATTR
     zebra_rec_keys_get_buf(zh->reg->sortKeys,
                           &rec->info[recInfo_sortKeys],
                           &rec->size[recInfo_sortKeys]);
-#else
-    rec->size[recInfo_sortKeys] = zh->reg->sortKeys.buf_used;
-    rec->info[recInfo_sortKeys] = zh->reg->sortKeys.buf;
-    zh->reg->sortKeys.buf = NULL;
-    zh->reg->sortKeys.buf_max = 0;
-#endif
 
     /* save file size of original record */
     zebraExplain_recordBytesIncrement (zh->reg->zei,
@@ -727,13 +743,14 @@ static int file_extract_record(ZebraHandle zh,
     /* commit this record */
     rec_put (zh->reg->records, &rec);
     logRecord (zh);
-    return 1;
+    return ZEBRA_OK;
 }
 
-int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname, 
-                int deleteFlag)
+ZEBRA_RES zebra_extract_file(ZebraHandle zh, SYSNO *sysno, const char *fname, 
+                            int deleteFlag)
 {
-    int r, i, fd;
+    ZEBRA_RES r = ZEBRA_OK;
+    int i, fd;
     char gprefix[128];
     char ext[128];
     char ext_res[128];
@@ -784,7 +801,7 @@ int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname,
                          &recTypeClientData)))
     {
         yaz_log(YLOG_WARN, "No such record type: %s", zh->m_record_type);
-        return 0;
+        return ZEBRA_FAIL;
     }
 
     switch(recType->version)
@@ -809,16 +826,15 @@ int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname,
         else
             strcpy (full_rep, fname);
         
-
         if ((fd = open (full_rep, O_BINARY|O_RDONLY)) == -1)
         {
             yaz_log (YLOG_WARN|YLOG_ERRNO, "open %s", full_rep);
            zh->m_record_type = original_record_type;
-            return 0;
+            return ZEBRA_FAIL;
         }
     }
     fi = file_read_start (fd);
-    do
+    while(1)
     {
        fi->file_moffset = fi->file_offset;
        fi->file_more = 0;  /* file_end not called (yet) */
@@ -829,8 +845,15 @@ int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname,
            fi->file_offset = fi->file_moffset;
            lseek(fi->fd, fi->file_moffset, SEEK_SET);
        }
+       if (r != ZEBRA_OK)
+       {
+           break;
+       }
+       if (sysno)
+       {
+           break;
+       }
     }
-    while (r && !sysno);
     file_read_stop (fi);
     if (fd != -1)
         close (fd);
@@ -887,12 +910,8 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh,
     extractCtrl.fh = &fc;
 
     zebra_rec_keys_reset(zh->reg->keys);
-
-#if NATTR
     zebra_rec_keys_reset(zh->reg->sortKeys);
-#else
-    zh->reg->sortKeys.buf_used = 0;
-#endif
+
     if (zebraExplain_curDatabase (zh->reg->zei, zh->basenames[0]))
     {
         if (zebraExplain_newDatabase (zh->reg->zei, zh->basenames[0], 
@@ -954,6 +973,8 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh,
        return ZEBRA_FAIL;
     }
 
+    all_matches_add(&extractCtrl);
+        
     if (extractCtrl.match_criteria[0])
        match_criteria = extractCtrl.match_criteria;
 
@@ -1020,15 +1041,9 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh,
             dict_insert_ord(zh->reg->matchDict, db_ord, matchStr,
                            sizeof(*sysno), sysno);
         }
-#if NATTR
+
+
        extract_flushSortKeys (zh, *sysno, 1, zh->reg->sortKeys);
-#else
-       extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
-#endif
-       
-#if 0
-       print_rec_keys(zh, zh->reg->keys);
-#endif
         extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys,
                         recordAttr->staticrank);
         zh->records_inserted++;
@@ -1037,12 +1052,7 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh,
     {
         /* record already exists */
        zebra_rec_keys_t delkeys = zebra_rec_keys_open();
-#if NATTR
        zebra_rec_keys_t sortKeys = zebra_rec_keys_open();
-#else
-        struct sortKeys sortKeys;
-#endif
-
        if (!allow_update)
        {
            yaz_log (YLOG_LOG, "skipped %s %s %ld", 
@@ -1060,21 +1070,12 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh,
                               rec->info[recInfo_delKeys],
                               rec->size[recInfo_delKeys],
                               0);
-#if NATTR
        zebra_rec_keys_set_buf(sortKeys,
                               rec->info[recInfo_sortKeys],
                               rec->size[recInfo_sortKeys],
                               0);
-#else
-        sortKeys.buf_used = rec->size[recInfo_sortKeys];
-        sortKeys.buf = rec->info[recInfo_sortKeys];
-#endif
 
-#if NATTR
        extract_flushSortKeys (zh, *sysno, 0, sortKeys);
-#else
-       extract_flushSortKeys (zh, *sysno, 0, &sortKeys);
-#endif
         extract_flushRecordKeys (zh, *sysno, 0, delkeys,
                                 recordAttr->staticrank);
         if (delete_flag)
@@ -1110,19 +1111,13 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh,
                    yaz_log (YLOG_LOG, "update %s %s %ld", recordType,
                             pr_fname, (long) recordOffset);
            recordAttr->staticrank = extractCtrl.staticrank;
-#if NATTR
             extract_flushSortKeys (zh, *sysno, 1, zh->reg->sortKeys);
-#else
-            extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
-#endif
             extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys, 
                                         recordAttr->staticrank);
             zh->records_updated++;
         }
        zebra_rec_keys_close(delkeys);
-#if NATTR
        zebra_rec_keys_close(sortKeys);
-#endif
     }
     /* update file type */
     xfree (rec->info[recInfo_fileType]);
@@ -1150,16 +1145,9 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh,
     /* update sort keys */
     xfree (rec->info[recInfo_sortKeys]);
 
-#if NATTR
     zebra_rec_keys_get_buf(zh->reg->sortKeys,
                           &rec->info[recInfo_sortKeys],
                           &rec->size[recInfo_sortKeys]);
-#else
-    rec->size[recInfo_sortKeys] = zh->reg->sortKeys.buf_used;
-    rec->info[recInfo_sortKeys] = zh->reg->sortKeys.buf;
-    zh->reg->sortKeys.buf = NULL;
-    zh->reg->sortKeys.buf_max = 0;
-#endif
 
     /* save file size of original record */
     zebraExplain_recordBytesIncrement (zh->reg->zei,
@@ -1229,12 +1217,8 @@ int explain_extract (void *handle, Record rec, data1_node *n)
     }
 
     zebra_rec_keys_reset(zh->reg->keys);
-    
-#if NATTR
     zebra_rec_keys_reset(zh->reg->sortKeys);
-#else
-    zh->reg->sortKeys.buf_used = 0;
-#endif
+
     extractCtrl.init = extract_init;
     extractCtrl.tokenAdd = extract_token_add;
     extractCtrl.schemaAdd = extract_schema_add;
@@ -1257,36 +1241,23 @@ int explain_extract (void *handle, Record rec, data1_node *n)
     {
        zebra_rec_keys_t delkeys = zebra_rec_keys_open();
        
-#if NATTR
        zebra_rec_keys_t sortkeys = zebra_rec_keys_open();
-#else
-       struct sortKeys sortkeys;
-#endif
 
        zebra_rec_keys_set_buf(delkeys, rec->info[recInfo_delKeys],
                               rec->size[recInfo_delKeys],
                               0);
        extract_flushRecordKeys (zh, rec->sysno, 0, delkeys, 0);
        zebra_rec_keys_close(delkeys);
-#if NATTR
+
        zebra_rec_keys_set_buf(sortkeys, rec->info[recInfo_sortKeys],
                               rec->size[recInfo_sortKeys],
                               0);
 
        extract_flushSortKeys (zh, rec->sysno, 0, sortkeys);
        zebra_rec_keys_close(sortkeys);
-#else
-       sortkeys.buf_used = rec->size[recInfo_sortKeys];
-       sortkeys.buf = rec->info[recInfo_sortKeys];
-       extract_flushSortKeys (zh, rec->sysno, 0, &sortkeys);
-#endif
     }
     extract_flushRecordKeys (zh, rec->sysno, 1, zh->reg->keys, 0);
-#if NATTR
     extract_flushSortKeys (zh, rec->sysno, 1, zh->reg->sortKeys);
-#else
-    extract_flushSortKeys (zh, rec->sysno, 1, &zh->reg->sortKeys);
-#endif
 
     xfree (rec->info[recInfo_delKeys]);
     zebra_rec_keys_get_buf(zh->reg->keys,
@@ -1294,20 +1265,65 @@ int explain_extract (void *handle, Record rec, data1_node *n)
                           &rec->size[recInfo_delKeys]);
 
     xfree (rec->info[recInfo_sortKeys]);
-#if NATTR
     zebra_rec_keys_get_buf(zh->reg->sortKeys,
                           &rec->info[recInfo_sortKeys],
                           &rec->size[recInfo_sortKeys]);
-#else
-    rec->size[recInfo_sortKeys] = zh->reg->sortKeys.buf_used;
-    rec->info[recInfo_sortKeys] = zh->reg->sortKeys.buf;
-    zh->reg->sortKeys.buf = NULL;
-    zh->reg->sortKeys.buf_max = 0;
-#endif
 
     return 0;
 }
 
+void extract_rec_keys_adjust(ZebraHandle zh, int is_insert,
+                             zebra_rec_keys_t reckeys)
+{
+    ZebraExplainInfo zei = zh->reg->zei;
+    struct ord_stat {
+        int no;
+        int ord;
+        struct ord_stat *next;
+    };
+
+    if (zebra_rec_keys_rewind(reckeys))
+    {
+        struct ord_stat *ord_list = 0;
+        struct ord_stat *p;
+       size_t slen;
+       const char *str;
+       struct it_key key_in;
+       while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
+        {
+            int ord = key_in.mem[0]; 
+
+            for (p = ord_list; p ; p = p->next)
+                if (p->ord == ord)
+                {
+                    p->no++;
+                    break;
+                }
+            if (!p)
+            {
+                p = xmalloc(sizeof(*p));
+                p->no = 1;
+                p->ord = ord;
+                p->next = ord_list;
+                ord_list = p;
+            }
+        }
+
+        p = ord_list;
+        while (p)
+        {
+            struct ord_stat *p1 = p;
+
+            if (is_insert)
+                zebraExplain_ord_adjust_occurrences(zei, p->ord, p->no, 1);
+            else
+                zebraExplain_ord_adjust_occurrences(zei, p->ord, - p->no, -1);
+            p = p->next;
+            xfree(p1);
+        }
+    }
+}
+
 void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno,
                               int cmd,
                              zebra_rec_keys_t reckeys,
@@ -1315,6 +1331,8 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno,
 {
     ZebraExplainInfo zei = zh->reg->zei;
 
+    extract_rec_keys_adjust(zh, cmd, reckeys);
+
     if (!zh->reg->key_buf)
     {
        int mem= 1024*1024* atoi( res_get_def( zh->res, "memmax", "8"));
@@ -1361,7 +1379,7 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno,
            zh->reg->key_buf_used +=
                key_SU_encode(ch, (char*)zh->reg->key_buf +
                              zh->reg->key_buf_used);
-           
+
            /* copy the 0-terminated stuff from str to output */
            memcpy((char*)zh->reg->key_buf + zh->reg->key_buf_used, str, slen);
            zh->reg->key_buf_used += slen;
@@ -1554,7 +1572,7 @@ ZEBRA_RES zebra_snippets_rec_keys(ZebraHandle zh,
            
            zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
                                    0/* db */, 0/* set */, 0/* use */,
-                                   0 /* string index */);
+                                   0 /* string_index */);
            assert(index_type);
            zebra_term_untrans_iconv(zh, nmem, index_type,
                                     &dst_term, str);
@@ -1595,7 +1613,7 @@ void print_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys)
     }
 }
 
-void extract_add_index_string (RecWord *p, const char *str, int length)
+void extract_add_index_string(RecWord *p, const char *str, int length)
 {
     struct it_key key;
 
@@ -1647,8 +1665,7 @@ void extract_add_index_string (RecWord *p, const char *str, int length)
     zebra_rec_keys_write(zh->reg->keys, str, length, &key);
 }
 
-#if NATTR
-static void extract_add_sort_string (RecWord *p, const char *str, int length)
+static void extract_add_sort_string(RecWord *p, const char *str, int length)
 {
     struct it_key key;
 
@@ -1664,7 +1681,15 @@ static void extract_add_sort_string (RecWord *p, const char *str, int length)
     }
     else
     {
+#if NATTR
        return;
+#else
+       ch = zebraExplain_lookup_attr_su(zei, p->index_type, 
+                                         VAL_IDXPATH, p->attrUse);
+       if (ch < 0)
+           ch = zebraExplain_add_attr_su(zei, p->index_type,
+                                         VAL_IDXPATH, p->attrUse);
+#endif
     }
     key.len = 4;
     key.mem[0] = ch;
@@ -1672,45 +1697,24 @@ static void extract_add_sort_string (RecWord *p, const char *str, int length)
     key.mem[2] = p->section_id;
     key.mem[3] = p->seqno;
 
-    zebra_rec_keys_write(zh->reg->sortKeys, str, length, &key);
-}
-#else
-static void extract_add_sort_string (RecWord *p, const char *str, int length)
-{
-    ZebraHandle zh = p->extractCtrl->handle;
-    struct sortKeys *sk = &zh->reg->sortKeys;
-    int off = 0;
-
-    while (off < sk->buf_used)
+#if 0
+    if (1)
     {
-        int set, use, slen;
+       char strz[80];
+       int i;
 
-        off += key_SU_decode(&set, (unsigned char *) sk->buf + off);
-        off += key_SU_decode(&use, (unsigned char *) sk->buf + off);
-        off += key_SU_decode(&slen, (unsigned char *) sk->buf + off);
-        off += slen;
-        if (p->attrSet == set && p->attrUse == use)
-            return;
+       strz[0] = 0;
+       for (i = 0; i<length && i < 20; i++)
+           sprintf(strz+strlen(strz), "%02X", str[i] & 0xff);
+       /* just for debugging .. */
+       yaz_log(YLOG_LOG, "add: set=%d use=%d "
+               "record_id=%lld section_id=%lld seqno=%lld %s",
+               p->attrSet, p->attrUse, p->record_id, p->section_id, p->seqno,
+               strz);
     }
-    assert (off == sk->buf_used);
-    
-    if (sk->buf_used + IT_MAX_WORD > sk->buf_max)
-    {
-        char *b;
-        
-        b = (char *) xmalloc (sk->buf_max += 128000);
-        if (sk->buf_used > 0)
-            memcpy (b, sk->buf, sk->buf_used);
-        xfree (sk->buf);
-        sk->buf = b;
-    }
-    off += key_SU_encode(p->attrSet, sk->buf + off);
-    off += key_SU_encode(p->attrUse, sk->buf + off);
-    off += key_SU_encode(length, sk->buf + off);
-    memcpy (sk->buf + off, str, length);
-    sk->buf_used = off + length;
-}
 #endif
+    zebra_rec_keys_write(zh->reg->sortKeys, str, length, &key);
+}
 
 void extract_add_string (RecWord *p, const char *string, int length)
 {
@@ -1884,67 +1888,30 @@ void extract_schema_add (struct recExtractCtrl *p, Odr_oid *oid)
     zebraExplain_addSchema (zh->reg->zei, oid);
 }
 
-#if NATTR
-#error not done yet with zebra_rec_keys_t
 void extract_flushSortKeys (ZebraHandle zh, SYSNO sysno,
-                            int cmd, struct recKeys *reckeys)
+                            int cmd, zebra_rec_keys_t reckeys)
 {
-    SortIdx sortIdx = zh->reg->sortIdx;
-    void *decode_handle = iscz1_start();
-    int off = 0;
-    int ch = 0;
-
-    while (off < reckeys->buf_used)
+    if (zebra_rec_keys_rewind(reckeys))
     {
-        const char *src = reckeys->buf + off;
-        struct it_key key;
-       char *dst = (char*) &key;
-
-       iscz1_decode(decode_handle, &dst, &src);
-       assert(key.len == 4);
-
-       ch = (int) key.mem[0];  /* ordinal for field/use/attribute */
-
-        sortIdx_type(sortIdx, ch);
-        if (cmd == 1)
-            sortIdx_add(sortIdx, src, strlen(src));
-        else
-            sortIdx_add(sortIdx, "", 1);
-       
-       src += strlen(src);
-        src++;
-
-        off = src - reckeys->buf;
-    }
-    assert (off == reckeys->buf_used);
-    iscz1_stop(decode_handle);
-}
-#else
-void extract_flushSortKeys (ZebraHandle zh, SYSNO sysno,
-                            int cmd, struct sortKeys *sk)
-{
-    SortIdx sortIdx = zh->reg->sortIdx;
-    int off = 0;
+        SortIdx sortIdx = zh->reg->sortIdx;
+       size_t slen;
+       const char *str;
+       struct it_key key_in;
 
-    sortIdx_sysno (sortIdx, sysno);
+        sortIdx_sysno (sortIdx, sysno);
 
-    while (off < sk->buf_used)
-    {
-        int set, use, slen;
-        
-        off += key_SU_decode(&set, (unsigned char *) sk->buf + off);
-        off += key_SU_decode(&use, (unsigned char *) sk->buf + off);
-        off += key_SU_decode(&slen, (unsigned char *) sk->buf + off);
-        
-        sortIdx_type(sortIdx, use);
-        if (cmd == 1)
-            sortIdx_add(sortIdx, sk->buf + off, slen);
-        else
-            sortIdx_add(sortIdx, "", 1);
-        off += slen;
+       while (zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
+        {
+            int ord = (int) key_in.mem[0];
+            
+            sortIdx_type(sortIdx, ord);
+            if (cmd == 1)
+                sortIdx_add(sortIdx, str, slen);
+            else
+                sortIdx_add(sortIdx, "", 1);
+        }
     }
 }
-#endif
 
 void encode_key_init (struct encode_info *i)
 {
@@ -2117,3 +2084,11 @@ void encode_key_flush (struct encode_info *i, FILE *outf)
     i->prevseq=0;
 }
 #endif
+/*
+ * Local variables:
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ * vim: shiftwidth=4 tabstop=8 expandtab
+ */
+