Changed the index type to a string everywhere.

[idzebra-moved-to-github.git] / index / extract.c

diff --git a/index/extract.c b/index/extract.c
index f0ea3ab..c70e6a1 100644 (file)
--- a/index/extract.c
+++ b/index/extract.c
@@ -1,4 +1,4 @@
-/* $Id: extract.c,v 1.265 2007-10-29 16:57:52 adam Exp $
+/* $Id: extract.c,v 1.267 2007-10-31 16:56:14 adam Exp $
    Copyright (C) 1995-2007
    Index Data ApS
 
@@ -67,9 +67,6 @@ static void zebra_init_log_level(void)
     }
 }
 
-static void extract_flush_record_keys(ZebraHandle zh, zint sysno,
-                                      int cmd, zebra_rec_keys_t reckeys,
-                                      zint staticrank);
 static void extract_flush_sort_keys(ZebraHandle zh, zint sysno,
                                     int cmd, zebra_rec_keys_t skp);
 static void extract_schema_add(struct recExtractCtrl *p, Odr_oid *oid);
@@ -100,14 +97,6 @@ static void logRecord(ZebraHandle zh)
 
 static void init_extractCtrl(ZebraHandle zh, struct recExtractCtrl *ctrl)
 {
-    int i;
-    for (i = 0; i<256; i++)
-    {
-       if (zebra_maps_is_positioned(zh->reg->zebra_maps, i))
-           ctrl->seqno[i] = 1;
-       else
-           ctrl->seqno[i] = 0;
-    }
     ctrl->flagShowRecords = !zh->m_flag_rw;
 }
 
@@ -135,10 +124,10 @@ struct snip_rec_info {
 };
 
 
-static void snippet_add_complete_field(RecWord *p, int ord)
+static void snippet_add_complete_field(RecWord *p, int ord,
+                                       zebra_map_t zm)
 {
     struct snip_rec_info *h = p->extractCtrl->handle;
-    ZebraHandle zh = h->zh;
 
     const char *b = p->term_buf;
     char buf[IT_MAX_WORD+1];
@@ -148,7 +137,7 @@ static void snippet_add_complete_field(RecWord *p, int ord)
     const char *last = 0;
 
     if (remain > 0)
-       map = zebra_maps_input(zh->reg->zebra_maps, *p->index_type, &b, remain, 1);
+       map = zebra_maps_input(zm, &b, remain, 1);
 
     while (remain > 0 && i < IT_MAX_WORD)
     {
@@ -162,8 +151,7 @@ static void snippet_add_complete_field(RecWord *p, int ord)
            {
                int first = i ? 0 : 1;  /* first position */
 
-               map = zebra_maps_input(zh->reg->zebra_maps, *p->index_type, 
-                                       &b, remain, first);
+               map = zebra_maps_input(zm, &b, remain, first);
            }
            else
                map = 0;
@@ -192,8 +180,7 @@ static void snippet_add_complete_field(RecWord *p, int ord)
            remain = p->term_len  - (b - p->term_buf);
            if (remain > 0)
            {
-               map = zebra_maps_input(zh->reg->zebra_maps, *p->index_type, &b,
-                                       remain, 0);
+               map = zebra_maps_input(zm, &b, remain, 0);
            }
            else
                map = 0;
@@ -206,10 +193,9 @@ static void snippet_add_complete_field(RecWord *p, int ord)
                                start, last - start);
 }
 
-static void snippet_add_incomplete_field(RecWord *p, int ord)
+static void snippet_add_incomplete_field(RecWord *p, int ord, zebra_map_t zm)
 {
     struct snip_rec_info *h = p->extractCtrl->handle;
-    ZebraHandle zh = h->zh;
     const char *b = p->term_buf;
     int remain = p->term_len;
     int first = 1;
@@ -218,7 +204,7 @@ static void snippet_add_incomplete_field(RecWord *p, int ord)
     const char *last = b;
 
     if (remain > 0)
-       map = zebra_maps_input(zh->reg->zebra_maps, *p->index_type, &b, remain, 0);
+       map = zebra_maps_input(zm, &b, remain, 0);
 
     while (map)
     {
@@ -231,8 +217,7 @@ static void snippet_add_incomplete_field(RecWord *p, int ord)
            remain = p->term_len - (b - p->term_buf);
             last = b;
            if (remain > 0)
-               map = zebra_maps_input(zh->reg->zebra_maps, *p->index_type, &b,
-                                      remain, 0);
+               map = zebra_maps_input(zm, &b, remain, 0);
            else
                map = 0;
        }
@@ -256,7 +241,7 @@ static void snippet_add_incomplete_field(RecWord *p, int ord)
            remain = p->term_len - (b - p->term_buf);
             last = b;
            if (remain > 0)
-               map = zebra_maps_input(zh->reg->zebra_maps, *p->index_type, &b, remain, 0);
+               map = zebra_maps_input(zm, &b, remain, 0);
            else
                map = 0;
        }
@@ -266,7 +251,7 @@ static void snippet_add_incomplete_field(RecWord *p, int ord)
         if (first)
         {   
             first = 0;
-            if (zebra_maps_is_first_in_field(zh->reg->zebra_maps, *p->index_type))
+            if (zebra_maps_is_first_in_field(zm))
             {
                 /* first in field marker */
                 p->seqno++;
@@ -285,17 +270,18 @@ static void snippet_token_add(RecWord *p)
 {
     struct snip_rec_info *h = p->extractCtrl->handle;
     ZebraHandle zh = h->zh;
+    zebra_map_t zm = zebra_map_get(zh->reg->zebra_maps, p->index_type);
 
-    if (zebra_maps_is_index(zh->reg->zebra_maps, *p->index_type))
+    if (zm && zebra_maps_is_index(zm))
     {
         ZebraExplainInfo zei = zh->reg->zei;
         int ch = zebraExplain_lookup_attr_str(
             zei, zinfo_index_category_index, p->index_type, p->index_name);
 
-        if(zebra_maps_is_complete (h->zh->reg->zebra_maps, *p->index_type))
-            snippet_add_complete_field(p, ch);
+        if (zebra_maps_is_complete(zm))
+            snippet_add_complete_field(p, ch, zm);
         else
-            snippet_add_incomplete_field(p, ch);
+            snippet_add_incomplete_field(p, ch, zm);
     }
 }
 
@@ -1279,7 +1265,7 @@ void extract_rec_keys_log(ZebraHandle zh, int is_insert,
             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
                                     0/* db */, &string_index);
             assert(index_type);
-            zebra_term_untrans_iconv(zh, nmem, *index_type,
+            zebra_term_untrans_iconv(zh, nmem, index_type,
                                      &dst_term, str);
             *keystr = '\0';
             for (i = 0; i<key.len; i++)
@@ -1446,47 +1432,6 @@ void extract_flush_record_keys2(ZebraHandle zh, zint sysno,
     yaz_log(log_level_extract, "normal=%d optimized=%d", normal, optimized);
 }
 
-void extract_flush_record_keys(ZebraHandle zh, zint sysno, int cmd,
-                               zebra_rec_keys_t reckeys,
-                               zint staticrank)
-{
-    ZebraExplainInfo zei = zh->reg->zei;
-
-    extract_rec_keys_adjust(zh, cmd, reckeys);
-
-    if (log_level_details)
-    {
-        yaz_log(log_level_details, "Keys for record " ZINT_FORMAT " %s",
-                sysno, cmd ? "insert" : "delete");
-        extract_rec_keys_log(zh, cmd, reckeys, log_level_details);
-    }
-
-    if (!zh->reg->key_block)
-    {
-       int mem = 1024*1024 * atoi( res_get_def( zh->res, "memmax", "8"));
-        const char *key_tmp_dir = res_get_def(zh->res, "keyTmpDir", ".");
-        int use_threads = atoi(res_get_def(zh->res, "threads", "1"));
-        zh->reg->key_block = key_block_create(mem, key_tmp_dir, use_threads);
-    }
-    zebraExplain_recordCountIncrement(zei, cmd ? 1 : -1);
-
-#if 0
-    yaz_log(YLOG_LOG, "sysno=" ZINT_FORMAT " cmd=%d", sysno, cmd);
-    print_rec_keys(zh, reckeys);
-#endif
-    if (zebra_rec_keys_rewind(reckeys))
-    {
-       size_t slen;
-       const char *str;
-       struct it_key key_in;
-       while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
-       {
-            key_block_write(zh->reg->key_block, sysno, 
-                            &key_in, cmd, str, slen,
-                            staticrank, zh->m_staticrank);
-       }
-    }
-}
 
 ZEBRA_RES zebra_rec_keys_to_snippets(ZebraHandle zh,
                                      zebra_rec_keys_t reckeys,
@@ -1512,7 +1457,7 @@ ZEBRA_RES zebra_rec_keys_to_snippets(ZebraHandle zh,
            zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
                                    0/* db */, 0 /* string_index */);
            assert(index_type);
-           zebra_term_untrans_iconv(zh, nmem, *index_type,
+           zebra_term_untrans_iconv(zh, nmem, index_type,
                                     &dst_term, str);
            zebra_snippets_append(snippets, seqno, 0, ord, dst_term);
            nmem_reset(nmem);
@@ -1543,7 +1488,7 @@ void print_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys)
            
            seqno = key.mem[key.len-1];
            
-           zebra_term_untrans(zh, *index_type, dst_buf, str);
+           zebra_term_untrans(zh, index_type, dst_buf, str);
            
            yaz_log(YLOG_LOG, "ord=%d seqno=" ZINT_FORMAT 
                     " term=%s", ord, seqno, dst_buf); 
@@ -1608,19 +1553,19 @@ static void extract_add_staticrank_string(RecWord *p,
     ctrl->staticrank = atozint(valz);
 }
 
-static void extract_add_string(RecWord *p, const char *string, int length)
+static void extract_add_string(RecWord *p, zebra_map_t zm,
+                               const char *string, int length)
 {
-    ZebraHandle zh = p->extractCtrl->handle;
     assert(length > 0);
 
     if (!p->index_name)
         return;
 
-    if (zebra_maps_is_index(zh->reg->zebra_maps, *p->index_type))
+    if (zebra_maps_is_index(zm))
     {
        extract_add_index_string(p, zinfo_index_category_index,
                                  string, length);
-        if (zebra_maps_is_alwaysmatches(zh->reg->zebra_maps, *p->index_type))
+        if (zebra_maps_is_alwaysmatches(zm))
         {
             RecWord word;
             memcpy(&word, p, sizeof(word));
@@ -1630,26 +1575,25 @@ static void extract_add_string(RecWord *p, const char *string, int length)
                 &word, zinfo_index_category_alwaysmatches, "", 0);
         }
     }
-    else if (zebra_maps_is_sort(zh->reg->zebra_maps, *p->index_type))
+    else if (zebra_maps_is_sort(zm))
     {
        extract_add_sort_string(p, string, length);
     }
-    else if (zebra_maps_is_staticrank(zh->reg->zebra_maps, *p->index_type))
+    else if (zebra_maps_is_staticrank(zm))
     {
        extract_add_staticrank_string(p, string, length);
     }
 }
 
-static void extract_add_incomplete_field(RecWord *p)
+static void extract_add_incomplete_field(RecWord *p, zebra_map_t zm)
 {
-    ZebraHandle zh = p->extractCtrl->handle;
     const char *b = p->term_buf;
     int remain = p->term_len;
     int first = 1;
     const char **map = 0;
     
     if (remain > 0)
-       map = zebra_maps_input(zh->reg->zebra_maps, *p->index_type, &b, remain, 0);
+       map = zebra_maps_input(zm, &b, remain, 0);
 
     while (map)
     {
@@ -1661,8 +1605,7 @@ static void extract_add_incomplete_field(RecWord *p)
        {
            remain = p->term_len - (b - p->term_buf);
            if (remain > 0)
-               map = zebra_maps_input(zh->reg->zebra_maps, *p->index_type, &b,
-                                      remain, 0);
+               map = zebra_maps_input(zm, &b, remain, 0);
            else
                map = 0;
        }
@@ -1677,7 +1620,7 @@ static void extract_add_incomplete_field(RecWord *p)
                buf[i++] = *(cp++);
            remain = p->term_len - (b - p->term_buf);
            if (remain > 0)
-               map = zebra_maps_input(zh->reg->zebra_maps, *p->index_type, &b, remain, 0);
+               map = zebra_maps_input(zm, &b, remain, 0);
            else
                map = 0;
        }
@@ -1687,28 +1630,27 @@ static void extract_add_incomplete_field(RecWord *p)
         if (first)
         {   
             first = 0;
-            if (zebra_maps_is_first_in_field(zh->reg->zebra_maps, *p->index_type))
+            if (zebra_maps_is_first_in_field(zm))
             {
                 /* first in field marker */
-                extract_add_string(p, FIRST_IN_FIELD_STR, FIRST_IN_FIELD_LEN);
+                extract_add_string(p, zm, FIRST_IN_FIELD_STR, FIRST_IN_FIELD_LEN);
                 p->seqno++;
             }
         }
-       extract_add_string(p, buf, i);
+       extract_add_string(p, zm, buf, i);
         p->seqno++;
     }
 }
 
-static void extract_add_complete_field(RecWord *p)
+static void extract_add_complete_field(RecWord *p, zebra_map_t zm)
 {
-    ZebraHandle zh = p->extractCtrl->handle;
     const char *b = p->term_buf;
     char buf[IT_MAX_WORD+1];
     const char **map = 0;
     int i = 0, remain = p->term_len;
 
     if (remain > 0)
-       map = zebra_maps_input(zh->reg->zebra_maps, *p->index_type, &b, remain, 1);
+       map = zebra_maps_input(zm, &b, remain, 1);
 
     while (remain > 0 && i < IT_MAX_WORD)
     {
@@ -1719,7 +1661,7 @@ static void extract_add_complete_field(RecWord *p)
            if (remain > 0)
            {
                int first = i ? 0 : 1;  /* first position */
-               map = zebra_maps_input(zh->reg->zebra_maps, *p->index_type, &b, remain, first);
+               map = zebra_maps_input(zm, &b, remain, first);
            }
            else
                map = 0;
@@ -1747,8 +1689,7 @@ static void extract_add_complete_field(RecWord *p)
            remain = p->term_len  - (b - p->term_buf);
            if (remain > 0)
            {
-               map = zebra_maps_input(zh->reg->zebra_maps, *p->index_type, &b,
-                                       remain, 0);
+               map = zebra_maps_input(zm, &b, remain, 0);
            }
            else
                map = 0;
@@ -1756,7 +1697,7 @@ static void extract_add_complete_field(RecWord *p)
     }
     if (!i)
        return;
-    extract_add_string(p, buf, i);
+    extract_add_string(p, zm, buf, i);
 }
 
 static void extract_token_add2_index(ZebraHandle zh, zebra_index_type_t type,
@@ -1827,6 +1768,7 @@ static void extract_token_add2(RecWord *p)
 static void extract_token_add(RecWord *p)
 {
     ZebraHandle zh = p->extractCtrl->handle;
+    zebra_map_t zm = zebra_map_get_or_add(zh->reg->zebra_maps, p->index_type);
     WRBUF wrbuf;
 
     if (log_level_details)
@@ -1836,16 +1778,15 @@ static void extract_token_add(RecWord *p)
                 p->index_type, p->index_name, 
                 p->seqno, p->term_len, p->term_buf);
     }
-    if ((wrbuf = zebra_replace(zh->reg->zebra_maps, *p->index_type, 0,
-                              p->term_buf, p->term_len)))
+    if ((wrbuf = zebra_replace(zm, 0, p->term_buf, p->term_len)))
     {
        p->term_buf = wrbuf_buf(wrbuf);
        p->term_len = wrbuf_len(wrbuf);
     }
-    if (zebra_maps_is_complete(zh->reg->zebra_maps, *p->index_type))
-       extract_add_complete_field(p);
+    if (zebra_maps_is_complete(zm))
+       extract_add_complete_field(p, zm);
     else
-       extract_add_incomplete_field(p);
+       extract_add_incomplete_field(p, zm);
 }
 
 static void extract_set_store_data_cb(struct recExtractCtrl *p,