Added function zebra_create_rset_isam .
[idzebra-moved-to-github.git] / index / retrieve.c
index bcb1556..d3928d9 100644 (file)
@@ -1,4 +1,4 @@
-/* $Id: retrieve.c,v 1.69 2007-04-16 21:54:37 adam Exp $
+/* $Id: retrieve.c,v 1.78 2007-12-03 13:04:04 adam Exp $
    Copyright (C) 1995-2007
    Index Data ApS
 
@@ -34,14 +34,18 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 
 #include "index.h"
 #include <yaz/diagbib1.h>
+#include <yaz/snprintf.h>
 #include <direntz.h>
 #include <yaz/oid_db.h>
+#include <zebra_strmap.h>
+
+#define MAX_SYSNOS_PER_RECORD 40
 
 #define ZEBRA_XML_HEADER_STR "<record xmlns=\"http://www.indexdata.com/zebra/\""
 
 static int zebra_create_record_stream(ZebraHandle zh, 
-                               Record *rec,
-                               struct ZebraRecStream *stream)
+                                      Record *rec,
+                                      struct ZebraRecStream *stream)
 {
     RecordAttr *recordAttr = rec_init_attr(zh->reg->zei, *rec);
 
@@ -61,8 +65,8 @@ static int zebra_create_record_stream(ZebraHandle zh,
         else
             strcpy(full_rep, (*rec)->info[recInfo_filename]);
             
-        if ((fd = open (full_rep, O_BINARY|O_RDONLY)) == -1){
-            yaz_log (YLOG_WARN|YLOG_ERRNO, "Retrieve fail; missing file: %s",
+        if ((fd = open(full_rep, O_BINARY|O_RDONLY)) == -1){
+            yaz_log(YLOG_WARN|YLOG_ERRNO, "Retrieve fail; missing file: %s",
                      full_rep);
             rec_free(rec);
             return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
@@ -73,7 +77,61 @@ static int zebra_create_record_stream(ZebraHandle zh,
 }
     
 
+struct index_spec {
+    const char *index_name;
+    const char *index_type;
+    struct index_spec *next;
+};
+
+
+struct index_spec *parse_index_spec(const char *elem, NMEM nmem,
+                                    int *error)
+{
+    struct index_spec *first = 0;
+    struct index_spec **last = &first;
+    const char *cp = elem;
+
+    *error = 0;
+    if (cp[0] == ':' && cp[1] == ':')
+    {
 
+        cp++; /* skip first ':' */
+
+        for (;;)
+        {
+            const char *cp0;
+            struct index_spec *spec = nmem_malloc(nmem, sizeof(*spec));
+            spec->index_type = 0;
+            spec->next = 0;
+
+            if (!first)
+                first = spec;
+            *last = spec;
+            last = &spec->next;
+
+            cp++; /* skip ',' or second ':' */
+            cp0 = cp;
+            while (*cp != ':' && *cp != '\0' && *cp != ',')
+                cp++;
+            spec->index_name = nmem_strdupn(nmem, cp0, cp - cp0);
+            if (*cp == ':') /* type as well */
+            {
+                cp++;
+                cp0 = cp;
+                
+                while (*cp != '\0' && *cp != ',')
+                    cp++;
+                spec->index_type = nmem_strdupn(nmem, cp0, cp - cp0);
+            }
+            if (*cp != ',')
+                break;
+        }
+    }
+    if (*cp != '\0')
+        *error = 1;
+    return first;
+}
+                            
 static int parse_zebra_elem(const char *elem,
                             const char **index, size_t *index_len,
                             const char **type, size_t *type_len)
@@ -121,8 +179,8 @@ static int parse_zebra_elem(const char *elem,
 
 int zebra_special_sort_fetch(ZebraHandle zh, zint sysno, ODR odr,
                              const char *elemsetname,
-                             const int *input_format,
-                             const int **output_format,
+                             const Odr_oid *input_format,
+                             const Odr_oid **output_format,
                              char **rec_bufp, int *rec_lenp)
 {
     const char *retrieval_index;
@@ -130,6 +188,7 @@ int zebra_special_sort_fetch(ZebraHandle zh, zint sysno, ODR odr,
     const char *retrieval_type;
     size_t retrieval_type_len;
     char retrieval_index_cstr[256];
+    char retrieval_type_cstr[256];
     int ord;
 
     /* only accept XML and SUTRS requests */
@@ -160,9 +219,12 @@ int zebra_special_sort_fetch(ZebraHandle zh, zint sysno, ODR odr,
     memcpy(retrieval_index_cstr, retrieval_index, retrieval_index_len);
     retrieval_index_cstr[retrieval_index_len] = '\0';
 
+    memcpy(retrieval_type_cstr, retrieval_type, retrieval_type_len);
+    retrieval_type_cstr[retrieval_type_len] = '\0';
+
     ord = zebraExplain_lookup_attr_str(zh->reg->zei,
                                        zinfo_index_category_sort,
-                                       retrieval_type[0],
+                                       retrieval_type_cstr,
                                        retrieval_index_cstr);
     if (ord == -1)
         return -1;  /* is not a sort index */
@@ -170,7 +232,7 @@ int zebra_special_sort_fetch(ZebraHandle zh, zint sysno, ODR odr,
     {
         char dst_buf[IT_MAX_WORD];
         char str[IT_MAX_WORD];
-        int index_type;
+        const char *index_type;
         const char *db = 0;
         const char *string_index = 0;
         WRBUF wrbuf = wrbuf_alloc();
@@ -182,7 +244,6 @@ int zebra_special_sort_fetch(ZebraHandle zh, zint sysno, ODR odr,
         zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db, &string_index);
         
         zebra_term_untrans(zh, index_type, dst_buf, str);
-        
 
         if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
         {
@@ -194,7 +255,7 @@ int zebra_special_sort_fetch(ZebraHandle zh, zint sysno, ODR odr,
 
             wrbuf_printf(wrbuf, "  <index name=\"%s\"", 
                          string_index);
-            wrbuf_printf(wrbuf, " type=\"%c\">", index_type);
+            wrbuf_printf(wrbuf, " type=\"%s\">", index_type);
             wrbuf_xmlputs(wrbuf, dst_buf);
             wrbuf_printf(wrbuf, "</index>\n");
             wrbuf_printf(wrbuf, "</record>\n");
@@ -203,7 +264,7 @@ int zebra_special_sort_fetch(ZebraHandle zh, zint sysno, ODR odr,
         {
             *output_format = yaz_oid_recsyn_sutrs;
             
-            wrbuf_printf(wrbuf, "%s %c %s\n", string_index, index_type,
+            wrbuf_printf(wrbuf, "%s %s %s\n", string_index, index_type,
                          dst_buf);
         }
         *rec_lenp = wrbuf_len(wrbuf);
@@ -217,8 +278,8 @@ int zebra_special_sort_fetch(ZebraHandle zh, zint sysno, ODR odr,
 int zebra_special_index_fetch(ZebraHandle zh, zint sysno, ODR odr,
                               Record rec,
                               const char *elemsetname,
-                              const int *input_format,
-                              const int **output_format,
+                              const Odr_oid *input_format,
+                              const Odr_oid **output_format,
                               char **rec_bufp, int *rec_lenp)
 {
     const char *retrieval_index;
@@ -227,6 +288,7 @@ int zebra_special_index_fetch(ZebraHandle zh, zint sysno, ODR odr,
     size_t retrieval_type_len;
     zebra_rec_keys_t keys;
     int ret_code = 0;
+    char retrieval_type_cstr[256];
     
     /* set output variables before processing possible error states */
     /* *rec_lenp = 0; */
@@ -246,11 +308,12 @@ int zebra_special_index_fetch(ZebraHandle zh, zint sysno, ODR odr,
                      &retrieval_type,  &retrieval_type_len))
         return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
 
-    if (retrieval_type_len != 0 && retrieval_type_len != 1)
+    if (retrieval_type_len)
     {
-        return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
+        memcpy(retrieval_type_cstr, retrieval_type, retrieval_type_len);
+        retrieval_type_cstr[retrieval_type_len] = '\0';
     }
-
+    
     if (retrieval_index_len)
     {
         char retrieval_index_cstr[256];
@@ -262,8 +325,8 @@ int zebra_special_index_fetch(ZebraHandle zh, zint sysno, ODR odr,
             
             if (zebraExplain_lookup_attr_str(zh->reg->zei,
                                              zinfo_index_category_index,
-                                             (retrieval_type_len == 0 ? -1 : 
-                                              retrieval_type[0]),
+                                             (retrieval_type_len == 0 ? 0 : 
+                                              retrieval_type_cstr),
                                              retrieval_index_cstr) == -1)
                 return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
         }
@@ -299,7 +362,7 @@ int zebra_special_index_fetch(ZebraHandle zh, zint sysno, ODR odr,
         {
             int i;
             int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
-            int index_type;
+            const char *index_type;
             const char *db = 0;
             const char *string_index = 0;
             size_t string_index_len;
@@ -318,8 +381,7 @@ int zebra_special_index_fetch(ZebraHandle zh, zint sysno, ODR odr,
             {
                 /* process only if type is not defined, or is matching */
                 if (retrieval_type == 0 
-                    || (retrieval_type_len == 1 
-                        && retrieval_type[0] == index_type))
+                    || !strcmp(retrieval_type_cstr, index_type))
                 {
                     zebra_term_untrans(zh, index_type, dst_buf, str);
                     if (strlen(dst_buf))
@@ -329,7 +391,7 @@ int zebra_special_index_fetch(ZebraHandle zh, zint sysno, ODR odr,
                             wrbuf_printf(wrbuf, "  <index name=\"%s\"", 
                                          string_index);
                             
-                            wrbuf_printf(wrbuf, " type=\"%c\"", index_type);
+                            wrbuf_printf(wrbuf, " type=\"%s\"", index_type);
                             
                             wrbuf_printf(wrbuf, " seq=\"" ZINT_FORMAT "\">", 
                                          key_in.mem[key_in.len -1]);
@@ -341,13 +403,12 @@ int zebra_special_index_fetch(ZebraHandle zh, zint sysno, ODR odr,
                         {
                             wrbuf_printf(wrbuf, "%s ", string_index);
                             
-                            wrbuf_printf(wrbuf, "%c", index_type);
+                            wrbuf_printf(wrbuf, "%s", index_type);
                             
                             for (i = 1; i < key_in.len; i++)
                                 wrbuf_printf(wrbuf, " " ZINT_FORMAT, 
                                              key_in.mem[i]);
                             
-                            /* zebra_term_untrans(zh, index_type, dst_buf, str); */
                             wrbuf_printf(wrbuf, " %s", dst_buf);
                         
                             wrbuf_printf(wrbuf, "\n");
@@ -399,10 +460,396 @@ static void retrieve_puts_int(WRBUF wrbuf, const char *name,
     wrbuf_printf(wrbuf, "%s %i\n", name, value);
 }
 
-int zebra_special_fetch(ZebraHandle zh, zint sysno, int score, ODR odr,
+
+static void snippet_xml_record(ZebraHandle zh, WRBUF wrbuf, zebra_snippets *doc)
+{
+    const zebra_snippet_word *doc_w;
+    int mark_state = 0;
+
+    wrbuf_printf(wrbuf, "%s>\n", ZEBRA_XML_HEADER_STR);
+    for (doc_w = zebra_snippets_constlist(doc); doc_w; doc_w = doc_w->next)
+    {
+        if (doc_w->mark)
+        {
+            const char *index_type;
+            const char *db = 0;
+            const char *string_index = 0;
+
+            zebraExplain_lookup_ord(zh->reg->zei, doc_w->ord, 
+                                    &index_type, &db, &string_index);
+
+            if (mark_state == 0)
+            {
+                wrbuf_printf(wrbuf, "  <snippet name=\"%s\"",  string_index);
+                wrbuf_printf(wrbuf, " type=\"%s\">", index_type);
+            }
+            if (doc_w->match)
+                wrbuf_puts(wrbuf, "<s>");
+            /* not printing leading ws */
+            if (mark_state || !doc_w->ws || doc_w->match) 
+                wrbuf_xmlputs(wrbuf, doc_w->term);
+            if (doc_w->match)
+                wrbuf_puts(wrbuf, "</s>");
+        }
+        else if (mark_state == 1)
+        {
+            wrbuf_puts(wrbuf, "</snippet>\n");
+        }
+        mark_state = doc_w->mark;
+    }
+    if (mark_state == 1)
+    {
+        wrbuf_puts(wrbuf, "</snippet>\n");
+    }
+    wrbuf_printf(wrbuf, "</record>");
+}
+
+int zebra_get_rec_snippets(ZebraHandle zh, zint sysno,
+                           zebra_snippets *snippets)
+{
+    int return_code = 0;
+    Record rec = rec_get(zh->reg->records, sysno);
+    if (!rec)
+    {
+        yaz_log(YLOG_WARN, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
+        return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
+    }
+    else
+    {
+        const char *file_type = rec->info[recInfo_fileType];
+        void *recTypeClientData;
+        RecType rt = recType_byName(zh->reg->recTypes, zh->res,
+                                    file_type, &recTypeClientData);
+
+        if (!rt)
+            return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
+        else
+        {
+            struct ZebraRecStream stream;
+            return_code = zebra_create_record_stream(zh, &rec, &stream);
+            if (return_code == 0)
+            {
+                extract_snippet(zh, snippets, &stream,
+                                rt, recTypeClientData);
+
+                stream.destroy(&stream);
+            }
+        }
+        rec_free(&rec);
+    }
+    return return_code;
+}
+
+static int snippet_fetch(ZebraHandle zh, const char *setname,
+                         zint sysno, ODR odr,
+                         const char *elemsetname,
+                         const Odr_oid *input_format,
+                         const Odr_oid **output_format,
+                         char **rec_bufp, int *rec_lenp)
+{
+    zebra_snippets *rec_snippets = zebra_snippets_create();
+    int return_code = zebra_get_rec_snippets(zh, sysno, rec_snippets);
+
+    if (!return_code)
+    {
+        WRBUF wrbuf = wrbuf_alloc();
+        zebra_snippets *hit_snippet = zebra_snippets_create();
+
+        zebra_snippets_hit_vector(zh, setname, sysno, hit_snippet);
+
+#if 0
+        /* for debugging purposes */
+        yaz_log(YLOG_LOG, "---------------------------");
+        yaz_log(YLOG_LOG, "REC SNIPPET:");
+        zebra_snippets_log(rec_snippet, YLOG_LOG, 1);
+        yaz_log(YLOG_LOG, "---------------------------");
+        yaz_log(YLOG_LOG, "HIT SNIPPET:");
+        zebra_snippets_log(hit_snippet, YLOG_LOG, 1);
+#endif
+        
+        zebra_snippets_ring(rec_snippets, hit_snippet, 5, 5);
+        
+#if 0
+        yaz_log(YLOG_LOG, "---------------------------");
+        yaz_log(YLOG_LOG, "RING SNIPPET:");
+        zebra_snippets_log(rec_snippets, YLOG_LOG, 1);
+#endif
+        snippet_xml_record(zh, wrbuf, rec_snippets);
+        
+        *output_format = yaz_oid_recsyn_xml;
+        
+        if (return_code == 0)
+        {
+            *rec_lenp = wrbuf_len(wrbuf);
+            *rec_bufp = odr_strdup(odr, wrbuf_cstr(wrbuf));
+        }
+        wrbuf_destroy(wrbuf);
+        zebra_snippets_destroy(hit_snippet);
+    }
+    zebra_snippets_destroy(rec_snippets);
+    return return_code;
+}
+
+struct term_collect {
+    const char *term;
+    int oc;
+    zint set_occur;
+};
+
+zint freq_term(ZebraHandle zh, int ord, const char *term, RSET rset_set)
+{
+    struct rset_key_control *kc = zebra_key_control_create(zh);
+    char ord_buf[IT_MAX_WORD];
+    int ord_len = key_SU_encode(ord, ord_buf);
+    char *info;
+    zint hits = 0;
+    NMEM nmem = nmem_create();
+    
+    strcpy(ord_buf + ord_len, term);
+    
+    info = dict_lookup(zh->reg->dict, ord_buf);
+    if (info)
+    {
+        ISAM_P isam_p;
+        RSET rsets[2], rset;
+        memcpy(&isam_p, info+1, sizeof(ISAM_P));
+        
+        rsets[0] = zebra_create_rset_isam(zh, nmem, kc, kc->scope, isam_p, 0);
+        rsets[1] = rset_dup(rset_set);
+        
+        rset = rset_create_and(nmem, kc, kc->scope, 2, rsets);
+
+        zebra_count_set(zh, rset, &hits, zh->approx_limit);
+
+        rset_delete(rsets[0]);
+        rset_delete(rset);
+    }
+    (*kc->dec)(kc);
+    nmem_destroy(nmem);
+    return hits;
+}
+
+void term_collect_freq(ZebraHandle zh,
+                       struct term_collect *col, int no_terms_collect,
+                       int ord, RSET rset)
+{
+    int i;
+    for (i = 0; i < no_terms_collect; i++)
+    {
+        if (col[i].term)
+            col[i].set_occur = freq_term(zh, ord, col[i].term, rset);
+    }
+}
+
+struct term_collect *term_collect_create(zebra_strmap_t sm, 
+                                         int no_terms_collect,
+                                         NMEM nmem)
+{
+    const char *term;
+    void *data_buf;
+    size_t data_len;
+    zebra_strmap_it it;
+    struct term_collect *col = nmem_malloc(nmem, 
+                                           sizeof *col *no_terms_collect);
+    int i;
+    for (i = 0; i < no_terms_collect; i++)
+    {
+        col[i].term = 0;
+        col[i].oc = 0;
+        col[i].set_occur = 0;
+    }
+    /* iterate over terms and collect the most frequent ones */
+    it = zebra_strmap_it_create(sm);
+    while ((term = zebra_strmap_it_next(it, &data_buf, &data_len)))
+    {
+        int oc = *(int*) data_buf;
+        int j = 0;
+        /* insertion may be slow but terms terms will be "infrequent" and
+           thus number of iterations should be small below */
+        while (j < no_terms_collect && oc > col[j].oc)
+            j++;
+        if (j)
+        {
+            --j;
+            memmove(col, col+1, sizeof(*col) * j);
+            col[j].term = term;
+            col[j].oc = oc;
+        }
+    }
+    zebra_strmap_it_destroy(it);
+    return col;
+}
+
+static ZEBRA_RES facet_fetch(ZebraHandle zh, const char *setname,
+                             ODR odr,
+                             const char *elemsetname,
+                             const Odr_oid *input_format,
+                             const Odr_oid **output_format,
+                             char **rec_bufp, int *rec_lenp)
+{
+    zint *pos_array;
+    int i;
+    int num_recs = 10; /* number of records to analyze */
+    int no_collect_terms = 20; /* number of term candidates */
+    ZebraMetaRecord *poset;
+    ZEBRA_RES ret = ZEBRA_OK;
+    int *ord_array;
+    WRBUF wr = wrbuf_alloc();
+    
+    int no_ord = 0;
+    struct index_spec *spec, *spec_list;
+    int error;
+
+
+    spec_list = parse_index_spec(elemsetname, odr_getmem(odr), &error);
+              
+    if (!spec_list || error)
+        return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
+            
+    for (spec = spec_list; spec; spec = spec->next)
+    {
+        if (!spec->index_type)
+            return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
+        no_ord++;
+    }
+
+    ord_array = odr_malloc(odr, sizeof(*ord_array) * no_ord);
+
+    for (spec = spec_list, i = 0; spec; spec = spec->next, i++)
+    {
+        int ord = zebraExplain_lookup_attr_str(zh->reg->zei,
+                                               zinfo_index_category_index,
+                                               spec->index_type,
+                                               spec->index_name);
+        if (ord == -1)
+        {
+            return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
+        }
+        ord_array[i] = ord;
+    }
+
+    pos_array = (zint *) xmalloc(num_recs * sizeof(*pos_array));
+    for (i = 0; i < num_recs; i++)
+       pos_array[i] = i+1;
+    poset = zebra_meta_records_create(zh, setname, num_recs, pos_array);
+    if (!poset)
+    {
+       zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
+                      setname);
+        xfree(pos_array);
+       ret = ZEBRA_FAIL;
+    }
+    else
+    {
+        zebra_strmap_t *map_array
+            = odr_malloc(odr, sizeof *map_array * no_ord);
+        for (i = 0; i < no_ord; i++)
+            map_array[i] = zebra_strmap_create();
+
+        for (i = 0; i < num_recs; i++)
+        {
+            int j;
+            zint sysnos[MAX_SYSNOS_PER_RECORD];
+            int no_sysnos = MAX_SYSNOS_PER_RECORD;
+            if (!poset[i].sysno)
+                continue;
+            ret = zebra_result_recid_to_sysno(zh,  setname,
+                                              poset[i].sysno,
+                                              sysnos, &no_sysnos);
+            assert(no_sysnos > 0);
+            for (j = 0; j < no_sysnos; j++)
+            {
+                size_t slen;
+                const char *str;
+                struct it_key key_in;
+                Record rec = rec_get(zh->reg->records, sysnos[j]);
+                zebra_rec_keys_t keys = zebra_rec_keys_open();
+                zebra_rec_keys_set_buf(keys, rec->info[recInfo_delKeys],
+                                       rec->size[recInfo_delKeys], 0);
+                
+                if (zebra_rec_keys_rewind(keys))
+                {
+                    while (zebra_rec_keys_read(keys, &str, &slen, &key_in))
+                    {
+                        int i;
+                        struct index_spec *spec;
+                        for (spec = spec_list, i = 0; i < no_ord; 
+                             i++, spec = spec->next)
+                        {
+                            int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
+                            if (ord == ord_array[i])
+                            {
+                                int *freq;
+                                zebra_strmap_t sm = map_array[i];
+                                
+                                freq = zebra_strmap_lookup(sm, str, 0, 0);
+                                if (freq)
+                                    (*freq)++;
+                                else
+                                {
+                                    int v = 1;
+                                    zebra_strmap_add(sm, str, &v, sizeof v);
+                                }
+                            }
+                        }
+                    }
+                }
+                zebra_rec_keys_close(keys);
+                rec_free(&rec);
+            }
+        }
+        wrbuf_puts(wr, "<facets>\n");
+        for (spec = spec_list, i = 0; i < no_ord; i++, spec = spec->next)
+        {
+            int j;
+            NMEM nmem = nmem_create();
+            struct term_collect *col = term_collect_create(map_array[i], 
+                                                           no_collect_terms,
+                                                           nmem);
+            term_collect_freq(zh, col, no_collect_terms, ord_array[i],
+                              resultSetRef(zh, setname));
+            
+            wrbuf_printf(wr, "  <facet type=\"%s\" index=\"%s\">\n",
+                         spec->index_type, spec->index_name);
+            for (j = 0; j < no_collect_terms; j++)
+            {
+                if (col[j].term)
+                {
+                    char dst_buf[IT_MAX_WORD];
+                    zebra_term_untrans(zh, spec->index_type, dst_buf, col[j].term);
+                    wrbuf_printf(wr, "    <term coccur=\"%d\"", col[j].oc);
+                    if (col[j].set_occur)
+                        wrbuf_printf(wr, " occur=\"" ZINT_FORMAT "\"", 
+                                     col[j].set_occur);
+                    wrbuf_printf(wr, ">");
+                    wrbuf_xmlputs(wr, dst_buf);
+                    wrbuf_printf(wr, "</term>\n");
+                }
+            }
+            wrbuf_puts(wr, "  </facet>\n");
+            nmem_destroy(nmem);
+        }
+        wrbuf_puts(wr, "</facets>\n");
+        for (i = 0; i < no_ord; i++)
+            zebra_strmap_destroy(map_array[i]);
+    }
+    
+
+    *rec_bufp = odr_strdup(odr, wrbuf_cstr(wr));
+    wrbuf_destroy(wr);
+    *rec_lenp = strlen(*rec_bufp);
+    *output_format = yaz_oid_recsyn_xml;
+
+    xfree(pos_array);
+    zebra_meta_records_destroy(zh, poset, num_recs);
+    return ret;
+}
+
+int zebra_special_fetch(ZebraHandle zh, const char *setname,
+                        zint sysno, int score, ODR odr,
                         const char *elemsetname,
-                        const int *input_format,
-                        const int **output_format,
+                        const Odr_oid *input_format,
+                        const Odr_oid **output_format,
                         char **rec_bufp, int *rec_lenp)
 {
     Record rec;
@@ -410,7 +857,21 @@ int zebra_special_fetch(ZebraHandle zh, zint sysno, int score, ODR odr,
     /* set output variables before processing possible error states */
     /* *rec_lenp = 0; */
 
+    if (elemsetname && 0 == strncmp(elemsetname, "facet", 5))
+    {
+        return facet_fetch(zh, setname, odr,
+                           elemsetname + 5,
+                           input_format, output_format,
+                           rec_bufp, rec_lenp);
+    }
 
+    if (elemsetname && 0 == strcmp(elemsetname, "snippet"))
+    {
+        return snippet_fetch(zh, setname, sysno, odr,
+                             elemsetname + 7,
+                             input_format, output_format,
+                             rec_bufp, rec_lenp);
+    }
 
     /* processing zebra::meta::sysno elemset without fetching binary data */
     if (elemsetname && 0 == strcmp(elemsetname, "meta::sysno"))
@@ -558,10 +1019,11 @@ int zebra_special_fetch(ZebraHandle zh, zint sysno, int score, ODR odr,
 }
 
                           
-int zebra_record_fetch(ZebraHandle zh, zint sysno, int score,
-                       zebra_snippets *hit_snippet, ODR odr,
-                       const int *input_format, Z_RecordComposition *comp,
-                       const int **output_format,
+int zebra_record_fetch(ZebraHandle zh, const char *setname,
+                       zint sysno, int score,
+                       ODR odr,
+                       const Odr_oid *input_format, Z_RecordComposition *comp,
+                       const Odr_oid **output_format,
                        char **rec_bufp, int *rec_lenp, char **basenamep,
                        char **addinfo)
 {
@@ -572,14 +1034,22 @@ int zebra_record_fetch(ZebraHandle zh, zint sysno, int score,
     RecordAttr *recordAttr;
     void *clientData;
     int return_code = 0;
+    zint sysnos[MAX_SYSNOS_PER_RECORD];
+    int no_sysnos = MAX_SYSNOS_PER_RECORD;
+    ZEBRA_RES res;
+
+    res = zebra_result_recid_to_sysno(zh, setname, sysno, sysnos, &no_sysnos);
+    if (res != ZEBRA_OK)
+        return ZEBRA_FAIL;
 
+    sysno = sysnos[0];
     *basenamep = 0;
     *addinfo = 0;
     elemsetname = yaz_get_esn(comp);
 
     /* processing zebra special elementset names of form 'zebra:: */
     if (elemsetname && 0 == strncmp(elemsetname, "zebra::", 7))
-        return  zebra_special_fetch(zh, sysno, score, odr,
+        return  zebra_special_fetch(zh, setname, sysno, score, odr,
                                     elemsetname + 7,
                                     input_format, output_format,
                                     rec_bufp, rec_lenp);
@@ -600,8 +1070,8 @@ int zebra_record_fetch(ZebraHandle zh, zint sysno, int score,
     file_type = rec->info[recInfo_fileType];
     fname = rec->info[recInfo_filename];
     basename = rec->info[recInfo_databaseName];
-    *basenamep = (char *) odr_malloc (odr, strlen(basename)+1);
-    strcpy (*basenamep, basename);
+    *basenamep = (char *) odr_malloc(odr, strlen(basename)+1);
+    strcpy(*basenamep, basename);
 
     yaz_log(YLOG_DEBUG, "retrieve localno=" ZINT_FORMAT " score=%d",
             sysno, score);
@@ -610,8 +1080,6 @@ int zebra_record_fetch(ZebraHandle zh, zint sysno, int score,
 
     if (rec)
     {
-       zebra_snippets *snippet;
-       zebra_rec_keys_t reckeys = zebra_rec_keys_open();
         RecType rt;
         struct recRetrieveCtrl retrieveCtrl;
 
@@ -631,31 +1099,6 @@ int zebra_record_fetch(ZebraHandle zh, zint sysno, int score,
         retrieveCtrl.res = zh->res;
         retrieveCtrl.rec_buf = 0;
         retrieveCtrl.rec_len = -1;
-        retrieveCtrl.hit_snippet = hit_snippet;
-        retrieveCtrl.doc_snippet = zebra_snippets_create();
-
-       zebra_rec_keys_set_buf(reckeys,
-                              rec->info[recInfo_delKeys],
-                              rec->size[recInfo_delKeys], 
-                              0);
-       zebra_rec_keys_to_snippets(zh, reckeys, retrieveCtrl.doc_snippet);
-       zebra_rec_keys_close(reckeys);
-
-#if 0
-       /* for debugging purposes */
-       yaz_log(YLOG_LOG, "DOC SNIPPET:");
-       zebra_snippets_log(retrieveCtrl.doc_snippet, YLOG_LOG);
-       yaz_log(YLOG_LOG, "HIT SNIPPET:");
-       zebra_snippets_log(retrieveCtrl.hit_snippet, YLOG_LOG);
-#endif
-       snippet = zebra_snippets_window(retrieveCtrl.doc_snippet,
-                                       retrieveCtrl.hit_snippet,
-                                       10);
-#if 0
-       /* for debugging purposes */
-       yaz_log(YLOG_LOG, "WINDOW SNIPPET:");
-       zebra_snippets_log(snippet, YLOG_LOG);
-#endif
 
         if (!(rt = recType_byName(zh->reg->recTypes, zh->res,
                                   file_type, &clientData)))
@@ -679,9 +1122,6 @@ int zebra_record_fetch(ZebraHandle zh, zint sysno, int score,
             *addinfo = retrieveCtrl.addinfo;
         }
 
-       zebra_snippets_destroy(snippet);
-        zebra_snippets_destroy(retrieveCtrl.doc_snippet);
-
         stream.destroy(&stream);
         rec_free(&rec);
     }