Generic snippet support. Unlike previous versions of snippet
[idzebra-moved-to-github.git] / index / retrieve.c
index 04f0b3a..d3ca5ad 100644 (file)
@@ -1,5 +1,5 @@
-/* $Id: retrieve.c,v 1.50 2006-11-15 14:09:43 marc Exp $
-   Copyright (C) 1995-2006
+/* $Id: retrieve.c,v 1.71 2007-08-21 11:06:47 adam Exp $
+   Copyright (C) 1995-2007
    Index Data ApS
 
 This file is part of the Zebra server.
@@ -34,22 +34,23 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 
 #include "index.h"
 #include <yaz/diagbib1.h>
+#include <yaz/snprintf.h>
 #include <direntz.h>
+#include <yaz/oid_db.h>
 
-
-
+#define ZEBRA_XML_HEADER_STR "<record xmlns=\"http://www.indexdata.com/zebra/\""
 
 static int zebra_create_record_stream(ZebraHandle zh, 
-                               Record *rec,
-                               struct ZebraRecStream *stream){
-
+                                      Record *rec,
+                                      struct ZebraRecStream *stream)
+{
     RecordAttr *recordAttr = rec_init_attr(zh->reg->zei, *rec);
 
-    if ((*rec)->size[recInfo_storeData] > 0){ 
+    if ((*rec)->size[recInfo_storeData] > 0)
         zebra_create_stream_mem(stream, (*rec)->info[recInfo_storeData],
                                 (*rec)->size[recInfo_storeData]);
-    }
-    else {
+    else
+    {
         char full_rep[1024];
         int fd;
             
@@ -65,7 +66,7 @@ static int zebra_create_record_stream(ZebraHandle zh,
             yaz_log (YLOG_WARN|YLOG_ERRNO, "Retrieve fail; missing file: %s",
                      full_rep);
             rec_free(rec);
-            return 14;
+            return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
         }
         zebra_create_stream_fd(stream, fd, recordAttr->recordOffset);
     }
@@ -74,31 +75,40 @@ static int zebra_create_record_stream(ZebraHandle zh,
     
 
 
-static void parse_zebra_elem(const char *elem,
-                             const char **index, size_t *index_len,
-                             const char **type, size_t *type_len)
+static int parse_zebra_elem(const char *elem,
+                            const char **index, size_t *index_len,
+                            const char **type, size_t *type_len)
 {
-    *type = 0;
-    *type_len = 0;
-
     *index = 0;
     *index_len = 0;
 
+    *type = 0;
+    *type_len = 0;
+
     if (elem && *elem)
     {
-        const char *cp = strchr(elem, ':');
+        char *cp;
+        /* verify that '::' is in the beginning of *elem 
+           and something more follows */
+        if (':' != *elem
+            || !(elem +1) || ':' != *(elem +1)
+            || !(elem +2) || '\0' == *(elem +2))
+            return 0;
+        /* pick out info from string after '::' */
+        elem = elem + 2;
+        cp = strchr(elem, ':');
 
-        if (!cp) /* no colon */
+        if (!cp) /* index, no colon, no type */
         {
             *index = elem;
             *index_len = strlen(elem);
         }
-        else if (cp[1] == '\0') /* 'index:' */
+        else if (cp[1] == '\0') /* colon, but no following type */
         {
-            *index = elem;
-            *index_len = cp - elem;
+            return 0;
         }
-        else
+        else  /* index, colon and type */
         {
             *index = elem;
             *index_len = cp - elem;
@@ -106,65 +116,188 @@ static void parse_zebra_elem(const char *elem,
             *type_len = strlen(cp+1);
         }
     }
+    return 1;
 }
 
 
-int zebra_special_index_fetch(ZebraHandle zh, SYSNO sysno, ODR odr,
-                          Record rec,
-                          const char *elemsetname,
-                          oid_value input_format,
-                          oid_value *output_format,
-                          char **rec_bufp, int *rec_lenp)
+int zebra_special_sort_fetch(ZebraHandle zh, zint sysno, ODR odr,
+                             const char *elemsetname,
+                             const Odr_oid *input_format,
+                             const Odr_oid **output_format,
+                             char **rec_bufp, int *rec_lenp)
 {
     const char *retrieval_index;
     size_t retrieval_index_len; 
     const char *retrieval_type;
     size_t retrieval_type_len;
-   
-    /* int return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS; */
+    char retrieval_index_cstr[256];
+    int ord;
 
-    WRBUF wrbuf = wrbuf_alloc();
+    /* only accept XML and SUTRS requests */
+    if (oid_oidcmp(input_format, yaz_oid_recsyn_xml) 
+        && oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
+    {
+        yaz_log(YLOG_WARN, "unsupported format for element set zebra::%s", 
+                elemsetname);
+        *output_format = 0;
+        return YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
+    }
+    
+    if (!parse_zebra_elem(elemsetname,
+                          &retrieval_index, &retrieval_index_len,
+                          &retrieval_type,  &retrieval_type_len))
+    {
+        return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
+    }
+    
+    if (retrieval_type_len == 0)
+        return -1;   /* must have a register type specified */
+    if (!retrieval_index_len ||
+        retrieval_index_len >= sizeof(retrieval_index_cstr)-1)
+    {
+        return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
+    }
+        
+    memcpy(retrieval_index_cstr, retrieval_index, retrieval_index_len);
+    retrieval_index_cstr[retrieval_index_len] = '\0';
+
+    ord = zebraExplain_lookup_attr_str(zh->reg->zei,
+                                       zinfo_index_category_sort,
+                                       retrieval_type[0],
+                                       retrieval_index_cstr);
+    if (ord == -1)
+        return -1;  /* is not a sort index */
+    else
+    {
+        char dst_buf[IT_MAX_WORD];
+        char str[IT_MAX_WORD];
+        int index_type;
+        const char *db = 0;
+        const char *string_index = 0;
+        WRBUF wrbuf = wrbuf_alloc();
+        
+        zebra_sort_sysno(zh->reg->sort_index, sysno);
+        zebra_sort_type(zh->reg->sort_index, ord);
+        zebra_sort_read(zh->reg->sort_index, str);
+
+        zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db, &string_index);
+        
+        zebra_term_untrans(zh, index_type, dst_buf, str);
+        
+
+        if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
+        {
+            *output_format = yaz_oid_recsyn_xml;
+            wrbuf_printf(wrbuf, ZEBRA_XML_HEADER_STR
+                         " sysno=\"" ZINT_FORMAT "\""
+                         " set=\"zebra::index%s/\">\n",
+                         sysno, elemsetname);
+
+            wrbuf_printf(wrbuf, "  <index name=\"%s\"", 
+                         string_index);
+            wrbuf_printf(wrbuf, " type=\"%c\">", index_type);
+            wrbuf_xmlputs(wrbuf, dst_buf);
+            wrbuf_printf(wrbuf, "</index>\n");
+            wrbuf_printf(wrbuf, "</record>\n");
+        }
+        else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
+        {
+            *output_format = yaz_oid_recsyn_sutrs;
+            
+            wrbuf_printf(wrbuf, "%s %c %s\n", string_index, index_type,
+                         dst_buf);
+        }
+        *rec_lenp = wrbuf_len(wrbuf);
+        *rec_bufp = odr_malloc(odr, *rec_lenp);
+        memcpy(*rec_bufp, wrbuf_buf(wrbuf), *rec_lenp);
+        wrbuf_destroy(wrbuf);
+        return 0;
+    }
+}
+                            
+int zebra_special_index_fetch(ZebraHandle zh, zint sysno, ODR odr,
+                              Record rec,
+                              const char *elemsetname,
+                              const Odr_oid *input_format,
+                              const Odr_oid **output_format,
+                              char **rec_bufp, int *rec_lenp)
+{
+    const char *retrieval_index;
+    size_t retrieval_index_len; 
+    const char *retrieval_type;
+    size_t retrieval_type_len;
     zebra_rec_keys_t keys;
+    int ret_code = 0;
     
+    /* set output variables before processing possible error states */
+    /* *rec_lenp = 0; */
+
     /* only accept XML and SUTRS requests */
-    if (input_format != VAL_TEXT_XML
-        && input_format != VAL_SUTRS){
+    if (oid_oidcmp(input_format, yaz_oid_recsyn_xml)
+        && oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
+    {
         yaz_log(YLOG_WARN, "unsupported format for element set zebra::%s", 
                 elemsetname);
-        *output_format = VAL_NONE;
-        return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
+        *output_format = 0;
+        return YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
+    }
+
+    if (!parse_zebra_elem(elemsetname,
+                     &retrieval_index, &retrieval_index_len,
+                     &retrieval_type,  &retrieval_type_len))
+        return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
+
+    if (retrieval_type_len != 0 && retrieval_type_len != 1)
+    {
+        return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
+    }
+
+    if (retrieval_index_len)
+    {
+        char retrieval_index_cstr[256];
+
+        if (retrieval_index_len < sizeof(retrieval_index_cstr) -1)
+        {
+            memcpy(retrieval_index_cstr, retrieval_index, retrieval_index_len);
+            retrieval_index_cstr[retrieval_index_len] = '\0';
+            
+            if (zebraExplain_lookup_attr_str(zh->reg->zei,
+                                             zinfo_index_category_index,
+                                             (retrieval_type_len == 0 ? -1 : 
+                                              retrieval_type[0]),
+                                             retrieval_index_cstr) == -1)
+                return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
+        }
     }
 
     keys = zebra_rec_keys_open();
     zebra_rec_keys_set_buf(keys, rec->info[recInfo_delKeys],
                            rec->size[recInfo_delKeys], 0);
 
-    parse_zebra_elem(elemsetname,
-                     &retrieval_index, &retrieval_index_len,
-                     &retrieval_type,  &retrieval_type_len);
-
-    if (zebra_rec_keys_rewind(keys)){
+    if (!zebra_rec_keys_rewind(keys))
+    {
+        ret_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
+    }
+    else
+    {
         size_t slen;
         const char *str;
         struct it_key key_in;
-
-        if (input_format == VAL_TEXT_XML){
-            *output_format = VAL_TEXT_XML;
-            /*wrbuf_printf(wrbuf, 
-              "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");*/
-            
-            wrbuf_printf(wrbuf, 
-                         "<record xmlns="
-                         "\"http://www.indexdata.com/zebra/\""
+        WRBUF wrbuf = wrbuf_alloc();
+    
+        if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
+        {
+            *output_format = input_format;
+            wrbuf_printf(wrbuf, ZEBRA_XML_HEADER_STR
                          " sysno=\"" ZINT_FORMAT "\""
-                         " set=\"zebra::index::%s\">\n",
+                         " set=\"zebra::index%s/\">\n",
                          sysno, elemsetname);
         }
-        else if (input_format == VAL_SUTRS)
-            *output_format = VAL_SUTRS;
-
+        else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
+            *output_format = input_format;
 
-        while(zebra_rec_keys_read(keys, &str, &slen, &key_in)){
+        while (zebra_rec_keys_read(keys, &str, &slen, &key_in))
+        {
             int i;
             int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
             int index_type;
@@ -176,124 +309,295 @@ int zebra_special_index_fetch(ZebraHandle zh, SYSNO sysno, ODR odr,
             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db,
                                     &string_index);
             string_index_len = strlen(string_index);
+
+            /* process only if index is not defined, 
+               or if defined and matching */
             if (retrieval_index == 0 
                 || (string_index_len == retrieval_index_len 
                     && !memcmp(string_index, retrieval_index,
-                               string_index_len))){
-                
+                               string_index_len)))
+            {
+                /* process only if type is not defined, or is matching */
                 if (retrieval_type == 0 
                     || (retrieval_type_len == 1 
-                        && retrieval_type[0] == index_type)){
-                    
-                    if (input_format == VAL_TEXT_XML){
-                        wrbuf_printf(wrbuf, "  <index name=\"%s\"", 
-                                     string_index);
-                        
-                        wrbuf_printf(wrbuf, " type=\"%c\"", index_type);
+                        && retrieval_type[0] == index_type))
+                {
+                    zebra_term_untrans(zh, index_type, dst_buf, str);
+                    if (strlen(dst_buf))
+                    {
+                        if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
+                        {
+                            wrbuf_printf(wrbuf, "  <index name=\"%s\"", 
+                                         string_index);
+                            
+                            wrbuf_printf(wrbuf, " type=\"%c\"", index_type);
+                            
+                            wrbuf_printf(wrbuf, " seq=\"" ZINT_FORMAT "\">", 
+                                         key_in.mem[key_in.len -1]);
                         
-                        wrbuf_printf(wrbuf, " seq=\"" ZINT_FORMAT "\">", 
-                                     key_in.mem[key_in.len -1]);
+                            wrbuf_xmlputs(wrbuf, dst_buf);
+                            wrbuf_printf(wrbuf, "</index>\n");
+                        }
+                        else 
+                        {
+                            wrbuf_printf(wrbuf, "%s ", string_index);
+                            
+                            wrbuf_printf(wrbuf, "%c", index_type);
+                            
+                            for (i = 1; i < key_in.len; i++)
+                                wrbuf_printf(wrbuf, " " ZINT_FORMAT, 
+                                             key_in.mem[i]);
+                            
+                            /* zebra_term_untrans(zh, index_type, dst_buf, str); */
+                            wrbuf_printf(wrbuf, " %s", dst_buf);
                         
-                        zebra_term_untrans(zh, index_type, dst_buf, str);
-                        wrbuf_xmlputs(wrbuf, dst_buf);
-                        wrbuf_printf(wrbuf, "</index>\n");
+                            wrbuf_printf(wrbuf, "\n");
+                        }
                     }
-                    else if (input_format == VAL_SUTRS){
-                        wrbuf_printf(wrbuf, "%s ", string_index);
                     
-                        wrbuf_printf(wrbuf, "%c", index_type);
-                    
-                        for (i = 1; i < key_in.len; i++)
-                            wrbuf_printf(wrbuf, " " ZINT_FORMAT, 
-                                         key_in.mem[i]);
-
-                        zebra_term_untrans(zh, index_type, dst_buf, str);
-                        wrbuf_printf(wrbuf, " %s", dst_buf);
-                        
-                        wrbuf_printf(wrbuf, "\n");
-                    }
                 }
             }
         }
-        if (input_format == VAL_TEXT_XML)
+        if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
             wrbuf_printf(wrbuf, "</record>\n");
-     }
-    
-    *rec_lenp = wrbuf_len(wrbuf);
-    *rec_bufp = odr_malloc(odr, *rec_lenp);
-    memcpy(*rec_bufp, wrbuf_buf(wrbuf), *rec_lenp);
-    wrbuf_free(wrbuf, 1);
+        *rec_lenp = wrbuf_len(wrbuf);
+        *rec_bufp = odr_malloc(odr, *rec_lenp);
+        memcpy(*rec_bufp, wrbuf_buf(wrbuf), *rec_lenp);
+        wrbuf_destroy(wrbuf);
+    }
     zebra_rec_keys_close(keys);
-    return 0;
+    return ret_code;
 }
 
 
-int zebra_special_fetch(ZebraHandle zh, SYSNO sysno, ODR odr,
-                           const char *elemsetname,
-                           oid_value input_format,
-                           oid_value *output_format,
-                           char **rec_bufp, int *rec_lenp)
+static void retrieve_puts_attr(WRBUF wrbuf, const char *name,
+                               const char *value)
+{
+    if (value)
+    {
+        wrbuf_printf(wrbuf, " %s=\"", name);
+        wrbuf_xmlputs(wrbuf, value);
+        wrbuf_printf(wrbuf, "\"");
+    }
+}
+
+static void retrieve_puts_attr_int(WRBUF wrbuf, const char *name,
+                               const int value)
 {
+    wrbuf_printf(wrbuf, " %s=\"%i\"", name, value);
+}
+
+static void retrieve_puts_str(WRBUF wrbuf, const char *name,
+                               const char *value)
+{
+    if (value)
+        wrbuf_printf(wrbuf, "%s %s\n", name, value);
+}
+
+static void retrieve_puts_int(WRBUF wrbuf, const char *name,
+                               const int value)
+{
+    wrbuf_printf(wrbuf, "%s %i\n", name, value);
+}
+
+
+static void snippet_xml_record(ZebraHandle zh, WRBUF wrbuf, zebra_snippets *doc)
+{
+    const zebra_snippet_word *doc_w;
+    int mark_state = 0;
+
+    wrbuf_printf(wrbuf, "%s>\n", ZEBRA_XML_HEADER_STR);
+    for (doc_w = zebra_snippets_constlist(doc); doc_w; doc_w = doc_w->next)
+    {
+        if (doc_w->mark)
+        {
+            int index_type;
+            const char *db = 0;
+            const char *string_index = 0;
+
+            zebraExplain_lookup_ord(zh->reg->zei, doc_w->ord, 
+                                    &index_type, &db, &string_index);
+
+            if (mark_state == 0)
+            {
+                wrbuf_printf(wrbuf, "  <snippet name=\"%s\"",  string_index);
+                wrbuf_printf(wrbuf, " type=\"%c\">", index_type);
+            }
+            if (doc_w->match)
+                wrbuf_puts(wrbuf, "<s>");
+            /* not printing leading ws */
+            if (mark_state || !doc_w->ws || doc_w->match) 
+                wrbuf_xmlputs(wrbuf, doc_w->term);
+            if (doc_w->match)
+                wrbuf_puts(wrbuf, "</s>");
+        }
+        else if (mark_state == 1)
+        {
+            wrbuf_puts(wrbuf, "</snippet>\n");
+        }
+        mark_state = doc_w->mark;
+    }
+    if (mark_state == 1)
+    {
+        wrbuf_puts(wrbuf, "</snippet>\n");
+    }
+    wrbuf_printf(wrbuf, "</record>");
+}
+
+int zebra_special_snippet_fetch(ZebraHandle zh, const char *setname,
+                                zint sysno, ODR odr,
+                                const char *elemsetname,
+                                const Odr_oid *input_format,
+                                const Odr_oid **output_format,
+                                char **rec_bufp, int *rec_lenp)
+{
+    int return_code = 0;
     Record rec;
+    
+    rec = rec_get(zh->reg->records, sysno);
+    if (!rec)
+    {
+        yaz_log(YLOG_WARN, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
+        return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
+    }
+    else
+    {
+        const char *file_type = rec->info[recInfo_fileType];
+        void *recTypeClientData;
+        RecType rt = recType_byName(zh->reg->recTypes, zh->res,
+                                    file_type, &recTypeClientData);
+        zebra_snippets *hit_snippet = zebra_snippets_create();
+        WRBUF wrbuf = wrbuf_alloc();
+
+        zebra_snippets_hit_vector(zh, setname, sysno, hit_snippet);
+        
+        if (!rt)
+            return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
+        else
+        {
+            struct ZebraRecStream stream;
+            
+            return_code = zebra_create_record_stream(zh, &rec, &stream);
+            if (return_code == 0)
+            {
+                zebra_snippets *rec_snippet = zebra_snippets_create();
+                extract_snippet(zh, rec_snippet, &stream,
+                                rt, recTypeClientData);
 
-    /* only accept XML and SUTRS requests */
-    if (input_format != VAL_TEXT_XML
-        && input_format != VAL_SUTRS){
-        yaz_log(YLOG_WARN, "unsupported format for element set zebra::%s", 
-                elemsetname);
-        return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
+#if 0
+                /* for debugging purposes */
+                yaz_log(YLOG_LOG, "---------------------------");
+                yaz_log(YLOG_LOG, "REC SNIPPET:");
+                zebra_snippets_log(rec_snippet, YLOG_LOG, 1);
+                yaz_log(YLOG_LOG, "---------------------------");
+                yaz_log(YLOG_LOG, "HIT SNIPPET:");
+                zebra_snippets_log(hit_snippet, YLOG_LOG, 1);
+#endif
+
+                zebra_snippets_ring(rec_snippet, hit_snippet, 5, 5);
+
+#if 0
+                yaz_log(YLOG_LOG, "---------------------------");
+                yaz_log(YLOG_LOG, "RING SNIPPET:");
+                zebra_snippets_log(rec_snippet, YLOG_LOG, 1);
+#endif
+                
+                snippet_xml_record(zh, wrbuf, rec_snippet);
+
+                *output_format = yaz_oid_recsyn_xml;
+
+                
+                zebra_snippets_destroy(rec_snippet);
+            }
+            stream.destroy(&stream);
+        }
+        if (return_code == 0)
+        {
+            *rec_lenp = wrbuf_len(wrbuf);
+            *rec_bufp = odr_strdup(odr, wrbuf_cstr(wrbuf));
+        }
+        wrbuf_destroy(wrbuf);
+        rec_free(&rec);
+        zebra_snippets_destroy(hit_snippet);
+    }
+    return return_code;
+}
+
+int zebra_special_fetch(ZebraHandle zh, const char *setname,
+                        zint sysno, int score, ODR odr,
+                        const char *elemsetname,
+                        const Odr_oid *input_format,
+                        const Odr_oid **output_format,
+                        char **rec_bufp, int *rec_lenp)
+{
+    Record rec;
+    
+    /* set output variables before processing possible error states */
+    /* *rec_lenp = 0; */
+
+
+    if (elemsetname && 0 == strcmp(elemsetname, "snippet"))
+    {
+        return zebra_special_snippet_fetch(zh, setname, sysno, odr,
+                                           elemsetname + 7,
+                                           input_format, output_format,
+                                           rec_bufp, rec_lenp);
     }
 
     /* processing zebra::meta::sysno elemset without fetching binary data */
-    if (elemsetname  && 0 == strcmp(elemsetname, "meta::sysno"))
+    if (elemsetname && 0 == strcmp(elemsetname, "meta::sysno"))
     {
-       char rec_str[128];
-        if (input_format == VAL_SUTRS){
-            sprintf(rec_str, ZINT_FORMAT, sysno);
-            *output_format = VAL_SUTRS;
+        int ret = 0;
+        WRBUF wrbuf = wrbuf_alloc();
+        if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
+        {
+            wrbuf_printf(wrbuf, ZINT_FORMAT, sysno);
+            *output_format = input_format;
         } 
-        else if (input_format == VAL_TEXT_XML){
-            sprintf(rec_str, "<record xmlns="
-                             "\"http://www.indexdata.com/zebra/\""
-                             " sysno=\"" ZINT_FORMAT "\""
-                             " set=\"zebra::%s\">\n",
-                             sysno, elemsetname);
-            *output_format = VAL_TEXT_XML;
-        }
-       *rec_lenp = strlen(rec_str);
-        if (*rec_lenp){
-            *rec_bufp = odr_strdup(odr, rec_str);
-            return 0;
-        } else {
-            return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
+        else if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
+        {
+            wrbuf_printf(wrbuf, ZEBRA_XML_HEADER_STR
+                         " sysno=\"" ZINT_FORMAT "\"/>\n",
+                         sysno);
+            *output_format = input_format;
         }
+       *rec_lenp = wrbuf_len(wrbuf);
+        if (*rec_lenp)
+            *rec_bufp = odr_strdup(odr, wrbuf_cstr(wrbuf));
+        else
+            ret = YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
+        wrbuf_destroy(wrbuf);
+        return ret;
     }
 
+    /* processing special elementsetname zebra::index:: for sort elements */
+    if (elemsetname && 0 == strncmp(elemsetname, "index", 5))
+    {
+        int ret = zebra_special_sort_fetch(zh, sysno, odr,
+                                           elemsetname + 5,
+                                           input_format, output_format,
+                                           rec_bufp, rec_lenp);
+        if (ret != -1)
+            return ret;
+        /* not a sort index so we continue to get the full record */
+    }
+
+
     /* fetching binary record up for all other display elementsets */
     rec = rec_get(zh->reg->records, sysno);
-    if (!rec){
-        yaz_log(YLOG_DEBUG, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
+    if (!rec)
+    {
+        yaz_log(YLOG_WARN, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
         return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
     }
 
-    /* processing special elementsetnames zebra::index:: */
-    if (elemsetname && 0 == strncmp(elemsetname, "index::", 7)){
-        
-        int ret = zebra_special_index_fetch(zh, sysno, odr, rec,
-                                            elemsetname + 7,
-                                            input_format, output_format,
-                                            rec_bufp, rec_lenp);
-        
-        rec_free(&rec);
-        return ret;
-    }
-
     /* processing special elementsetnames zebra::data */    
-    if (elemsetname && 0 == strcmp(elemsetname, "data")){
+    if (elemsetname && 0 == strcmp(elemsetname, "data"))
+    {
         struct ZebraRecStream stream;
         RecordAttr *recordAttr = rec_init_attr(zh->reg->zei, rec); 
         zebra_create_record_stream(zh, &rec, &stream);
-        *output_format = VAL_SUTRS;
+        *output_format = input_format;
         *rec_lenp = recordAttr->recordSize;
         *rec_bufp = (char *) odr_malloc(odr, *rec_lenp);
         stream.readf(&stream, *rec_bufp, *rec_lenp);
@@ -302,16 +606,95 @@ int zebra_special_fetch(ZebraHandle zh, SYSNO sysno, ODR odr,
         return 0;
     }
 
+    /* only accept XML and SUTRS requests from now */
+    if (oid_oidcmp(input_format, yaz_oid_recsyn_xml)
+        && oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
+    {
+        yaz_log(YLOG_WARN, "unsupported format for element set zebra::%s", 
+                elemsetname);
+        return YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
+    }
+    
+
+    /* processing special elementsetnames zebra::meta:: */
+    if (elemsetname && 0 == strcmp(elemsetname, "meta"))
+    {
+        int ret = 0;
+        WRBUF wrbuf = wrbuf_alloc();
+        RecordAttr *recordAttr = rec_init_attr(zh->reg->zei, rec); 
+
+        if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
+        {
+            *output_format = input_format;
+            
+            wrbuf_printf(wrbuf, ZEBRA_XML_HEADER_STR
+                         " sysno=\"" ZINT_FORMAT "\"", sysno);
+            retrieve_puts_attr(wrbuf, "base", rec->info[recInfo_databaseName]);
+            retrieve_puts_attr(wrbuf, "file", rec->info[recInfo_filename]);
+            retrieve_puts_attr(wrbuf, "type", rec->info[recInfo_fileType]);
+            if (score >= 0)
+                retrieve_puts_attr_int(wrbuf, "score", score);
+           
+            wrbuf_printf(wrbuf,
+                         " rank=\"" ZINT_FORMAT "\""
+                         " size=\"%i\""
+                         " set=\"zebra::%s\"/>\n",
+                         recordAttr->staticrank,
+                         recordAttr->recordSize,
+                         elemsetname);
+        }
+        else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
+        {
+            *output_format = input_format;
+            wrbuf_printf(wrbuf, "sysno " ZINT_FORMAT "\n", sysno);
+            retrieve_puts_str(wrbuf, "base", rec->info[recInfo_databaseName]);
+            retrieve_puts_str(wrbuf, "file", rec->info[recInfo_filename]);
+            retrieve_puts_str(wrbuf, "type", rec->info[recInfo_fileType]);
+            if (score >= 0)
+                retrieve_puts_int(wrbuf, "score", score);
+
+            wrbuf_printf(wrbuf,
+                         "rank " ZINT_FORMAT "\n"
+                         "size %i\n"
+                         "set zebra::%s\n",
+                         recordAttr->staticrank,
+                         recordAttr->recordSize,
+                         elemsetname);
+        }
+       *rec_lenp = wrbuf_len(wrbuf);
+        if (*rec_lenp)
+            *rec_bufp = odr_strdup(odr, wrbuf_cstr(wrbuf));
+        else
+            ret = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
+
+        wrbuf_destroy(wrbuf);
+        rec_free(&rec);
+        return ret;
+    }
+
+    /* processing special elementsetnames zebra::index:: */
+    if (elemsetname && 0 == strncmp(elemsetname, "index", 5))
+    {
+        int ret = zebra_special_index_fetch(zh, sysno, odr, rec,
+                                            elemsetname + 5,
+                                            input_format, output_format,
+                                            rec_bufp, rec_lenp);
+        
+        rec_free(&rec);
+        return ret;
+    }
+
     if (rec)
         rec_free(&rec);
-    return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
+    return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
 }
 
                           
-int zebra_record_fetch(ZebraHandle zh, SYSNO sysno, int score,
+int zebra_record_fetch(ZebraHandle zh, const char *setname,
+                       zint sysno, int score,
                        zebra_snippets *hit_snippet, ODR odr,
-                       oid_value input_format, Z_RecordComposition *comp,
-                       oid_value *output_format,
+                       const Odr_oid *input_format, Z_RecordComposition *comp,
+                       const Odr_oid **output_format,
                        char **rec_bufp, int *rec_lenp, char **basenamep,
                        char **addinfo)
 {
@@ -327,21 +710,9 @@ int zebra_record_fetch(ZebraHandle zh, SYSNO sysno, int score,
     *addinfo = 0;
     elemsetname = yaz_get_esn(comp);
 
-    /*
-    yaz_log(YLOG_LOG, "ELEMENTSET: '%s'", elemsetname);
-    if (comp && comp->which)
-        {
-            yaz_log(YLOG_LOG, "%i %i", comp, comp->which);
-        }
-        If SRU, comp->which ==2 , and yaz_get_esn(comp) does not work correctly
-        IF Z3055, comp->which ==1 , and everything is fine - except reord 
-        encodning
-    */
-
     /* processing zebra special elementset names of form 'zebra:: */
-    /* SUGGESTION: do not check elemset nema here, buuuut ... */  
-    if (elemsetname  && 0 == strncmp(elemsetname, "zebra::", 7))
-        return  zebra_special_fetch(zh, sysno, odr,
+    if (elemsetname && 0 == strncmp(elemsetname, "zebra::", 7))
+        return  zebra_special_fetch(zh, setname, sysno, score, odr,
                                     elemsetname + 7,
                                     input_format, output_format,
                                     rec_bufp, rec_lenp);
@@ -351,7 +722,7 @@ int zebra_record_fetch(ZebraHandle zh, SYSNO sysno, int score,
     rec = rec_get(zh->reg->records, sysno);
     if (!rec)
     {
-        yaz_log(YLOG_DEBUG, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
+        yaz_log(YLOG_WARN, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
         *basenamep = 0;
         return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
     }
@@ -365,15 +736,13 @@ int zebra_record_fetch(ZebraHandle zh, SYSNO sysno, int score,
     *basenamep = (char *) odr_malloc (odr, strlen(basename)+1);
     strcpy (*basenamep, basename);
 
+    yaz_log(YLOG_DEBUG, "retrieve localno=" ZINT_FORMAT " score=%d",
+            sysno, score);
 
-    yaz_log (YLOG_DEBUG, "retrieve localno=" ZINT_FORMAT " score=%d",
-             sysno, score);
+    return_code = zebra_create_record_stream(zh, &rec, &stream);
 
-    zebra_create_record_stream(zh, &rec, &stream);
-    
+    if (rec)
     {
-       /* snippets code */
-       zebra_snippets *snippet;
        zebra_rec_keys_t reckeys = zebra_rec_keys_open();
         RecType rt;
         struct recRetrieveCtrl retrieveCtrl;
@@ -404,41 +773,19 @@ int zebra_record_fetch(ZebraHandle zh, SYSNO sysno, int score,
        zebra_rec_keys_to_snippets(zh, reckeys, retrieveCtrl.doc_snippet);
        zebra_rec_keys_close(reckeys);
 
-#if 0
-       /* for debugging purposes */
-       yaz_log(YLOG_LOG, "DOC SNIPPET:");
-       zebra_snippets_log(retrieveCtrl.doc_snippet, YLOG_LOG);
-       yaz_log(YLOG_LOG, "HIT SNIPPET:");
-       zebra_snippets_log(retrieveCtrl.hit_snippet, YLOG_LOG);
-#endif
-       snippet = zebra_snippets_window(retrieveCtrl.doc_snippet,
-                                       retrieveCtrl.hit_snippet,
-                                       10);
-#if 0
-       /* for debugging purposes */
-       yaz_log(YLOG_LOG, "WINDOW SNIPPET:");
-       zebra_snippets_log(snippet, YLOG_LOG);
-#endif
-
-        /* SUGGESTION: do not check elemset name here, buuuut ... 
-           add another recType Struct with zebra internal stuff here,
-           which overrides the Alvis/GRS-1/Safari filters .... 
-        */
         if (!(rt = recType_byName(zh->reg->recTypes, zh->res,
                                   file_type, &clientData)))
         {
+            char addinfo_str[100];
+
+            sprintf(addinfo_str, "Could not handle record type %.40s",
+                    file_type);
+                    
+            *addinfo = odr_strdup(odr, addinfo_str);
             return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
         }
         else
         {
-        /* SUGGESTION: do not check elemset name here, buuuut ... 
-           add another recType Struct with zebra internal stuff here,
-           which overrides the Alvis/GRS-1/Safari filters .... 
-           (*rt->retrieve) method to make the correct encoded, etc, retrieval,
-           where all needed info already is found in the   &retrieveCtr
-           parameter. This way, we do not need to re-code/dublicate a lot of 
-           logic. 
-        */
             (*rt->retrieve)(clientData, &retrieveCtrl);
             return_code = retrieveCtrl.diagnostic;
 
@@ -447,17 +794,12 @@ int zebra_record_fetch(ZebraHandle zh, SYSNO sysno, int score,
             *rec_lenp = retrieveCtrl.rec_len;
             *addinfo = retrieveCtrl.addinfo;
         }
-        /* another SUGGESTION: throw out all this snippet stuff in this 
-           file, and do it correctly - either inside filters, or in another 
-           new   (*rt->retrieve)(clientData, &retrieveCtrl); type 'snippet'
-        */
 
-       zebra_snippets_destroy(snippet);
         zebra_snippets_destroy(retrieveCtrl.doc_snippet);
-     }
 
-    stream.destroy(&stream);
-    rec_free(&rec);
+        stream.destroy(&stream);
+        rec_free(&rec);
+    }
 
     return return_code;
 }