X-Path attribute expressions with spaces in them is now handled.
[idzebra-moved-to-github.git] / index / zrpn.c
index 2a14aee..f09278d 100644 (file)
@@ -1,10 +1,26 @@
-/*
- * Copyright (C) 1995-2002, Index Data
- * All rights reserved.
- * Sebastian Hammer, Adam Dickmeiss
- *
- * $Id: zrpn.c,v 1.112 2002-04-04 14:14:13 adam Exp $
- */
+/* $Id: zrpn.c,v 1.135 2004-01-15 13:31:31 adam Exp $
+   Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003,2004
+   Index Data Aps
+
+This file is part of the Zebra server.
+
+Zebra is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 2, or (at your option) any later
+version.
+
+Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with Zebra; see the file LICENSE.zebra.  If not, write to the
+Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
+02111-1307, USA.
+*/
+
+
 #include <stdio.h>
 #include <assert.h>
 #ifdef WIN32
 #include <ctype.h>
 
 #include "index.h"
+#include <zebra_xpath.h>
 
 #include <charmap.h>
 #include <rstemp.h>
 #include <rsnull.h>
 #include <rsbool.h>
+#include <rsbetween.h>
 
 struct rpn_char_map_info {
     ZebraMaps zm;
     int reg_type;
 };
 
+typedef struct {
+    int type;
+    int major;
+    int minor;
+    Z_AttributesPlusTerm *zapt;
+} AttrType;
+
+
 static const char **rpn_char_map_handler (void *vp, const char **from, int len)
 {
     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
-    return zebra_maps_input (p->zm, p->reg_type, from, len);
+    const char **out = zebra_maps_input (p->zm, p->reg_type, from, len);
+#if 0
+    if (out && *out)
+    {
+        const char *outp = *out;
+        yaz_log (LOG_LOG, "---");
+        while (*outp)
+        {
+            yaz_log (LOG_LOG, "%02X", *outp);
+            outp++;
+        }
+    }
+#endif
+    return out;
 }
 
 static void rpn_char_map_prepare (struct zebra_register *reg, int reg_type,
@@ -40,32 +79,17 @@ static void rpn_char_map_prepare (struct zebra_register *reg, int reg_type,
     dict_grep_cmap (reg->dict, map_info, rpn_char_map_handler);
 }
 
-typedef struct {
-    int type;
-    int major;
-    int minor;
-    Z_AttributesPlusTerm *zapt;
-} AttrType;
-
 static int attr_find_ex (AttrType *src, oid_value *attributeSetP,
                         const char **string_value)
 {
     int num_attributes;
 
-#ifdef ASN_COMPILED
     num_attributes = src->zapt->attributes->num_attributes;
-#else
-    num_attributes = src->zapt->num_attributes;
-#endif
     while (src->major < num_attributes)
     {
         Z_AttributeElement *element;
 
-#ifdef ASN_COMPILED
         element = src->zapt->attributes->attributes[src->major];
-#else
-        element = src->zapt->attributeList[src->major];
-#endif
         if (src->type == *element->attributeType)
         {
             switch (element->which) 
@@ -84,17 +108,17 @@ static int attr_find_ex (AttrType *src, oid_value *attributeSetP,
             case Z_AttributeValue_complex:
                 if (src->minor >= element->value.complex->num_list)
                    break;
+                if (element->attributeSet && attributeSetP)
+                {
+                    oident *attrset;
+                    
+                    attrset = oid_getentbyoid (element->attributeSet);
+                    *attributeSetP = attrset->value;
+                }
                 if (element->value.complex->list[src->minor]->which ==  
                     Z_StringOrNumeric_numeric)
                {
                    ++(src->minor);
-                   if (element->attributeSet && attributeSetP)
-                   {
-                       oident *attrset;
-                       
-                       attrset = oid_getentbyoid (element->attributeSet);
-                       *attributeSetP = attrset->value;
-                   }
                    return
                        *element->value.complex->list[src->minor-1]->u.numeric;
                }
@@ -150,17 +174,18 @@ struct grep_info {
 static void term_untrans  (ZebraHandle zh, int reg_type,
                           char *dst, const char *src)
 {
+    int len = 0;
     while (*src)
     {
         const char *cp = zebra_maps_output (zh->reg->zebra_maps,
                                            reg_type, &src);
-       if (!cp)
-           *dst++ = *src++;
+       if (!cp && len < IT_MAX_WORD-1)
+           dst[len++] = *src++;
        else
-           while (*cp)
-               *dst++ = *cp++;
+           while (*cp && len < IT_MAX_WORD-1)
+               dst[len++] = *cp++;
     }
-    *dst = '\0';
+    dst[len] = '\0';
 }
 
 static void add_isam_p (const char *name, const char *info,
@@ -203,7 +228,7 @@ static void add_isam_p (const char *name, const char *info,
     {
        const char *db;
        int set, use;
-       char term_tmp[512];
+       char term_tmp[IT_MAX_WORD];
        int su_code = 0;
        int len = key_SU_decode (&su_code, name);
        
@@ -249,6 +274,8 @@ static int term_pre (ZebraMaps zebra_maps, int reg_type, const char **src,
     return *s0;
 }
 
+#define REGEX_CHARS " []()|.*+?!"
+
 /* term_100: handle term, where trunc=none (no operators at all) */
 static int term_100 (ZebraMaps zebra_maps, int reg_type,
                     const char **src, char *dst, int space_split,
@@ -286,7 +313,7 @@ static int term_100 (ZebraMaps zebra_maps, int reg_type,
            {   /* reload last space */
                while (space_start < space_end)
                {
-                   if (!isalnum (*space_start) && *space_start != '-')
+                    if (strchr (REGEX_CHARS, *space_start))
                        dst[i++] = '\\';
                    dst_term[j++] = *space_start;
                    dst[i++] = *space_start++;
@@ -298,7 +325,7 @@ static int term_100 (ZebraMaps zebra_maps, int reg_type,
        /* add non-space char */
         while (s1 < s0)
         {
-            if (!isalnum (*s1) && *s1 != '-')
+            if (strchr(REGEX_CHARS, *s1))
                 dst[i++] = '\\';
            dst_term[j++] = *s1;
             dst[i++] = *s1++;
@@ -339,7 +366,7 @@ static int term_101 (ZebraMaps zebra_maps, int reg_type,
                 break;
             while (s1 < s0)
             {
-                if (!isalnum (*s1))
+                if (strchr(REGEX_CHARS, *s1))
                     dst[i++] = '\\';
                dst_term[j++] = *s1;
                 dst[i++] = *s1++;
@@ -388,7 +415,7 @@ static int term_103 (ZebraMaps zebra_maps, int reg_type, const char **src,
                 break;
             while (s1 < s0)
             {
-                if (!isalnum (*s1))
+                if (strchr(REGEX_CHARS, *s1))
                     dst[i++] = '\\';
                dst_term[j++] = *s1;
                 dst[i++] = *s1++;
@@ -420,18 +447,43 @@ static int term_104 (ZebraMaps zebra_maps, int reg_type,
     int i = 0;
     int j = 0;
 
-    if (!term_pre (zebra_maps, reg_type, src, "#!", "#!"))
+    if (!term_pre (zebra_maps, reg_type, src, "?*#", "?*#"))
         return 0;
     s0 = *src;
     while (*s0)
     {
-        if (*s0 == '#')
+        if (*s0 == '?')
+        {
+           dst_term[j++] = *s0++;
+            if (*s0 >= '0' && *s0 <= '9')
+            {
+                int limit = 0;
+                while (*s0 >= '0' && *s0 <= '9')
+                {
+                    limit = limit * 10 + (*s0 - '0');
+                    dst_term[j++] = *s0++;
+                }
+                if (limit > 20)
+                    limit = 20;
+                while (--limit >= 0)
+                {
+                    dst[i++] = '.';
+                    dst[i++] = '?';
+                }
+            }
+            else
+            {
+                dst[i++] = '.';
+                dst[i++] = '*';
+            }
+        }
+        else if (*s0 == '*')
         {
             dst[i++] = '.';
             dst[i++] = '*';
            dst_term[j++] = *s0++;
         }
-        else if (*s0 == '!')
+        else if (*s0 == '#')
        {
             dst[i++] = '.';
            dst_term[j++] = *s0++;
@@ -443,7 +495,7 @@ static int term_104 (ZebraMaps zebra_maps, int reg_type,
                 break;
             while (s1 < s0)
             {
-                if (!isalnum (*s1))
+                if (strchr(REGEX_CHARS, *s1))
                     dst[i++] = '\\';
                dst_term[j++] = *s1;
                 dst[i++] = *s1++;
@@ -489,7 +541,7 @@ static int term_105 (ZebraMaps zebra_maps, int reg_type,
                 break;
             while (s1 < s0)
             {
-                if (!isalnum (*s1))
+                if (strchr(REGEX_CHARS, *s1))
                     dst[i++] = '\\';
                dst_term[j++] = *s1;
                 dst[i++] = *s1++;
@@ -647,7 +699,7 @@ static int string_relation (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
     int relation_value;
     int i;
     char *term_tmp = term_dict + strlen(term_dict);
-    char term_component[256];
+    char term_component[2*IT_MAX_WORD+20];
 
     attr_init (&relation, zapt, 2);
     relation_value = attr_find (&relation, NULL);
@@ -681,6 +733,9 @@ static int string_relation (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
            *term_tmp++ = ']';
            *term_tmp++ = '.';
            *term_tmp++ = '*';
+
+            if ((term_tmp - term_dict) > IT_MAX_WORD)
+                break;
        }
        *term_tmp++ = ')';
        *term_tmp = '\0';
@@ -710,6 +765,9 @@ static int string_relation (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
            *term_tmp++ = '*';
 
            *term_tmp++ = '|';
+
+            if ((term_tmp - term_dict) > IT_MAX_WORD)
+                break;
        }
        for (i = 0; term_component[i]; )
            string_rel_add_char (&term_tmp, term_component, &i);
@@ -740,6 +798,9 @@ static int string_relation (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
            *term_tmp++ = '*';
 
            *term_tmp++ = '|';
+
+            if ((term_tmp - term_dict) > IT_MAX_WORD)
+                break;
        }
        for (i = 0; term_component[i];)
            string_rel_add_char (&term_tmp, term_component, &i);
@@ -779,6 +840,9 @@ static int string_relation (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
            *term_tmp++ = ']';
            *term_tmp++ = '.';
            *term_tmp++ = '*';
+
+            if ((term_tmp - term_dict) > IT_MAX_WORD)
+                break;
        }
        *term_tmp++ = ')';
        *term_tmp = '\0';
@@ -797,12 +861,44 @@ static int string_relation (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 }
 
 static int string_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
-                       const char **term_sub, 
-                       oid_value attributeSet, NMEM stream,
-                       struct grep_info *grep_info,
-                       int reg_type, int complete_flag,
-                       int num_bases, char **basenames,
-                       char *term_dst)
+                        const char **term_sub, 
+                        oid_value attributeSet, NMEM stream,
+                        struct grep_info *grep_info,
+                        int reg_type, int complete_flag,
+                        int num_bases, char **basenames,
+                        char *term_dst, int xpath_use);
+
+static RSET term_trunc (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
+                        const char **term_sub, 
+                        oid_value attributeSet, NMEM stream,
+                        struct grep_info *grep_info,
+                        int reg_type, int complete_flag,
+                        int num_bases, char **basenames,
+                        char *term_dst,
+                        const char *rank_type, int xpath_use)
+{
+    int r;
+    grep_info->isam_p_indx = 0;
+    r = string_term (zh, zapt, term_sub, attributeSet, stream, grep_info,
+                     reg_type, complete_flag, num_bases, basenames,
+                     term_dst, xpath_use);
+    if (r < 1)
+        return 0;
+    logf (LOG_DEBUG, "term: %s", term_dst);
+    return rset_trunc (zh, grep_info->isam_p_buf,
+                       grep_info->isam_p_indx, term_dst,
+                       strlen(term_dst), rank_type, 1 /* preserve pos */,
+                       zapt->term->which);
+}
+
+
+static int string_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
+                        const char **term_sub, 
+                        oid_value attributeSet, NMEM stream,
+                        struct grep_info *grep_info,
+                        int reg_type, int complete_flag,
+                        int num_bases, char **basenames,
+                        char *term_dst, int xpath_use)
 {
     char term_dict[2*IT_MAX_WORD+4000];
     int j, r, base_no;
@@ -810,93 +906,120 @@ static int string_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
     int truncation_value;
     AttrType use;
     int use_value;
+    const char *use_string = 0;
     oid_value curAttributeSet = attributeSet;
     const char *termp;
     struct rpn_char_map_info rcmi;
     int space_split = complete_flag ? 0 : 1;
 
+    int bases_ok = 0;     /* no of databases with OK attribute */
+    int errCode = 0;      /* err code (if any is not OK) */
+    char *errString = 0;  /* addinfo */
+
     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
     attr_init (&use, zapt, 1);
-    use_value = attr_find (&use, &curAttributeSet);
+    use_value = attr_find_ex (&use, &curAttributeSet, &use_string);
     logf (LOG_DEBUG, "string_term, use value %d", use_value);
     attr_init (&truncation, zapt, 5);
     truncation_value = attr_find (&truncation, NULL);
     logf (LOG_DEBUG, "truncation value %d", truncation_value);
 
-    if (use_value == -1)
+    if (use_value == -1)    /* no attribute - assumy "any" */
         use_value = 1016;
-
     for (base_no = 0; base_no < num_bases; base_no++)
     {
         attent attp;
+        data1_local_attribute id_xpath_attr;
         data1_local_attribute *local_attr;
         int max_pos, prefix_len = 0;
 
         termp = *term_sub;
-        if ((r=att_getentbyatt (zh, &attp, curAttributeSet, use_value)))
-        {
-            logf (LOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
-                  curAttributeSet, use_value, r);
-           if (r == -1)
-           {
-               char val_str[32];
-               sprintf (val_str, "%d", use_value);
-               zh->errCode = 114;
-               zh->errString = nmem_strdup (stream, val_str);
-           }
-           else
-           {
-               int oid[OID_SIZE];
-               struct oident oident;
 
-               oident.proto = PROTO_Z3950;
-               oident.oclass = CLASS_ATTSET;
-               oident.value = curAttributeSet;
-               oid_ent_to_oid (&oident, oid);
-
-               zh->errCode = 121;
-               zh->errString = nmem_strdup (stream, oident.desc);
-           }
-            return -1;
-        }
         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
         {
             zh->errCode = 109; /* Database unavailable */
             zh->errString = basenames[base_no];
             return -1;
         }
+        if (use_value == -2)  /* string attribute (assume IDXPATH/any) */
+        {
+            use_value = xpath_use;
+            attp.local_attributes = &id_xpath_attr;
+            attp.attset_ordinal = VAL_IDXPATH;
+            id_xpath_attr.next = 0;
+            id_xpath_attr.local = use_value;
+        }
+       else if (curAttributeSet == VAL_IDXPATH)
+        {
+            attp.local_attributes = &id_xpath_attr;
+            attp.attset_ordinal = VAL_IDXPATH;
+            id_xpath_attr.next = 0;
+            id_xpath_attr.local = use_value;
+        }
+        else
+        {
+            if ((r=att_getentbyatt (zh, &attp, curAttributeSet, use_value)))
+            {
+                logf (LOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
+                      curAttributeSet, use_value, r);
+                if (r == -1)
+                {
+                    /* set was found, but value wasn't defined */
+                    char val_str[32];
+                    sprintf (val_str, "%d", use_value);
+                    errCode = 114;
+                    errString = nmem_strdup (stream, val_str);
+                }
+                else
+                {
+                    int oid[OID_SIZE];
+                    struct oident oident;
+                    
+                    oident.proto = PROTO_Z3950;
+                    oident.oclass = CLASS_ATTSET;
+                    oident.value = curAttributeSet;
+                    oid_ent_to_oid (&oident, oid);
+                    
+                    errCode = 121;
+                    errString = nmem_strdup (stream, oident.desc);
+                }
+                continue;
+            }
+        }
         for (local_attr = attp.local_attributes; local_attr;
              local_attr = local_attr->next)
         {
             int ord;
-           char ord_buf[32];
-           int i, ord_len;
-
+            char ord_buf[32];
+            int i, ord_len;
+            
             ord = zebraExplain_lookupSU (zh->reg->zei, attp.attset_ordinal,
-                                          local_attr->local);
+                                         local_attr->local);
             if (ord < 0)
                 continue;
             if (prefix_len)
                 term_dict[prefix_len++] = '|';
             else
                 term_dict[prefix_len++] = '(';
-
-           ord_len = key_SU_encode (ord, ord_buf);
-           for (i = 0; i<ord_len; i++)
-           {
-               term_dict[prefix_len++] = 1;
-               term_dict[prefix_len++] = ord_buf[i];
-           }
+            
+            ord_len = key_SU_encode (ord, ord_buf);
+            for (i = 0; i<ord_len; i++)
+            {
+                term_dict[prefix_len++] = 1;
+                term_dict[prefix_len++] = ord_buf[i];
+            }
         }
         if (!prefix_len)
         {
-           char val_str[32];
-           sprintf (val_str, "%d", use_value);
-           zh->errCode = 114;
-           zh->errString = nmem_strdup (stream, val_str);
-            return -1;
+            char val_str[32];
+            sprintf (val_str, "%d", use_value);
+            errCode = 114;
+            errString = nmem_strdup (stream, val_str);
+           continue;
         }
-        term_dict[prefix_len++] = ')';        
+       bases_ok++; /* this has OK attributes */
+
+        term_dict[prefix_len++] = ')';
         term_dict[prefix_len++] = 1;
         term_dict[prefix_len++] = reg_type;
        logf (LOG_DEBUG, "reg_type = %d", term_dict[prefix_len-1]);
@@ -910,11 +1033,11 @@ static int string_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
                                  attributeSet,
                                  reg_type, space_split, term_dst))
                return 0;
-           logf (LOG_DEBUG, "dict_lookup_grep: %s", term_dict+prefix_len);
+           logf (LOG_LOG, "dict_lookup_grep: %s", term_dict+prefix_len);
            r = dict_lookup_grep (zh->reg->dict, term_dict, 0,
                                  grep_info, &max_pos, 0, grep_handle);
            if (r)
-               logf (LOG_WARN, "dict_lookup_grep fail, rel=gt: %d", r);
+               logf (LOG_WARN, "dict_lookup_grep fail %d", r);
            break;
        case 1:          /* right truncation */
            term_dict[j++] = '(';
@@ -1018,52 +1141,105 @@ static int string_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
            break;
         }
     }
+    if (!bases_ok)
+    {
+       zh->errCode = errCode;
+       zh->errString = errString;
+       return -1;
+    }
     *term_sub = termp;
     logf (LOG_DEBUG, "%d positions", grep_info->isam_p_indx);
     return 1;
 }
 
-static void trans_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
-                        char *termz)
+
+/* convert APT search term to UTF8 */
+static int zapt_term_to_utf8 (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
+                              char *termz)
 {
     size_t sizez;
     Z_Term *term = zapt->term;
 
-    sizez = term->u.general->len;
-    if (sizez > IT_MAX_WORD-1)
-        sizez = IT_MAX_WORD-1;
-    memcpy (termz, term->u.general->buf, sizez);
-    termz[sizez] = '\0';
+    switch (term->which)
+    {
+    case Z_Term_general:
+        if (zh->iconv_to_utf8 != 0)
+        {
+            char *inbuf = term->u.general->buf;
+            size_t inleft = term->u.general->len;
+            char *outbuf = termz;
+            size_t outleft = IT_MAX_WORD-1;
+            size_t ret;
+
+            ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
+                        &outbuf, &outleft);
+            if (ret == (size_t)(-1))
+            {
+                ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
+                zh->errCode = 125;
+                return -1;
+            }
+            *outbuf = 0;
+        }
+        else
+        {
+            sizez = term->u.general->len;
+            if (sizez > IT_MAX_WORD-1)
+                sizez = IT_MAX_WORD-1;
+            memcpy (termz, term->u.general->buf, sizez);
+            termz[sizez] = '\0';
+        }
+        break;
+    case Z_Term_characterString:
+        sizez = strlen(term->u.characterString);
+        if (sizez > IT_MAX_WORD-1)
+            sizez = IT_MAX_WORD-1;
+        memcpy (termz, term->u.characterString, sizez);
+        termz[sizez] = '\0';
+        break;
+    default:
+        zh->errCode = 124;
+        return -1;
+    }
+    return 0;
 }
 
-static void trans_scan_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
-                             char *termz, int reg_type)
+/* convert APT SCAN term to internal cmap */
+static int trans_scan_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
+                            char *termz, int reg_type)
 {
-    Z_Term *term = zapt->term;
-    const char **map;
-    const char *cp = (const char *) term->u.general->buf;
-    const char *cp_end = cp + term->u.general->len;
-    const char *src;
-    int i = 0;
-    const char *space_map = NULL;
-    int len;
-    
-    while ((len = (cp_end - cp)) > 0)
+    char termz0[IT_MAX_WORD];
+
+    if (zapt_term_to_utf8(zh, zapt, termz0))
+        return -1;    /* error */
+    else
     {
-        map = zebra_maps_input (zh->reg->zebra_maps, reg_type, &cp, len);
-        if (**map == *CHR_SPACE)
-            space_map = *map;
-        else
+        const char **map;
+        const char *cp = (const char *) termz0;
+        const char *cp_end = cp + strlen(cp);
+        const char *src;
+        int i = 0;
+        const char *space_map = NULL;
+        int len;
+            
+        while ((len = (cp_end - cp)) > 0)
         {
-            if (i && space_map)
-                for (src = space_map; *src; src++)
+            map = zebra_maps_input (zh->reg->zebra_maps, reg_type, &cp, len);
+            if (**map == *CHR_SPACE)
+                space_map = *map;
+            else
+            {
+                if (i && space_map)
+                    for (src = space_map; *src; src++)
+                        termz[i++] = *src;
+                space_map = NULL;
+                for (src = *map; *src; src++)
                     termz[i++] = *src;
-            space_map = NULL;
-            for (src = *map; *src; src++)
-                termz[i++] = *src;
+            }
         }
+        termz[i] = '\0';
     }
-    termz[i] = '\0';
+    return 0;
 }
 
 static RSET rpn_prox (ZebraHandle zh, RSET *rset, int rset_no,
@@ -1078,6 +1254,7 @@ static RSET rpn_prox (ZebraHandle zh, RSET *rset, int rset_no,
     int length_prox_term = 0;
     int min_nn = 10000000;
     int term_index;
+    int term_type = Z_Term_characterString;
     const char *flags = NULL;
     
     rsfd = (RSFD *) xmalloc (sizeof(*rsfd)*rset_no);
@@ -1103,6 +1280,12 @@ static RSET rpn_prox (ZebraHandle zh, RSET *rset, int rset_no,
            if (min_nn > rset[i]->rset_terms[j]->nn)
                min_nn = rset[i]->rset_terms[j]->nn;
            flags = nflags;
+            term_type = rset[i]->rset_terms[j]->type;
+
+            /* only if all term types are of type characterString .. */
+            /* the resulting term is of that type */
+            if (term_type != Z_Term_characterString)
+                term_type = Z_Term_general;
        }
     }
     for (i = 0; i<rset_no; i++)
@@ -1123,7 +1306,7 @@ static RSET rpn_prox (ZebraHandle zh, RSET *rset, int rset_no,
        rset_null_parms parms;
        
        parms.rset_term = rset_term_create (prox_term, length_prox_term,
-                                           flags);
+                                           flags, term_type);
        parms.rset_term->nn = 0;
        result = rset_create (rset_kind_null, &parms);
     }
@@ -1134,14 +1317,14 @@ static RSET rpn_prox (ZebraHandle zh, RSET *rset, int rset_no,
        RSFD rsfd_result;
 
        parms.rset_term = rset_term_create (prox_term, length_prox_term,
-                                           flags);
+                                           flags, term_type);
        parms.rset_term->nn = min_nn;
         parms.cmp = key_compare_it;
        parms.key_size = sizeof (struct it_key);
        parms.temp_path = res_get (zh->res, "setTmpDir");
        result = rset_create (rset_kind_temp, &parms);
        rsfd_result = rset_open (result, RSETF_WRITE);
-       
+
        while (*more)
        {
            for (i = 1; i<rset_no; i++)
@@ -1190,10 +1373,11 @@ static RSET rpn_prox (ZebraHandle zh, RSET *rset, int rset_no,
        rset_temp_parms parms;
        RSFD rsfd_result;
 
-       logf (LOG_LOG, "generic prox, dist = %d, relation = %d, ordered =%d, exclusion=%d",
-             distance, relation, ordered, exclusion);
+       yaz_log (LOG_LOG, "generic prox, dist=%d, relation=%d, ordered=%d"
+                         ", exclusion=%d",
+                         distance, relation, ordered, exclusion);
        parms.rset_term = rset_term_create (prox_term, length_prox_term,
-                                           flags);
+                                           flags, term_type);
        parms.rset_term->nn = min_nn;
         parms.cmp = key_compare_it;
        parms.key_size = sizeof (struct it_key);
@@ -1273,7 +1457,7 @@ static RSET rpn_prox (ZebraHandle zh, RSET *rset, int rset_no,
        rset_null_parms parms;
        
        parms.rset_term = rset_term_create (prox_term, length_prox_term,
-                                           flags);
+                                           flags, term_type);
        parms.rset_term->nn = 0;
        result = rset_create (rset_kind_null, &parms);
     }
@@ -1334,6 +1518,14 @@ char *normalize_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
     }
 }
 
+static void grep_info_delete (struct grep_info *grep_info)
+{
+#ifdef TERM_COUNT
+    xfree(grep_info->term_no);
+#endif
+    xfree (grep_info->isam_p_buf);
+}
+
 static int grep_info_prepare (ZebraHandle zh,
                              Z_AttributesPlusTerm *zapt,
                              struct grep_info *grep_info,
@@ -1353,6 +1545,8 @@ static int grep_info_prepare (ZebraHandle zh,
     grep_info->reg_type = reg_type;
     grep_info->termset = 0;
 
+    if (!zapt)
+        return 0;
     attr_init (&termset, zapt, 8);
     termset_value_numeric =
        attr_find_ex (&termset, NULL, &termset_value_string);
@@ -1387,12 +1581,12 @@ static RSET rpn_search_APT_phrase (ZebraHandle zh,
                                    oid_value attributeSet,
                                   NMEM stream,
                                   int reg_type, int complete_flag,
-                                  const char *rank_type,
+                                  const char *rank_type, int xpath_use,
                                   int num_bases, char **basenames)
 {
     char term_dst[IT_MAX_WORD+1];
     RSET rset[60], result;
-    int i, r, rset_no = 0;
+    int i, rset_no = 0;
     struct grep_info grep_info;
     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
     const char *termp = termz;
@@ -1403,29 +1597,24 @@ static RSET rpn_search_APT_phrase (ZebraHandle zh,
     while (1)
     { 
        logf (LOG_DEBUG, "APT_phrase termp=%s", termp);
-       grep_info.isam_p_indx = 0;
-        r = string_term (zh, zapt, &termp, attributeSet, stream, &grep_info,
-                       reg_type, complete_flag, num_bases, basenames,
-                       term_dst);
-        if (r < 1)
+        rset[rset_no] = term_trunc (zh, zapt, &termp, attributeSet,
+                                    stream, &grep_info,
+                                    reg_type, complete_flag,
+                                    num_bases, basenames,
+                                    term_dst, rank_type,
+                                    xpath_use);
+        if (!rset[rset_no])
             break;
-       logf (LOG_DEBUG, "term: %s", term_dst);
-        rset[rset_no] = rset_trunc (zh, grep_info.isam_p_buf,
-                                    grep_info.isam_p_indx, term_dst,
-                                   strlen(term_dst), rank_type);
-        assert (rset[rset_no]);
         if (++rset_no >= (int) (sizeof(rset)/sizeof(*rset)))
             break;
     }
-#ifdef TERM_COUNT
-    xfree(grep_info.term_no);
-#endif
-    xfree (grep_info.isam_p_buf);
+    grep_info_delete (&grep_info);
     if (rset_no == 0)
     {
        rset_null_parms parms;
        
-       parms.rset_term = rset_term_create (termz, -1, rank_type);
+       parms.rset_term = rset_term_create (termz, -1, rank_type,
+                                            zapt->term->which);
         return rset_create (rset_kind_null, &parms);
     }
     else if (rset_no == 1)
@@ -1443,11 +1632,12 @@ static RSET rpn_search_APT_or_list (ZebraHandle zh,
                                    NMEM stream,
                                    int reg_type, int complete_flag,
                                    const char *rank_type,
+                                    int xpath_use,
                                    int num_bases, char **basenames)
 {
     char term_dst[IT_MAX_WORD+1];
     RSET rset[60], result;
-    int i, r, rset_no = 0;
+    int i, rset_no = 0;
     struct grep_info grep_info;
     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
     const char *termp = termz;
@@ -1457,29 +1647,24 @@ static RSET rpn_search_APT_or_list (ZebraHandle zh,
     while (1)
     { 
        logf (LOG_DEBUG, "APT_or_list termp=%s", termp);
-       grep_info.isam_p_indx = 0;
-        r = string_term (zh, zapt, &termp, attributeSet, stream, &grep_info,
-                       reg_type, complete_flag, num_bases, basenames,
-                       term_dst);
-        if (r < 1)
+        rset[rset_no] = term_trunc (zh, zapt, &termp, attributeSet,
+                                    stream, &grep_info,
+                                    reg_type, complete_flag,
+                                    num_bases, basenames,
+                                    term_dst, rank_type,
+                                    xpath_use);
+        if (!rset[rset_no])
             break;
-       logf (LOG_DEBUG, "term: %s", term_dst);
-        rset[rset_no] = rset_trunc (zh, grep_info.isam_p_buf,
-                                    grep_info.isam_p_indx, term_dst,
-                                   strlen(term_dst), rank_type);
-        assert (rset[rset_no]);
         if (++rset_no >= (int) (sizeof(rset)/sizeof(*rset)))
             break;
     }
-#ifdef TERM_COUNT
-    xfree(grep_info.term_no);
-#endif
-    xfree (grep_info.isam_p_buf);
+    grep_info_delete (&grep_info);
     if (rset_no == 0)
     {
        rset_null_parms parms;
        
-       parms.rset_term = rset_term_create (termz, -1, rank_type);
+       parms.rset_term = rset_term_create (termz, -1, rank_type,
+                                            zapt->term->which);
         return rset_create (rset_kind_null, &parms);
     }
     result = rset[0];
@@ -1502,12 +1687,13 @@ static RSET rpn_search_APT_and_list (ZebraHandle zh,
                                      oid_value attributeSet,
                                     NMEM stream,
                                     int reg_type, int complete_flag,
-                                    const char *rank_type,
+                                    const char *rank_type, 
+                                     int xpath_use,
                                     int num_bases, char **basenames)
 {
     char term_dst[IT_MAX_WORD+1];
     RSET rset[60], result;
-    int i, r, rset_no = 0;
+    int i, rset_no = 0;
     struct grep_info grep_info;
     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
     const char *termp = termz;
@@ -1517,29 +1703,25 @@ static RSET rpn_search_APT_and_list (ZebraHandle zh,
     while (1)
     { 
        logf (LOG_DEBUG, "APT_and_list termp=%s", termp);
-       grep_info.isam_p_indx = 0;
-        r = string_term (zh, zapt, &termp, attributeSet, stream, &grep_info,
-                       reg_type, complete_flag, num_bases, basenames,
-                       term_dst);
-        if (r < 1)
+        rset[rset_no] = term_trunc (zh, zapt, &termp, attributeSet,
+                                    stream, &grep_info,
+                                    reg_type, complete_flag,
+                                    num_bases, basenames,
+                                    term_dst, rank_type,
+                                    xpath_use);
+        if (!rset[rset_no])
             break;
-       logf (LOG_DEBUG, "term: %s", term_dst);
-        rset[rset_no] = rset_trunc (zh, grep_info.isam_p_buf,
-                                    grep_info.isam_p_indx, term_dst,
-                                   strlen(term_dst), rank_type);
         assert (rset[rset_no]);
         if (++rset_no >= (int) (sizeof(rset)/sizeof(*rset)))
             break;
     }
-#ifdef TERM_COUNT
-    xfree(grep_info.term_no);
-#endif
-    xfree (grep_info.isam_p_buf);
+    grep_info_delete (&grep_info);
     if (rset_no == 0)
     {
        rset_null_parms parms;
        
-       parms.rset_term = rset_term_create (termz, -1, rank_type);
+       parms.rset_term = rset_term_create (termz, -1, rank_type,
+                                            zapt->term->which);
         return rset_create (rset_kind_null, &parms);
     }
     result = rset[0];
@@ -1617,20 +1799,24 @@ static int numeric_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
                         oid_value attributeSet, struct grep_info *grep_info,
                         int reg_type, int complete_flag,
                         int num_bases, char **basenames,
-                        char *term_dst)
+                        char *term_dst, int xpath_use, NMEM stream)
 {
     char term_dict[2*IT_MAX_WORD+2];
     int r, base_no;
     AttrType use;
     int use_value;
+    const char *use_string = 0;
     oid_value curAttributeSet = attributeSet;
     const char *termp;
     struct rpn_char_map_info rcmi;
 
+    int bases_ok = 0;     /* no of databases with OK attribute */
+    int errCode = 0;      /* err code (if any is not OK) */
+    char *errString = 0;  /* addinfo */
+
     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
     attr_init (&use, zapt, 1);
-    use_value = attr_find (&use, &curAttributeSet);
-    logf (LOG_DEBUG, "numeric_term, use value %d", use_value);
+    use_value = attr_find_ex (&use, &curAttributeSet, &use_string);
 
     if (use_value == -1)
         use_value = 1016;
@@ -1638,19 +1824,43 @@ static int numeric_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
     for (base_no = 0; base_no < num_bases; base_no++)
     {
         attent attp;
+        data1_local_attribute id_xpath_attr;
         data1_local_attribute *local_attr;
         int max_pos, prefix_len = 0;
 
         termp = *term_sub;
-        if ((r=att_getentbyatt (zh, &attp, curAttributeSet, use_value)))
+        if (use_value == -2)  /* string attribute (assume IDXPATH/any) */
         {
-            logf (LOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
-                  curAttributeSet, use_value, r);
-           if (r == -1)
-               zh->errCode = 114;
-           else
-               zh->errCode = 121;
-            return -1;
+            use_value = xpath_use;
+            attp.local_attributes = &id_xpath_attr;
+            attp.attset_ordinal = VAL_IDXPATH;
+            id_xpath_attr.next = 0;
+            id_xpath_attr.local = use_value;
+        }
+       else if (curAttributeSet == VAL_IDXPATH)
+        {
+            attp.local_attributes = &id_xpath_attr;
+            attp.attset_ordinal = VAL_IDXPATH;
+            id_xpath_attr.next = 0;
+            id_xpath_attr.local = use_value;
+        }
+        else
+        {
+            if ((r=att_getentbyatt (zh, &attp, curAttributeSet, use_value)))
+            {
+                logf (LOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
+                      curAttributeSet, use_value, r);
+                if (r == -1)
+               {
+                    char val_str[32];
+                    sprintf (val_str, "%d", use_value);
+                    errString = nmem_strdup (stream, val_str);
+                    errCode = 114;
+               }
+                else
+                    errCode = 121;
+                continue;
+            }
         }
         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
         {
@@ -1683,9 +1893,13 @@ static int numeric_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
         }
         if (!prefix_len)
         {
-            zh->errCode = 114;
-            return -1;
+            char val_str[32];
+            sprintf (val_str, "%d", use_value);
+            errCode = 114;
+            errString = nmem_strdup (stream, val_str);
+            continue;
         }
+       bases_ok++;
         term_dict[prefix_len++] = ')';        
         term_dict[prefix_len++] = 1;
         term_dict[prefix_len++] = reg_type;
@@ -1696,6 +1910,12 @@ static int numeric_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
                               term_dst))
            return 0;
     }
+    if (!bases_ok)
+    {
+       zh->errCode = errCode;
+       zh->errString = errString;
+       return -1;
+    }
     *term_sub = termp;
     logf (LOG_DEBUG, "%d positions", grep_info->isam_p_indx);
     return 1;
@@ -1707,7 +1927,7 @@ static RSET rpn_search_APT_numeric (ZebraHandle zh,
                                    oid_value attributeSet,
                                    NMEM stream,
                                    int reg_type, int complete_flag,
-                                   const char *rank_type,
+                                   const char *rank_type, int xpath_use,
                                    int num_bases, char **basenames)
 {
     char term_dst[IT_MAX_WORD+1];
@@ -1724,26 +1944,27 @@ static RSET rpn_search_APT_numeric (ZebraHandle zh,
        grep_info.isam_p_indx = 0;
         r = numeric_term (zh, zapt, &termp, attributeSet, &grep_info,
                          reg_type, complete_flag, num_bases, basenames,
-                         term_dst);
+                         term_dst, xpath_use,
+                         stream);
         if (r < 1)
             break;
        logf (LOG_DEBUG, "term: %s", term_dst);
         rset[rset_no] = rset_trunc (zh, grep_info.isam_p_buf,
                                     grep_info.isam_p_indx, term_dst,
-                                   strlen(term_dst), rank_type);
+                                   strlen(term_dst), rank_type,
+                                    0 /* preserve position */,
+                                    zapt->term->which);
         assert (rset[rset_no]);
         if (++rset_no >= (int) (sizeof(rset)/sizeof(*rset)))
             break;
     }
-#ifdef TERM_COUNT
-    xfree(grep_info.term_no);
-#endif
-    xfree (grep_info.isam_p_buf);
+    grep_info_delete (&grep_info);
     if (rset_no == 0)
     {
        rset_null_parms parms;
        
-       parms.rset_term = rset_term_create (term_dst, -1, rank_type);
+       parms.rset_term = rset_term_create (term_dst, -1, rank_type,
+                                            zapt->term->which);
         return rset_create (rset_kind_null, &parms);
     }
     result = rset[0];
@@ -1771,7 +1992,8 @@ static RSET rpn_search_APT_local (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
     struct it_key key;
     rset_temp_parms parms;
 
-    parms.rset_term = rset_term_create (termz, -1, rank_type);
+    parms.rset_term = rset_term_create (termz, -1, rank_type,
+                                        zapt->term->which);
     parms.cmp = key_compare_it;
     parms.key_size = sizeof (struct it_key);
     parms.temp_path = res_get (zh->res, "setTmpDir");
@@ -1875,19 +2097,272 @@ static RSET rpn_sort_spec (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
        nmem_malloc (stream, sizeof(*sks->caseSensitivity));
     *sks->caseSensitivity = 0;
 
-#ifdef ASN_COMPILED
     sks->which = Z_SortKeySpec_null;
     sks->u.null = odr_nullval ();
-#else
-    sks->missingValueAction = 0;
-#endif
-
     sort_sequence->specs[i] = sks;
 
-    parms.rset_term = rset_term_create (termz, -1, rank_type);
+    parms.rset_term = rset_term_create (termz, -1, rank_type,
+                                        zapt->term->which);
     return rset_create (rset_kind_null, &parms);
 }
 
+/* pop - moved to xpath.c */
+#if 0
+
+struct xpath_predicate {
+    int which;
+    union {
+#define XPATH_PREDICATE_RELATION 1
+        struct {
+            char *name;
+            char *op;
+            char *value;
+        } relation;
+#define XPATH_PREDICATE_BOOLEAN 2
+        struct {
+            const char *op;
+            struct xpath_predicate *left;
+            struct xpath_predicate *right;
+        } boolean;
+    } u;
+};
+
+struct xpath_location_step {
+    char *part;
+    struct xpath_predicate *predicate;
+};
+
+#endif
+
+static int parse_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
+                       oid_value attributeSet,
+                       struct xpath_location_step *xpath, int max, NMEM mem)
+{
+    oid_value curAttributeSet = attributeSet;
+    AttrType use;
+    const char *use_string = 0;
+    
+    attr_init (&use, zapt, 1);
+    attr_find_ex (&use, &curAttributeSet, &use_string);
+
+    if (!use_string || *use_string != '/')
+        return -1;
+
+    return zebra_parse_xpath_str(use_string, xpath, max, mem);
+}
+               
+
+static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
+                        int reg_type, const char *term, int use,
+                        oid_value curAttributeSet)
+{
+    RSET rset;
+    struct grep_info grep_info;
+    char term_dict[2048];
+    char ord_buf[32];
+    int prefix_len = 0;
+    int ord = zebraExplain_lookupSU (zh->reg->zei, curAttributeSet, use);
+    int ord_len, i, r, max_pos;
+    int term_type = Z_Term_characterString;
+    const char *flags = "void";
+
+    if (grep_info_prepare (zh, 0 /* zapt */, &grep_info, '0', stream))
+    {
+       rset_null_parms parms;
+       
+       parms.rset_term = rset_term_create (term, strlen(term),
+                                           flags, term_type);
+       parms.rset_term->nn = 0;
+       return rset_create (rset_kind_null, &parms);
+    }
+
+    if (ord < 0)
+    {
+       rset_null_parms parms;
+       
+       parms.rset_term = rset_term_create (term, strlen(term),
+                                           flags, term_type);
+       parms.rset_term->nn = 0;
+       return rset_create (rset_kind_null, &parms);
+    }
+    if (prefix_len)
+        term_dict[prefix_len++] = '|';
+    else
+        term_dict[prefix_len++] = '(';
+    
+    ord_len = key_SU_encode (ord, ord_buf);
+    for (i = 0; i<ord_len; i++)
+    {
+        term_dict[prefix_len++] = 1;
+        term_dict[prefix_len++] = ord_buf[i];
+    }
+    term_dict[prefix_len++] = ')';
+    term_dict[prefix_len++] = 1;
+    term_dict[prefix_len++] = reg_type;
+    
+    strcpy (term_dict+prefix_len, term);
+    
+    grep_info.isam_p_indx = 0;
+    r = dict_lookup_grep (zh->reg->dict, term_dict, 0,
+                          &grep_info, &max_pos, 0, grep_handle);
+    yaz_log (LOG_LOG, "%s %d positions", term,
+             grep_info.isam_p_indx);
+    rset = rset_trunc (zh, grep_info.isam_p_buf,
+                       grep_info.isam_p_indx, term, strlen(term),
+                       flags, 1, term_type);
+    grep_info_delete (&grep_info);
+    return rset;
+}
+
+static RSET rpn_search_xpath (ZebraHandle zh,
+                              oid_value attributeSet,
+                              int num_bases, char **basenames,
+                              NMEM stream, const char *rank_type, RSET rset,
+                              int xpath_len, struct xpath_location_step *xpath)
+{
+    oid_value curAttributeSet = attributeSet;
+    int base_no;
+    int i;
+
+    if (xpath_len < 0)
+        return rset;
+
+    yaz_log (LOG_LOG, "len=%d", xpath_len);
+    for (i = 0; i<xpath_len; i++)
+    {
+        yaz_log (LOG_LOG, "XPATH %d %s", i, xpath[i].part);
+
+    }
+
+    curAttributeSet = VAL_IDXPATH;
+
+    /*
+      //a    ->    a/.*
+      //a/b  ->    b/a/.*
+      /a     ->    a/
+      /a/b   ->    b/a/
+
+      /      ->    none
+
+   a[@attr=value]/b[@other=othervalue]
+
+ /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
+ /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
+ /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
+ /a/b[@c=y] val range(b/a/,freetext(w,1016,val),b/a/,@c=y)
+ /a[@c=y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c=y)
+ /a[@c=x]/b[@c=y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c=y),a/,@c=x)
+      
+    */
+
+    dict_grep_cmap (zh->reg->dict, 0, 0);
+
+    for (base_no = 0; base_no < num_bases; base_no++)
+    {
+        int level = xpath_len;
+        int first_path = 1;
+        
+        if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
+        {
+            zh->errCode = 109; /* Database unavailable */
+            zh->errString = basenames[base_no];
+            return rset;
+        }
+        while (--level >= 0)
+        {
+            char xpath_rev[128];
+            int i, len;
+            rset_between_parms parms;
+            RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
+
+            *xpath_rev = 0;
+            len = 0;
+            for (i = level; i >= 1; --i)
+            {
+                const char *cp = xpath[i].part;
+                if (*cp)
+                {
+                    for (;*cp; cp++)
+                        if (*cp == '*')
+                        {
+                            memcpy (xpath_rev + len, "[^/]*", 5);
+                            len += 5;
+                        }
+                        else if (*cp == ' ')
+                        {
+
+                            xpath_rev[len++] = 1;
+                            xpath_rev[len++] = ' ';
+                        }
+
+                        else
+                            xpath_rev[len++] = *cp;
+                    xpath_rev[len++] = '/';
+                }
+                else if (i == 1)  /* // case */
+                {
+                    xpath_rev[len++] = '.';
+                    xpath_rev[len++] = '*';
+                }
+            }
+            xpath_rev[len] = 0;
+
+            if (xpath[level].predicate &&
+                xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
+                xpath[level].predicate->u.relation.name[0])
+            {
+               WRBUF wbuf = wrbuf_alloc();
+               wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
+                if (xpath[level].predicate->u.relation.value)
+                {
+                   const char *cp = xpath[level].predicate->u.relation.value;
+                   wrbuf_putc(wbuf, '=');
+                   
+                   while (*cp)
+                   {
+                       if (strchr(REGEX_CHARS, *cp))
+                           wrbuf_putc(wbuf, '\\');
+                       wrbuf_putc(wbuf, *cp);
+                       cp++;
+                   }
+                }
+               wrbuf_puts(wbuf, "");
+                rset_attr = xpath_trunc (
+                    zh, stream, '0', wrbuf_buf(wbuf), 3, curAttributeSet);
+               wrbuf_free(wbuf, 1);
+            } 
+            else 
+            {
+                if (!first_path)
+                    continue;
+            }
+            yaz_log (LOG_LOG, "xpath_rev (%d) = %s", level, xpath_rev);
+           if (strlen(xpath_rev))
+           {
+                rset_start_tag = xpath_trunc(zh, stream, 
+                                         '0', xpath_rev, 1, curAttributeSet);
+            
+                rset_end_tag = xpath_trunc(zh, stream,
+                                       '0', xpath_rev, 2, curAttributeSet);
+
+                parms.key_size = sizeof(struct it_key);
+                parms.cmp = key_compare_it;
+                parms.rset_l = rset_start_tag;
+                parms.rset_m = rset;
+                parms.rset_r = rset_end_tag;
+                parms.rset_attr = rset_attr;
+                parms.printer = key_print_it;
+                rset = rset_create (rset_kind_between, &parms);
+            }
+            first_path = 0;
+        }
+    }
+
+    return rset;
+}
+
+
 
 static RSET rpn_search_APT (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
                             oid_value attributeSet, NMEM stream,
@@ -1900,6 +2375,10 @@ static RSET rpn_search_APT (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
     int complete_flag;
     int sort_flag;
     char termz[IT_MAX_WORD+1];
+    RSET rset = 0;
+    int xpath_len;
+    int xpath_use = 0;
+    struct xpath_location_step xpath[10];
 
     zebra_maps_attr (zh->reg->zebra_maps, zapt, &reg_id, &search_type,
                     rank_type, &complete_flag, &sort_flag);
@@ -1909,48 +2388,61 @@ static RSET rpn_search_APT (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
     logf (LOG_DEBUG, "search_type=%s", search_type);
     logf (LOG_DEBUG, "rank_type=%s", rank_type);
 
-    if (zapt->term->which != Z_Term_general)
-    {
-        zh->errCode = 124;
-        return NULL;
-    }
-    trans_term (zh, zapt, termz);
+    if (zapt_term_to_utf8(zh, zapt, termz))
+        return 0;
 
     if (sort_flag)
        return rpn_sort_spec (zh, zapt, attributeSet, stream, sort_sequence,
                              rank_type);
+    xpath_len = parse_xpath(zh, zapt, attributeSet, xpath, 10, stream);
+    if (xpath_len >= 0)
+    {
+        xpath_use = 1016;
+        if (xpath[xpath_len-1].part[0] == '@')
+            xpath_use = 1015;
+    }
 
     if (!strcmp (search_type, "phrase"))
     {
-       return rpn_search_APT_phrase (zh, zapt, termz, attributeSet, stream,
+       rset = rpn_search_APT_phrase (zh, zapt, termz, attributeSet, stream,
                                      reg_id, complete_flag, rank_type,
+                                      xpath_use,
                                      num_bases, basenames);
     }
     else if (!strcmp (search_type, "and-list"))
     {
-       return rpn_search_APT_and_list (zh, zapt, termz, attributeSet, stream,
+       rset = rpn_search_APT_and_list (zh, zapt, termz, attributeSet, stream,
                                        reg_id, complete_flag, rank_type,
+                                        xpath_use,
                                        num_bases, basenames);
     }
     else if (!strcmp (search_type, "or-list"))
     {
-       return rpn_search_APT_or_list (zh, zapt, termz, attributeSet, stream,
+       rset = rpn_search_APT_or_list (zh, zapt, termz, attributeSet, stream,
                                       reg_id, complete_flag, rank_type,
+                                       xpath_use,
                                       num_bases, basenames);
     }
     else if (!strcmp (search_type, "local"))
     {
-        return rpn_search_APT_local (zh, zapt, termz, attributeSet, stream,
+        rset = rpn_search_APT_local (zh, zapt, termz, attributeSet, stream,
                                     rank_type);
     }
     else if (!strcmp (search_type, "numeric"))
     {
-       return rpn_search_APT_numeric (zh, zapt, termz, attributeSet, stream,
+       rset = rpn_search_APT_numeric (zh, zapt, termz, attributeSet, stream,
                                       reg_id, complete_flag, rank_type,
+                                       xpath_use,
                                       num_bases, basenames);
     }
-    zh->errCode = 118;
-    return NULL;
+    else if (!strcmp (search_type, "always"))
+    {
+        rset = 0;
+    }
+    else
+        zh->errCode = 118;
+    return rpn_search_xpath (zh, attributeSet, num_bases, basenames,
+                             stream, rank_type, rset, xpath_len, xpath);
 }
 
 static RSET rpn_search_structure (ZebraHandle zh, Z_RPNStructure *zs,
@@ -1994,21 +2486,11 @@ static RSET rpn_search_structure (ZebraHandle zh, Z_RPNStructure *zs,
             r = rset_create (rset_kind_not, &bool_parms);
             break;
         case Z_Operator_prox:
-#ifdef ASN_COMPILED
             if (zop->u.prox->which != Z_ProximityOperator_known)
             {
                 zh->errCode = 132;
                 return NULL;
             }
-#else
-            if (zop->u.prox->which != Z_ProxCode_known)
-            {
-                zh->errCode = 132;
-                return NULL;
-            }
-#endif
-
-#ifdef ASN_COMPILED
             if (*zop->u.prox->u.known != Z_ProxUnit_word)
             {
                 char *val = (char *) nmem_malloc (stream, 16);
@@ -2017,16 +2499,6 @@ static RSET rpn_search_structure (ZebraHandle zh, Z_RPNStructure *zs,
                 sprintf (val, "%d", *zop->u.prox->u.known);
                 return NULL;
             }
-#else
-            if (*zop->u.prox->proximityUnitCode != Z_ProxUnit_word)
-            {
-                char *val = (char *) nmem_malloc (stream, 16);
-                zh->errCode = 132;
-                zh->errString = val;
-                sprintf (val, "%d", *zop->u.prox->proximityUnitCode);
-                return NULL;
-            }
-#endif
            else
            {
                RSET rsets[2];
@@ -2070,6 +2542,8 @@ static RSET rpn_search_structure (ZebraHandle zh, Z_RPNStructure *zs,
                    nmem_strdup (stream, zs->u.simple->u.resultSetId);
                return 0;
            }
+            else
+                rset_dup(r);
         }
         else
         {
@@ -2172,12 +2646,33 @@ static int scan_handle (char *name, const char *info, int pos, void *client)
 static void scan_term_untrans (ZebraHandle zh, NMEM stream, int reg_type,
                               char **dst, const char *src)
 {
-    char term_dst[1024];
+    char term_src[IT_MAX_WORD];
+    char term_dst[IT_MAX_WORD];
     
-    term_untrans (zh, reg_type, term_dst, src);
-    
-    *dst = (char *) nmem_malloc (stream, strlen(term_dst)+1);
-    strcpy (*dst, term_dst);
+    term_untrans (zh, reg_type, term_src, src);
+
+    if (zh->iconv_from_utf8 != 0)
+    {
+        int len;
+        char *inbuf = term_src;
+        size_t inleft = strlen(term_src);
+        char *outbuf = term_dst;
+        size_t outleft = sizeof(term_dst)-1;
+        size_t ret;
+        
+        ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
+                         &outbuf, &outleft);
+        if (ret == (size_t)(-1))
+            len = 0;
+        else
+            len = outbuf - term_dst;
+        *dst = nmem_malloc (stream, len + 1);
+        if (len > 0)
+            memcpy (*dst, term_dst, len);
+        (*dst)[len] = '\0';
+    }
+    else
+        *dst = nmem_strdup (stream, term_src);
 }
 
 static void count_set (RSET r, int *count)
@@ -2209,7 +2704,7 @@ void rpn_scan (ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
               oid_value attributeset,
               int num_bases, char **basenames,
               int *position, int *num_entries, ZebraScanEntry **list,
-              int *is_partial)
+              int *is_partial, RSET limit_set, int return_zero)
 {
     int i;
     int pos = *position;
@@ -2225,16 +2720,47 @@ void rpn_scan (ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
     int ords[32], ord_no = 0;
     int ptr[32];
 
+    int bases_ok = 0;     /* no of databases with OK attribute */
+    int errCode = 0;      /* err code (if any is not OK) */
+    char *errString = 0;  /* addinfo */
+
     unsigned reg_id;
     char *search_type = NULL;
     char rank_type[128];
     int complete_flag;
     int sort_flag;
+
     *list = 0;
 
     if (attributeset == VAL_NONE)
         attributeset = VAL_BIB1;
 
+    if (!limit_set)
+    {
+        AttrType termset;
+        int termset_value_numeric;
+        const char *termset_value_string;
+        attr_init (&termset, zapt, 8);
+        termset_value_numeric =
+            attr_find_ex (&termset, NULL, &termset_value_string);
+        if (termset_value_numeric != -1)
+        {
+            char resname[32];
+            const char *termset_name = 0;
+            
+            if (termset_value_numeric != -2)
+            {
+                
+                sprintf (resname, "%d", termset_value_numeric);
+                termset_name = resname;
+            }
+            else
+                termset_name = termset_value_string;
+            
+            limit_set = resultSetRef (zh, termset_name);
+        }
+    }
+        
     yaz_log (LOG_DEBUG, "position = %d, num = %d set=%d",
              pos, num, attributeset);
         
@@ -2263,11 +2789,15 @@ void rpn_scan (ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
             logf (LOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
                   attributeset, use_value);
            if (r == -1)
-               zh->errCode = 114;
+           {
+                char val_str[32];
+                sprintf (val_str, "%d", use_value);
+                errCode = 114;
+                errString = odr_strdup (stream, val_str);
+           }   
            else
-               zh->errCode = 121;
-           *num_entries = 0;
-           return;
+               errCode = 121;
+           continue;
         }
         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
         {
@@ -2276,6 +2806,7 @@ void rpn_scan (ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
            *num_entries = 0;
            return;
         }
+       bases_ok++;
         for (local_attr = attp.local_attributes; local_attr && ord_no < 32;
              local_attr = local_attr->next)
         {
@@ -2287,10 +2818,20 @@ void rpn_scan (ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
                 ords[ord_no++] = ord;
         }
     }
+    if (!bases_ok && errCode)
+    {
+       zh->errCode = errCode;
+       zh->errString = errString;
+       *num_entries = 0;
+    }
     if (ord_no == 0)
     {
+       char val_str[32];
+       sprintf (val_str, "%d", use_value);
+       zh->errCode = 114;
+       zh->errString = odr_strdup (stream, val_str);
+
        *num_entries = 0;
-        zh->errCode = 113;
        return;
     }
     /* prepare dictionary scanning */
@@ -2321,7 +2862,8 @@ void rpn_scan (ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
         termz[prefix_len] = 0;
         strcpy (scan_info->prefix, termz);
 
-        trans_scan_term (zh, zapt, termz+prefix_len, reg_id);
+        if (trans_scan_term (zh, zapt, termz+prefix_len, reg_id))
+            return ;
                     
         dict_scan (zh->reg->dict, termz, &before_tmp, &after_tmp,
                   scan_info, scan_handle);
@@ -2357,7 +2899,7 @@ void rpn_scan (ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
                           &glist[i+before].term, mterm);
         rset = rset_trunc (zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
                           glist[i+before].term, strlen(glist[i+before].term),
-                          NULL);
+                          NULL, 0, zapt->term->which);
 
         ptr[j0]++;
         for (j = j0+1; j<ord_no; j++)
@@ -2372,7 +2914,8 @@ void rpn_scan (ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
                 rset2 =
                    rset_trunc (zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
                               glist[i+before].term,
-                              strlen(glist[i+before].term), NULL);
+                              strlen(glist[i+before].term), NULL, 0,
+                               zapt->term->which);
 
                 bool_parms.key_size = sizeof(struct it_key);
                 bool_parms.cmp = key_compare_it;
@@ -2384,6 +2927,17 @@ void rpn_scan (ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
                 ptr[j]++;
             }
         }
+        if (limit_set)
+        {
+            rset_bool_parms bool_parms;
+
+            bool_parms.key_size = sizeof(struct it_key);
+            bool_parms.cmp = key_compare_it;
+            bool_parms.rset_l = rset;
+            bool_parms.rset_r = rset_dup(limit_set);
+
+            rset = rset_create (rset_kind_and, &bool_parms);
+        }
         count_set (rset, &glist[i+before].occurrences);
         rset_delete (rset);
     }
@@ -2423,7 +2977,7 @@ void rpn_scan (ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
         rset = rset_trunc
                (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
                glist[before-1-i].term, strlen(glist[before-1-i].term),
-               NULL);
+               NULL, 0, zapt->term->which);
 
         ptr[j0]++;
 
@@ -2439,7 +2993,8 @@ void rpn_scan (ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
                 rset2 = rset_trunc (zh,
                          &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
                                    glist[before-1-i].term,
-                                   strlen(glist[before-1-i].term), NULL);
+                                   strlen(glist[before-1-i].term), NULL, 0,
+                                    zapt->term->which);
 
                 bool_parms.key_size = sizeof(struct it_key);
                 bool_parms.cmp = key_compare_it;
@@ -2451,6 +3006,17 @@ void rpn_scan (ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
                 ptr[j]++;
             }
         }
+        if (limit_set)
+        {
+            rset_bool_parms bool_parms;
+
+            bool_parms.key_size = sizeof(struct it_key);
+            bool_parms.cmp = key_compare_it;
+            bool_parms.rset_l = rset;
+            bool_parms.rset_r = rset_dup(limit_set);
+
+            rset = rset_create (rset_kind_and, &bool_parms);
+        }
         count_set (rset, &glist[before-1-i].occurrences);
         rset_delete (rset);
     }