Fix crash that occured for string-queries in database that didn't have them
[idzebra-moved-to-github.git] / index / zrpn.c
index 2a14aee..badb91e 100644 (file)
@@ -1,10 +1,26 @@
-/*
- * Copyright (C) 1995-2002, Index Data
- * All rights reserved.
- * Sebastian Hammer, Adam Dickmeiss
- *
- * $Id: zrpn.c,v 1.112 2002-04-04 14:14:13 adam Exp $
- */
+/* $Id: zrpn.c,v 1.121 2002-08-23 14:30:51 adam Exp $
+   Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002
+   Index Data Aps
+
+This file is part of the Zebra server.
+
+Zebra is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 2, or (at your option) any later
+version.
+
+Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with Zebra; see the file LICENSE.zebra.  If not, write to the
+Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
+02111-1307, USA.
+*/
+
+
 #include <stdio.h>
 #include <assert.h>
 #ifdef WIN32
@@ -20,6 +36,7 @@
 #include <rstemp.h>
 #include <rsnull.h>
 #include <rsbool.h>
+#include <rsbetween.h>
 
 struct rpn_char_map_info {
     ZebraMaps zm;
@@ -84,17 +101,17 @@ static int attr_find_ex (AttrType *src, oid_value *attributeSetP,
             case Z_AttributeValue_complex:
                 if (src->minor >= element->value.complex->num_list)
                    break;
+                if (element->attributeSet && attributeSetP)
+                {
+                    oident *attrset;
+                    
+                    attrset = oid_getentbyoid (element->attributeSet);
+                    *attributeSetP = attrset->value;
+                }
                 if (element->value.complex->list[src->minor]->which ==  
                     Z_StringOrNumeric_numeric)
                {
                    ++(src->minor);
-                   if (element->attributeSet && attributeSetP)
-                   {
-                       oident *attrset;
-                       
-                       attrset = oid_getentbyoid (element->attributeSet);
-                       *attributeSetP = attrset->value;
-                   }
                    return
                        *element->value.complex->list[src->minor-1]->u.numeric;
                }
@@ -249,6 +266,8 @@ static int term_pre (ZebraMaps zebra_maps, int reg_type, const char **src,
     return *s0;
 }
 
+#define REGEX_CHARS "[]()|.*+!"
+
 /* term_100: handle term, where trunc=none (no operators at all) */
 static int term_100 (ZebraMaps zebra_maps, int reg_type,
                     const char **src, char *dst, int space_split,
@@ -286,7 +305,7 @@ static int term_100 (ZebraMaps zebra_maps, int reg_type,
            {   /* reload last space */
                while (space_start < space_end)
                {
-                   if (!isalnum (*space_start) && *space_start != '-')
+                    if (strchr (REGEX_CHARS, *space_start))
                        dst[i++] = '\\';
                    dst_term[j++] = *space_start;
                    dst[i++] = *space_start++;
@@ -298,7 +317,7 @@ static int term_100 (ZebraMaps zebra_maps, int reg_type,
        /* add non-space char */
         while (s1 < s0)
         {
-            if (!isalnum (*s1) && *s1 != '-')
+            if (strchr(REGEX_CHARS, *s1))
                 dst[i++] = '\\';
            dst_term[j++] = *s1;
             dst[i++] = *s1++;
@@ -339,7 +358,7 @@ static int term_101 (ZebraMaps zebra_maps, int reg_type,
                 break;
             while (s1 < s0)
             {
-                if (!isalnum (*s1))
+                if (strchr(REGEX_CHARS, *s1))
                     dst[i++] = '\\';
                dst_term[j++] = *s1;
                 dst[i++] = *s1++;
@@ -388,7 +407,7 @@ static int term_103 (ZebraMaps zebra_maps, int reg_type, const char **src,
                 break;
             while (s1 < s0)
             {
-                if (!isalnum (*s1))
+                if (strchr(REGEX_CHARS, *s1))
                     dst[i++] = '\\';
                dst_term[j++] = *s1;
                 dst[i++] = *s1++;
@@ -443,7 +462,7 @@ static int term_104 (ZebraMaps zebra_maps, int reg_type,
                 break;
             while (s1 < s0)
             {
-                if (!isalnum (*s1))
+                if (strchr(REGEX_CHARS, *s1))
                     dst[i++] = '\\';
                dst_term[j++] = *s1;
                 dst[i++] = *s1++;
@@ -489,7 +508,7 @@ static int term_105 (ZebraMaps zebra_maps, int reg_type,
                 break;
             while (s1 < s0)
             {
-                if (!isalnum (*s1))
+                if (strchr(REGEX_CHARS, *s1))
                     dst[i++] = '\\';
                dst_term[j++] = *s1;
                 dst[i++] = *s1++;
@@ -797,12 +816,44 @@ static int string_relation (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 }
 
 static int string_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
-                       const char **term_sub, 
-                       oid_value attributeSet, NMEM stream,
-                       struct grep_info *grep_info,
-                       int reg_type, int complete_flag,
-                       int num_bases, char **basenames,
-                       char *term_dst)
+                        const char **term_sub, 
+                        oid_value attributeSet, NMEM stream,
+                        struct grep_info *grep_info,
+                        int reg_type, int complete_flag,
+                        int num_bases, char **basenames,
+                        char *term_dst, int xpath_use);
+
+static RSET term_trunc (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
+                        const char **term_sub, 
+                        oid_value attributeSet, NMEM stream,
+                        struct grep_info *grep_info,
+                        int reg_type, int complete_flag,
+                        int num_bases, char **basenames,
+                        char *term_dst,
+                        const char *rank_type, int xpath_use)
+{
+    int r;
+    grep_info->isam_p_indx = 0;
+    r = string_term (zh, zapt, term_sub, attributeSet, stream, grep_info,
+                     reg_type, complete_flag, num_bases, basenames,
+                     term_dst, xpath_use);
+    if (r < 1)
+        return 0;
+    logf (LOG_DEBUG, "term: %s", term_dst);
+    return rset_trunc (zh, grep_info->isam_p_buf,
+                       grep_info->isam_p_indx, term_dst,
+                       strlen(term_dst), rank_type, 1 /* preserve pos */,
+                       zapt->term->which);
+}
+
+
+static int string_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
+                        const char **term_sub, 
+                        oid_value attributeSet, NMEM stream,
+                        struct grep_info *grep_info,
+                        int reg_type, int complete_flag,
+                        int num_bases, char **basenames,
+                        char *term_dst, int xpath_use)
 {
     char term_dict[2*IT_MAX_WORD+4000];
     int j, r, base_no;
@@ -810,6 +861,7 @@ static int string_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
     int truncation_value;
     AttrType use;
     int use_value;
+    const char *use_string = 0;
     oid_value curAttributeSet = attributeSet;
     const char *termp;
     struct rpn_char_map_info rcmi;
@@ -817,86 +869,106 @@ static int string_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 
     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
     attr_init (&use, zapt, 1);
-    use_value = attr_find (&use, &curAttributeSet);
+    use_value = attr_find_ex (&use, &curAttributeSet, &use_string);
     logf (LOG_DEBUG, "string_term, use value %d", use_value);
     attr_init (&truncation, zapt, 5);
     truncation_value = attr_find (&truncation, NULL);
     logf (LOG_DEBUG, "truncation value %d", truncation_value);
 
-    if (use_value == -1)
+    if (use_value == -1)    /* no attribute - assumy "any" */
         use_value = 1016;
-
     for (base_no = 0; base_no < num_bases; base_no++)
     {
         attent attp;
+        data1_local_attribute id_xpath_attr;
         data1_local_attribute *local_attr;
         int max_pos, prefix_len = 0;
 
         termp = *term_sub;
-        if ((r=att_getentbyatt (zh, &attp, curAttributeSet, use_value)))
-        {
-            logf (LOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
-                  curAttributeSet, use_value, r);
-           if (r == -1)
-           {
-               char val_str[32];
-               sprintf (val_str, "%d", use_value);
-               zh->errCode = 114;
-               zh->errString = nmem_strdup (stream, val_str);
-           }
-           else
-           {
-               int oid[OID_SIZE];
-               struct oident oident;
-
-               oident.proto = PROTO_Z3950;
-               oident.oclass = CLASS_ATTSET;
-               oident.value = curAttributeSet;
-               oid_ent_to_oid (&oident, oid);
 
-               zh->errCode = 121;
-               zh->errString = nmem_strdup (stream, oident.desc);
-           }
-            return -1;
-        }
         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
         {
             zh->errCode = 109; /* Database unavailable */
             zh->errString = basenames[base_no];
             return -1;
         }
+        if (use_value == -2)  /* string attribute (assume IDXPATH/any) */
+        {
+            use_value = xpath_use;
+            attp.local_attributes = &id_xpath_attr;
+            attp.attset_ordinal = VAL_IDXPATH;
+            id_xpath_attr.next = 0;
+            id_xpath_attr.local = use_value;
+        }
+       else if (curAttributeSet == VAL_IDXPATH)
+        {
+            attp.local_attributes = &id_xpath_attr;
+            attp.attset_ordinal = VAL_IDXPATH;
+            id_xpath_attr.next = 0;
+            id_xpath_attr.local = use_value;
+        }
+        else
+        {
+            if ((r=att_getentbyatt (zh, &attp, curAttributeSet, use_value)))
+            {
+                logf (LOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
+                      curAttributeSet, use_value, r);
+                if (r == -1)
+                {
+                    /* set was found, but value wasn't defined */
+                    char val_str[32];
+                    sprintf (val_str, "%d (1)", use_value);
+                    zh->errCode = 114;
+                    zh->errString = nmem_strdup (stream, val_str);
+                }
+                else
+                {
+                    int oid[OID_SIZE];
+                    struct oident oident;
+                    
+                    oident.proto = PROTO_Z3950;
+                    oident.oclass = CLASS_ATTSET;
+                    oident.value = curAttributeSet;
+                    oid_ent_to_oid (&oident, oid);
+                    
+                    zh->errCode = 121;
+                    zh->errString = nmem_strdup (stream, oident.desc);
+                }
+                return -1;
+            }
+        }
         for (local_attr = attp.local_attributes; local_attr;
              local_attr = local_attr->next)
         {
             int ord;
-           char ord_buf[32];
-           int i, ord_len;
-
+            char ord_buf[32];
+            int i, ord_len;
+            
             ord = zebraExplain_lookupSU (zh->reg->zei, attp.attset_ordinal,
-                                          local_attr->local);
+                                         local_attr->local);
             if (ord < 0)
                 continue;
             if (prefix_len)
                 term_dict[prefix_len++] = '|';
             else
                 term_dict[prefix_len++] = '(';
-
-           ord_len = key_SU_encode (ord, ord_buf);
-           for (i = 0; i<ord_len; i++)
-           {
-               term_dict[prefix_len++] = 1;
-               term_dict[prefix_len++] = ord_buf[i];
-           }
+            
+            ord_len = key_SU_encode (ord, ord_buf);
+            for (i = 0; i<ord_len; i++)
+            {
+                term_dict[prefix_len++] = 1;
+                term_dict[prefix_len++] = ord_buf[i];
+            }
         }
         if (!prefix_len)
         {
-           char val_str[32];
-           sprintf (val_str, "%d", use_value);
-           zh->errCode = 114;
-           zh->errString = nmem_strdup (stream, val_str);
+            char val_str[32];
+            sprintf (val_str, "%d (2)", use_value);
+            zh->errCode = 114;
+            zh->errString = nmem_strdup (stream, val_str);
             return -1;
         }
-        term_dict[prefix_len++] = ')';        
+        term_dict[prefix_len++] = ')';
         term_dict[prefix_len++] = 1;
         term_dict[prefix_len++] = reg_type;
        logf (LOG_DEBUG, "reg_type = %d", term_dict[prefix_len-1]);
@@ -1023,17 +1095,55 @@ static int string_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
     return 1;
 }
 
-static void trans_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
-                        char *termz)
+
+static int trans_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
+                       char *termz)
 {
     size_t sizez;
     Z_Term *term = zapt->term;
 
-    sizez = term->u.general->len;
-    if (sizez > IT_MAX_WORD-1)
-        sizez = IT_MAX_WORD-1;
-    memcpy (termz, term->u.general->buf, sizez);
-    termz[sizez] = '\0';
+    switch (term->which)
+    {
+    case Z_Term_general:
+#if HAVE_ICONV_H
+        if (zh->iconv_to_utf8 != (iconv_t)(-1))
+        {
+            char *inbuf = term->u.general->buf;
+            size_t inleft = term->u.general->len;
+            char *outbuf = termz;
+            size_t outleft = IT_MAX_WORD-1;
+            size_t ret;
+
+            yaz_log (LOG_DEBUG, "converting general from ISO-8859-1");
+            ret = iconv(zh->iconv_to_utf8, &inbuf, &inleft,
+                        &outbuf, &outleft);
+            if (ret == (size_t)(-1))
+            {
+                ret = iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
+                zh->errCode = 125;
+                return -1;
+            }
+            *outbuf = 0;
+            return 0;
+        }
+#endif
+        sizez = term->u.general->len;
+        if (sizez > IT_MAX_WORD-1)
+            sizez = IT_MAX_WORD-1;
+        memcpy (termz, term->u.general->buf, sizez);
+        termz[sizez] = '\0';
+        break;
+    case Z_Term_characterString:
+        sizez = strlen(term->u.characterString);
+        if (sizez > IT_MAX_WORD-1)
+            sizez = IT_MAX_WORD-1;
+        memcpy (termz, term->u.characterString, sizez);
+        termz[sizez] = '\0';
+        break;
+    default:
+        zh->errCode = 124;
+    }
+    return 0;
 }
 
 static void trans_scan_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
@@ -1078,6 +1188,7 @@ static RSET rpn_prox (ZebraHandle zh, RSET *rset, int rset_no,
     int length_prox_term = 0;
     int min_nn = 10000000;
     int term_index;
+    int term_type = Z_Term_characterString;
     const char *flags = NULL;
     
     rsfd = (RSFD *) xmalloc (sizeof(*rsfd)*rset_no);
@@ -1103,6 +1214,12 @@ static RSET rpn_prox (ZebraHandle zh, RSET *rset, int rset_no,
            if (min_nn > rset[i]->rset_terms[j]->nn)
                min_nn = rset[i]->rset_terms[j]->nn;
            flags = nflags;
+            term_type = rset[i]->rset_terms[j]->type;
+
+            /* only if all term types are of type characterString .. */
+            /* the resulting term is of that type */
+            if (term_type != Z_Term_characterString)
+                term_type = Z_Term_general;
        }
     }
     for (i = 0; i<rset_no; i++)
@@ -1123,7 +1240,7 @@ static RSET rpn_prox (ZebraHandle zh, RSET *rset, int rset_no,
        rset_null_parms parms;
        
        parms.rset_term = rset_term_create (prox_term, length_prox_term,
-                                           flags);
+                                           flags, term_type);
        parms.rset_term->nn = 0;
        result = rset_create (rset_kind_null, &parms);
     }
@@ -1134,7 +1251,7 @@ static RSET rpn_prox (ZebraHandle zh, RSET *rset, int rset_no,
        RSFD rsfd_result;
 
        parms.rset_term = rset_term_create (prox_term, length_prox_term,
-                                           flags);
+                                           flags, term_type);
        parms.rset_term->nn = min_nn;
         parms.cmp = key_compare_it;
        parms.key_size = sizeof (struct it_key);
@@ -1193,7 +1310,7 @@ static RSET rpn_prox (ZebraHandle zh, RSET *rset, int rset_no,
        logf (LOG_LOG, "generic prox, dist = %d, relation = %d, ordered =%d, exclusion=%d",
              distance, relation, ordered, exclusion);
        parms.rset_term = rset_term_create (prox_term, length_prox_term,
-                                           flags);
+                                           flags, term_type);
        parms.rset_term->nn = min_nn;
         parms.cmp = key_compare_it;
        parms.key_size = sizeof (struct it_key);
@@ -1273,7 +1390,7 @@ static RSET rpn_prox (ZebraHandle zh, RSET *rset, int rset_no,
        rset_null_parms parms;
        
        parms.rset_term = rset_term_create (prox_term, length_prox_term,
-                                           flags);
+                                           flags, term_type);
        parms.rset_term->nn = 0;
        result = rset_create (rset_kind_null, &parms);
     }
@@ -1334,6 +1451,14 @@ char *normalize_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
     }
 }
 
+static void grep_info_delete (struct grep_info *grep_info)
+{
+#ifdef TERM_COUNT
+    xfree(grep_info->term_no);
+#endif
+    xfree (grep_info->isam_p_buf);
+}
+
 static int grep_info_prepare (ZebraHandle zh,
                              Z_AttributesPlusTerm *zapt,
                              struct grep_info *grep_info,
@@ -1353,6 +1478,8 @@ static int grep_info_prepare (ZebraHandle zh,
     grep_info->reg_type = reg_type;
     grep_info->termset = 0;
 
+    if (!zapt)
+        return 0;
     attr_init (&termset, zapt, 8);
     termset_value_numeric =
        attr_find_ex (&termset, NULL, &termset_value_string);
@@ -1387,12 +1514,12 @@ static RSET rpn_search_APT_phrase (ZebraHandle zh,
                                    oid_value attributeSet,
                                   NMEM stream,
                                   int reg_type, int complete_flag,
-                                  const char *rank_type,
+                                  const char *rank_type, int xpath_use,
                                   int num_bases, char **basenames)
 {
     char term_dst[IT_MAX_WORD+1];
     RSET rset[60], result;
-    int i, r, rset_no = 0;
+    int i, rset_no = 0;
     struct grep_info grep_info;
     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
     const char *termp = termz;
@@ -1403,29 +1530,24 @@ static RSET rpn_search_APT_phrase (ZebraHandle zh,
     while (1)
     { 
        logf (LOG_DEBUG, "APT_phrase termp=%s", termp);
-       grep_info.isam_p_indx = 0;
-        r = string_term (zh, zapt, &termp, attributeSet, stream, &grep_info,
-                       reg_type, complete_flag, num_bases, basenames,
-                       term_dst);
-        if (r < 1)
+        rset[rset_no] = term_trunc (zh, zapt, &termp, attributeSet,
+                                    stream, &grep_info,
+                                    reg_type, complete_flag,
+                                    num_bases, basenames,
+                                    term_dst, rank_type,
+                                    xpath_use);
+        if (!rset[rset_no])
             break;
-       logf (LOG_DEBUG, "term: %s", term_dst);
-        rset[rset_no] = rset_trunc (zh, grep_info.isam_p_buf,
-                                    grep_info.isam_p_indx, term_dst,
-                                   strlen(term_dst), rank_type);
-        assert (rset[rset_no]);
         if (++rset_no >= (int) (sizeof(rset)/sizeof(*rset)))
             break;
     }
-#ifdef TERM_COUNT
-    xfree(grep_info.term_no);
-#endif
-    xfree (grep_info.isam_p_buf);
+    grep_info_delete (&grep_info);
     if (rset_no == 0)
     {
        rset_null_parms parms;
        
-       parms.rset_term = rset_term_create (termz, -1, rank_type);
+       parms.rset_term = rset_term_create (termz, -1, rank_type,
+                                            zapt->term->which);
         return rset_create (rset_kind_null, &parms);
     }
     else if (rset_no == 1)
@@ -1443,11 +1565,12 @@ static RSET rpn_search_APT_or_list (ZebraHandle zh,
                                    NMEM stream,
                                    int reg_type, int complete_flag,
                                    const char *rank_type,
+                                    int xpath_use,
                                    int num_bases, char **basenames)
 {
     char term_dst[IT_MAX_WORD+1];
     RSET rset[60], result;
-    int i, r, rset_no = 0;
+    int i, rset_no = 0;
     struct grep_info grep_info;
     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
     const char *termp = termz;
@@ -1457,29 +1580,24 @@ static RSET rpn_search_APT_or_list (ZebraHandle zh,
     while (1)
     { 
        logf (LOG_DEBUG, "APT_or_list termp=%s", termp);
-       grep_info.isam_p_indx = 0;
-        r = string_term (zh, zapt, &termp, attributeSet, stream, &grep_info,
-                       reg_type, complete_flag, num_bases, basenames,
-                       term_dst);
-        if (r < 1)
+        rset[rset_no] = term_trunc (zh, zapt, &termp, attributeSet,
+                                    stream, &grep_info,
+                                    reg_type, complete_flag,
+                                    num_bases, basenames,
+                                    term_dst, rank_type,
+                                    xpath_use);
+        if (!rset[rset_no])
             break;
-       logf (LOG_DEBUG, "term: %s", term_dst);
-        rset[rset_no] = rset_trunc (zh, grep_info.isam_p_buf,
-                                    grep_info.isam_p_indx, term_dst,
-                                   strlen(term_dst), rank_type);
-        assert (rset[rset_no]);
         if (++rset_no >= (int) (sizeof(rset)/sizeof(*rset)))
             break;
     }
-#ifdef TERM_COUNT
-    xfree(grep_info.term_no);
-#endif
-    xfree (grep_info.isam_p_buf);
+    grep_info_delete (&grep_info);
     if (rset_no == 0)
     {
        rset_null_parms parms;
        
-       parms.rset_term = rset_term_create (termz, -1, rank_type);
+       parms.rset_term = rset_term_create (termz, -1, rank_type,
+                                            zapt->term->which);
         return rset_create (rset_kind_null, &parms);
     }
     result = rset[0];
@@ -1502,12 +1620,13 @@ static RSET rpn_search_APT_and_list (ZebraHandle zh,
                                      oid_value attributeSet,
                                     NMEM stream,
                                     int reg_type, int complete_flag,
-                                    const char *rank_type,
+                                    const char *rank_type, 
+                                     int xpath_use,
                                     int num_bases, char **basenames)
 {
     char term_dst[IT_MAX_WORD+1];
     RSET rset[60], result;
-    int i, r, rset_no = 0;
+    int i, rset_no = 0;
     struct grep_info grep_info;
     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
     const char *termp = termz;
@@ -1517,29 +1636,25 @@ static RSET rpn_search_APT_and_list (ZebraHandle zh,
     while (1)
     { 
        logf (LOG_DEBUG, "APT_and_list termp=%s", termp);
-       grep_info.isam_p_indx = 0;
-        r = string_term (zh, zapt, &termp, attributeSet, stream, &grep_info,
-                       reg_type, complete_flag, num_bases, basenames,
-                       term_dst);
-        if (r < 1)
+        rset[rset_no] = term_trunc (zh, zapt, &termp, attributeSet,
+                                    stream, &grep_info,
+                                    reg_type, complete_flag,
+                                    num_bases, basenames,
+                                    term_dst, rank_type,
+                                    xpath_use);
+        if (!rset[rset_no])
             break;
-       logf (LOG_DEBUG, "term: %s", term_dst);
-        rset[rset_no] = rset_trunc (zh, grep_info.isam_p_buf,
-                                    grep_info.isam_p_indx, term_dst,
-                                   strlen(term_dst), rank_type);
         assert (rset[rset_no]);
         if (++rset_no >= (int) (sizeof(rset)/sizeof(*rset)))
             break;
     }
-#ifdef TERM_COUNT
-    xfree(grep_info.term_no);
-#endif
-    xfree (grep_info.isam_p_buf);
+    grep_info_delete (&grep_info);
     if (rset_no == 0)
     {
        rset_null_parms parms;
        
-       parms.rset_term = rset_term_create (termz, -1, rank_type);
+       parms.rset_term = rset_term_create (termz, -1, rank_type,
+                                            zapt->term->which);
         return rset_create (rset_kind_null, &parms);
     }
     result = rset[0];
@@ -1730,20 +1845,20 @@ static RSET rpn_search_APT_numeric (ZebraHandle zh,
        logf (LOG_DEBUG, "term: %s", term_dst);
         rset[rset_no] = rset_trunc (zh, grep_info.isam_p_buf,
                                     grep_info.isam_p_indx, term_dst,
-                                   strlen(term_dst), rank_type);
+                                   strlen(term_dst), rank_type,
+                                    0 /* preserve position */,
+                                    zapt->term->which);
         assert (rset[rset_no]);
         if (++rset_no >= (int) (sizeof(rset)/sizeof(*rset)))
             break;
     }
-#ifdef TERM_COUNT
-    xfree(grep_info.term_no);
-#endif
-    xfree (grep_info.isam_p_buf);
+    grep_info_delete (&grep_info);
     if (rset_no == 0)
     {
        rset_null_parms parms;
        
-       parms.rset_term = rset_term_create (term_dst, -1, rank_type);
+       parms.rset_term = rset_term_create (term_dst, -1, rank_type,
+                                            zapt->term->which);
         return rset_create (rset_kind_null, &parms);
     }
     result = rset[0];
@@ -1771,7 +1886,8 @@ static RSET rpn_search_APT_local (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
     struct it_key key;
     rset_temp_parms parms;
 
-    parms.rset_term = rset_term_create (termz, -1, rank_type);
+    parms.rset_term = rset_term_create (termz, -1, rank_type,
+                                        zapt->term->which);
     parms.cmp = key_compare_it;
     parms.key_size = sizeof (struct it_key);
     parms.temp_path = res_get (zh->res, "setTmpDir");
@@ -1884,10 +2000,334 @@ static RSET rpn_sort_spec (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 
     sort_sequence->specs[i] = sks;
 
-    parms.rset_term = rset_term_create (termz, -1, rank_type);
+    parms.rset_term = rset_term_create (termz, -1, rank_type,
+                                        zapt->term->which);
     return rset_create (rset_kind_null, &parms);
 }
 
+struct xpath_predicate {
+    int which;
+    union {
+#define XPATH_PREDICATE_RELATION 1
+        struct {
+            char *name;
+            char *op;
+            char *value;
+        } relation;
+#define XPATH_PREDICATE_BOOLEAN 2
+        struct {
+            const char *op;
+            struct xpath_predicate *left;
+            struct xpath_predicate *right;
+        } boolean;
+    } u;
+};
+
+struct xpath_location_step {
+    char *part;
+    struct xpath_predicate *predicate;
+};
+
+static int parse_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
+                       oid_value attributeSet,
+                       struct xpath_location_step *xpath, NMEM mem)
+{
+    oid_value curAttributeSet = attributeSet;
+    AttrType use;
+    const char *use_string = 0;
+    const char *cp;
+    int no = 0;
+    
+    attr_init (&use, zapt, 1);
+    attr_find_ex (&use, &curAttributeSet, &use_string);
+
+    if (!use_string || *use_string != '/')
+        return -1;
+    cp = use_string;
+    while (*cp)
+    {
+        int i = 0;
+        while (*cp && !strchr("/[",*cp))
+        {
+            i++;
+            cp++;
+        }
+        xpath[no].predicate = 0;
+        xpath[no].part = nmem_malloc (mem, i+1);
+        memcpy (xpath[no].part,  cp - i, i);
+        xpath[no].part[i] = 0;
+
+        if (*cp == '[')
+        {
+            struct xpath_predicate *p = xpath[no].predicate =
+                nmem_malloc (mem, sizeof(struct xpath_predicate));
+
+            p->which = XPATH_PREDICATE_RELATION;
+            cp++;
+            while (*cp == ' ')
+                cp++;
+
+            for (i = 0; *cp && !strchr("><=] ", *cp); i++)
+                cp++;
+            p->u.relation.name = nmem_malloc (mem, i+1);
+            memcpy (p->u.relation.name, cp - i, i);
+            p->u.relation.name[i] = 0;
+            while (*cp == ' ')
+                cp++;
+            if (*cp != ']')
+            {
+                for (i = 0; *cp && strchr(">=<!", *cp); i++)
+                    cp++;
+
+                p->u.relation.op = nmem_malloc (mem, i+1);
+                if (i)
+                    memcpy (p->u.relation.op, cp - i, i);
+                p->u.relation.op[i] = 0;
+                
+                while (*cp == ' ')
+                    cp++;
+                
+                if (strchr("\"'", *cp))
+                {
+                    cp++;
+                    for (i = 0; *cp && !strchr("\"'", *cp); i++)
+                        cp++;
+
+                    p->u.relation.value = nmem_malloc (mem, i+1);
+                    if (i)
+                        memcpy (p->u.relation.value, cp - i, i);
+                    p->u.relation.value[i] = 0;
+                    yaz_log (LOG_LOG, "value=%s", p->u.relation.value);
+
+                    cp++;
+                }                           
+                else
+                {
+                    for (i = 0; *cp && !strchr(" ]", *cp); i++)
+                        cp++;
+                    p->u.relation.value = nmem_malloc (mem, i+1);
+                    if (i)
+                        memcpy (p->u.relation.value, cp - i, i);
+                    p->u.relation.value[i] = 0;
+                }
+                while (*cp == ' ')
+                    cp++;
+            }
+            if (*cp == ']')
+                cp++;
+        } /* end of ] predicate */
+        no++;
+        if (*cp != '/')
+            break;
+        cp++;
+    }
+    return no;
+}
+                
+
+static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
+                        int reg_type, const char *term, int use,
+                        oid_value curAttributeSet)
+{
+    RSET rset;
+    struct grep_info grep_info;
+    char term_dict[2048];
+    char ord_buf[32];
+    int prefix_len = 0;
+    int ord = zebraExplain_lookupSU (zh->reg->zei, curAttributeSet, use);
+    int ord_len, i, r, max_pos;
+    int term_type = Z_Term_characterString;
+    const char *flags = "void";
+
+    if (grep_info_prepare (zh, 0 /* zapt */, &grep_info, '0', stream))
+    {
+       rset_null_parms parms;
+       
+       parms.rset_term = rset_term_create (term, strlen(term),
+                                           flags, term_type);
+       parms.rset_term->nn = 0;
+       return rset_create (rset_kind_null, &parms);
+    }
+
+    if (ord < 0)
+    {
+       rset_null_parms parms;
+       
+       parms.rset_term = rset_term_create (term, strlen(term),
+                                           flags, term_type);
+       parms.rset_term->nn = 0;
+       return rset_create (rset_kind_null, &parms);
+    }
+    if (prefix_len)
+        term_dict[prefix_len++] = '|';
+    else
+        term_dict[prefix_len++] = '(';
+    
+    ord_len = key_SU_encode (ord, ord_buf);
+    for (i = 0; i<ord_len; i++)
+    {
+        term_dict[prefix_len++] = 1;
+        term_dict[prefix_len++] = ord_buf[i];
+    }
+    term_dict[prefix_len++] = ')';
+    term_dict[prefix_len++] = 1;
+    term_dict[prefix_len++] = reg_type;
+    
+    strcpy (term_dict+prefix_len, term);
+    
+    grep_info.isam_p_indx = 0;
+    r = dict_lookup_grep (zh->reg->dict, term_dict, 0,
+                          &grep_info, &max_pos, 0, grep_handle);
+    yaz_log (LOG_LOG, "%s %d positions", term,
+             grep_info.isam_p_indx);
+    rset = rset_trunc (zh, grep_info.isam_p_buf,
+                       grep_info.isam_p_indx, term, strlen(term),
+                       flags, 1, term_type);
+    grep_info_delete (&grep_info);
+    return rset;
+}
+
+static RSET rpn_search_xpath (ZebraHandle zh,
+                              oid_value attributeSet,
+                              int num_bases, char **basenames,
+                              NMEM stream, const char *rank_type, RSET rset,
+                              int xpath_len, struct xpath_location_step *xpath)
+{
+    oid_value curAttributeSet = attributeSet;
+    int base_no;
+    int i;
+
+    if (xpath_len < 0)
+        return rset;
+
+    yaz_log (LOG_LOG, "len=%d", xpath_len);
+    for (i = 0; i<xpath_len; i++)
+    {
+        yaz_log (LOG_LOG, "XPATH %d %s", i, xpath[i].part);
+
+    }
+
+    curAttributeSet = VAL_IDXPATH;
+
+    /*
+      //a    ->    a/.*
+      //a/b  ->    b/a/.*
+      /a     ->    a/
+      /a/b   ->    b/a/
+
+      /      ->    none
+
+   a[@attr=value]/b[@other=othervalue]
+
+ /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
+ /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
+ /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
+ /a/b[@c=y] val range(b/a/,freetext(w,1016,val),b/a/,@c=y)
+ /a[@c=y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c=y)
+ /a[@c=x]/b[@c=y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c=y),a/,@c=x)
+      
+    */
+
+    dict_grep_cmap (zh->reg->dict, 0, 0);
+
+    for (base_no = 0; base_no < num_bases; base_no++)
+    {
+        int level = xpath_len;
+        int first_path = 1;
+        
+        if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
+        {
+            zh->errCode = 109; /* Database unavailable */
+            zh->errString = basenames[base_no];
+            return rset;
+        }
+        while (--level >= 0)
+        {
+            char xpath_rev[128];
+            int i, len;
+            rset_between_parms parms;
+            RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
+
+            *xpath_rev = 0;
+            len = 0;
+            for (i = level; i >= 1; --i)
+            {
+                const char *cp = xpath[i].part;
+                if (*cp)
+                {
+                    for (;*cp; cp++)
+                        if (*cp == '*')
+                        {
+                            memcpy (xpath_rev + len, "[^/]*", 5);
+                            len += 5;
+                        }
+                        else if (*cp == ' ')
+                        {
+
+                            xpath_rev[len++] = 1;
+                            xpath_rev[len++] = ' ';
+                        }
+
+                        else
+                            xpath_rev[len++] = *cp;
+                    xpath_rev[len++] = '/';
+                }
+                else if (i == 1)  /* // case */
+                {
+                    xpath_rev[len++] = '.';
+                    xpath_rev[len++] = '*';
+                }
+            }
+            xpath_rev[len] = 0;
+
+            if (xpath[level].predicate &&
+                xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
+                xpath[level].predicate->u.relation.name[0])
+            {
+                char predicate_str[128];
+
+                strcpy (predicate_str,
+                        xpath[level].predicate->u.relation.name+1);
+                if (xpath[level].predicate->u.relation.value)
+                {
+                    strcat (predicate_str, "=");
+                    strcat (predicate_str,
+                            xpath[level].predicate->u.relation.value);
+                }
+                rset_attr = xpath_trunc (
+                    zh, stream, '0', predicate_str, 3, curAttributeSet);
+            } 
+            else 
+            {
+                if (!first_path)
+                    continue;
+            }
+            yaz_log (LOG_LOG, "xpath_rev (%d) = %s", level, xpath_rev);
+           if (strlen(xpath_rev))
+           {
+                rset_start_tag = xpath_trunc(zh, stream, 
+                                         '0', xpath_rev, 1, curAttributeSet);
+            
+                rset_end_tag = xpath_trunc(zh, stream,
+                                       '0', xpath_rev, 2, curAttributeSet);
+
+                parms.key_size = sizeof(struct it_key);
+                parms.cmp = key_compare_it;
+                parms.rset_l = rset_start_tag;
+                parms.rset_m = rset;
+                parms.rset_r = rset_end_tag;
+                parms.rset_attr = rset_attr;
+                parms.printer = key_print_it;
+                rset = rset_create (rset_kind_between, &parms);
+            }
+            first_path = 0;
+        }
+    }
+
+    return rset;
+}
+
+
 
 static RSET rpn_search_APT (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
                             oid_value attributeSet, NMEM stream,
@@ -1900,6 +2340,10 @@ static RSET rpn_search_APT (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
     int complete_flag;
     int sort_flag;
     char termz[IT_MAX_WORD+1];
+    RSET rset = 0;
+    int xpath_len;
+    int xpath_use = 0;
+    struct xpath_location_step xpath[10];
 
     zebra_maps_attr (zh->reg->zebra_maps, zapt, &reg_id, &search_type,
                     rank_type, &complete_flag, &sort_flag);
@@ -1909,48 +2353,60 @@ static RSET rpn_search_APT (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
     logf (LOG_DEBUG, "search_type=%s", search_type);
     logf (LOG_DEBUG, "rank_type=%s", rank_type);
 
-    if (zapt->term->which != Z_Term_general)
-    {
-        zh->errCode = 124;
-        return NULL;
-    }
-    trans_term (zh, zapt, termz);
+    if (trans_term (zh, zapt, termz))
+        return 0;
 
     if (sort_flag)
        return rpn_sort_spec (zh, zapt, attributeSet, stream, sort_sequence,
                              rank_type);
+    xpath_len = parse_xpath(zh, zapt, attributeSet, xpath, stream);
+    if (xpath_len >= 0)
+    {
+        xpath_use = 1016;
+        if (xpath[xpath_len-1].part[0] == '@')
+            xpath_use = 1015;
+    }
 
     if (!strcmp (search_type, "phrase"))
     {
-       return rpn_search_APT_phrase (zh, zapt, termz, attributeSet, stream,
+       rset = rpn_search_APT_phrase (zh, zapt, termz, attributeSet, stream,
                                      reg_id, complete_flag, rank_type,
+                                      xpath_use,
                                      num_bases, basenames);
     }
     else if (!strcmp (search_type, "and-list"))
     {
-       return rpn_search_APT_and_list (zh, zapt, termz, attributeSet, stream,
+       rset = rpn_search_APT_and_list (zh, zapt, termz, attributeSet, stream,
                                        reg_id, complete_flag, rank_type,
+                                        xpath_use,
                                        num_bases, basenames);
     }
     else if (!strcmp (search_type, "or-list"))
     {
-       return rpn_search_APT_or_list (zh, zapt, termz, attributeSet, stream,
+       rset = rpn_search_APT_or_list (zh, zapt, termz, attributeSet, stream,
                                       reg_id, complete_flag, rank_type,
+                                       xpath_use,
                                       num_bases, basenames);
     }
     else if (!strcmp (search_type, "local"))
     {
-        return rpn_search_APT_local (zh, zapt, termz, attributeSet, stream,
+        rset = rpn_search_APT_local (zh, zapt, termz, attributeSet, stream,
                                     rank_type);
     }
     else if (!strcmp (search_type, "numeric"))
     {
-       return rpn_search_APT_numeric (zh, zapt, termz, attributeSet, stream,
+       rset = rpn_search_APT_numeric (zh, zapt, termz, attributeSet, stream,
                                       reg_id, complete_flag, rank_type,
                                       num_bases, basenames);
     }
-    zh->errCode = 118;
-    return NULL;
+    else if (!strcmp (search_type, "always"))
+    {
+        rset = 0;
+    }
+    else
+        zh->errCode = 118;
+    return rpn_search_xpath (zh, attributeSet, num_bases, basenames,
+                             stream, rank_type, rset, xpath_len, xpath);
 }
 
 static RSET rpn_search_structure (ZebraHandle zh, Z_RPNStructure *zs,
@@ -2357,7 +2813,7 @@ void rpn_scan (ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
                           &glist[i+before].term, mterm);
         rset = rset_trunc (zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
                           glist[i+before].term, strlen(glist[i+before].term),
-                          NULL);
+                          NULL, 0, zapt->term->which);
 
         ptr[j0]++;
         for (j = j0+1; j<ord_no; j++)
@@ -2372,7 +2828,8 @@ void rpn_scan (ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
                 rset2 =
                    rset_trunc (zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
                               glist[i+before].term,
-                              strlen(glist[i+before].term), NULL);
+                              strlen(glist[i+before].term), NULL, 0,
+                               zapt->term->which);
 
                 bool_parms.key_size = sizeof(struct it_key);
                 bool_parms.cmp = key_compare_it;
@@ -2423,7 +2880,7 @@ void rpn_scan (ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
         rset = rset_trunc
                (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
                glist[before-1-i].term, strlen(glist[before-1-i].term),
-               NULL);
+               NULL, 0, zapt->term->which);
 
         ptr[j0]++;
 
@@ -2439,7 +2896,8 @@ void rpn_scan (ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
                 rset2 = rset_trunc (zh,
                          &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
                                    glist[before-1-i].term,
-                                   strlen(glist[before-1-i].term), NULL);
+                                   strlen(glist[before-1-i].term), NULL, 0,
+                                    zapt->term->which);
 
                 bool_parms.key_size = sizeof(struct it_key);
                 bool_parms.cmp = key_compare_it;