Compact use/set bytes used in search service.
[idzebra-moved-to-github.git] / index / zrpn.c
index 501a294..409cea0 100644 (file)
@@ -4,7 +4,52 @@
  * Sebastian Hammer, Adam Dickmeiss
  *
  * $Log: zrpn.c,v $
- * Revision 1.33  1995-11-01 13:58:28  quinn
+ * Revision 1.44  1996-05-14 06:16:44  adam
+ * Compact use/set bytes used in search service.
+ *
+ * Revision 1.43  1996/05/09 09:54:43  adam
+ * Server supports maps from one logical attributes to a list of physical
+ * attributes.
+ * The extraction process doesn't make space consuming 'any' keys.
+ *
+ * Revision 1.42  1996/05/09  07:28:56  quinn
+ * Work towards phrases and multiple registers
+ *
+ * Revision 1.41  1996/03/20  09:36:43  adam
+ * Function dict_lookup_grep got extra parameter, init_pos, which marks
+ * from which position in pattern approximate pattern matching should occur.
+ * Approximate pattern matching is used in relevance=re-2.
+ *
+ * Revision 1.40  1996/02/02  13:44:44  adam
+ * The public dictionary functions simply use char instead of Dict_char
+ * to represent search strings. Dict_char is used internally only.
+ *
+ * Revision 1.39  1996/01/03  16:22:13  quinn
+ * operator->roperator
+ *
+ * Revision 1.38  1995/12/11  09:12:55  adam
+ * The rec_get function returns NULL if record doesn't exist - will
+ * happen in the server if the result set records have been deleted since
+ * the creation of the set (i.e. the search).
+ * The server saves a result temporarily if it is 'volatile', i.e. the
+ * set is register dependent.
+ *
+ * Revision 1.37  1995/12/06  15:05:28  adam
+ * More verbose in count_set.
+ *
+ * Revision 1.36  1995/12/06  12:41:27  adam
+ * New command 'stat' for the index program.
+ * Filenames can be read from stdin by specifying '-'.
+ * Bug fix/enhancement of the transformation from terms to regular
+ * expressons in the search engine.
+ *
+ * Revision 1.35  1995/11/27  09:29:00  adam
+ * Bug fixes regarding conversion to regular expressions.
+ *
+ * Revision 1.34  1995/11/16  17:00:56  adam
+ * Better logging of rpn query.
+ *
+ * Revision 1.33  1995/11/01  13:58:28  quinn
  * Moving data1 to yaz/retrieval
  *
  * Revision 1.32  1995/10/27  14:00:11  adam
 #include <stdio.h>
 #include <assert.h>
 #include <unistd.h>
+#include <ctype.h>
 
 #include "zserver.h"
 #include "attribute.h"
@@ -134,46 +180,7 @@ int index_word_prefix_map (char *string, oid_value attrSet, int attrUse,
         return -1;
     logf (LOG_DEBUG, "ord=%d", attp->attset_ordinal);
     return index_word_prefix (string, attp->attset_ordinal,
-                              attp->local_attribute, basename);
-}
-
-/*
- * attr_print: log attributes
- */
-static void attr_print (Z_AttributesPlusTerm *t)
-{
-    int of, i;
-    for (of = 0; of < t->num_attributes; of++)
-    {
-        Z_AttributeElement *element;
-        element = t->attributeList[of];
-
-        switch (element->which) 
-        {
-        case Z_AttributeValue_numeric:
-            logf (LOG_DEBUG, "attributeType=%d value=%d", 
-                  *element->attributeType,
-                  *element->value.numeric);
-            break;
-        case Z_AttributeValue_complex:
-            logf (LOG_DEBUG, "attributeType=%d complex", 
-                  *element->attributeType);
-            for (i = 0; i<element->value.complex->num_list; i++)
-            {
-                if (element->value.complex->list[i]->which ==
-                    Z_StringOrNumeric_string)
-                    logf (LOG_DEBUG, "   string: '%s'",
-                          element->value.complex->list[i]->u.string);
-                else if (element->value.complex->list[i]->which ==
-                         Z_StringOrNumeric_numeric)
-                    logf (LOG_DEBUG, "   numeric: '%d'",
-                          *element->value.complex->list[i]->u.numeric);
-            }
-            break;
-        default:
-            assert (0);
-        }
-    }
+                              attp->local_attributes->local, basename);
 }
 
 typedef struct {
@@ -507,7 +514,7 @@ static void add_isam_p (const char *info, struct grep_info *p)
     (p->isam_p_indx)++;
 }
 
-static int grep_handle (Dict_char *name, const char *info, void *p)
+static int grep_handle (char *name, const char *info, void *p)
 {
     logf (LOG_DEBUG, "dict name: %s", name);
     add_isam_p (info, p);
@@ -643,14 +650,21 @@ static int relational_term (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
     }
     logf (LOG_DEBUG, "dict_lookup_grep: %s", term_dict);
     r = dict_lookup_grep (zi->wordDict, term_dict, 0, grep_info, max_pos,
-                          grep_handle);
+                          0, grep_handle);
     if (r)
         logf (LOG_WARN, "dict_lookup_grep fail, rel=gt: %d", r);
     logf (LOG_DEBUG, "%d positions", grep_info->isam_p_indx);
     return 1;
 }
 
-static int trunc_term (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
+static void verbatim_char (int ch, int *indx, char *dst)
+{
+    if (!isalnum (ch))
+        dst[(*indx)++] = '\\';
+    dst[(*indx)++] = ch;
+}
+
+static int field_term (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
                        const char *term_sub,
                        oid_value attributeSet, struct grep_info *grep_info,
                        int num_bases, char **basenames)
@@ -675,7 +689,224 @@ static int trunc_term (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
 
     for (base_no = 0; base_no < num_bases; base_no++)
     {
+#if 1
+        attent *attp;
+        data1_local_attribute *local_attr;
+        int max_pos, prefix_len = 0;
+
+        attp = att_getentbyatt (curAttributeSet, use_value);
+        if (!attp)
+        {
+            zi->errCode = 114;
+            return -1;
+        }
+        if (zebTargetInfo_curDatabase (zi->zti, basenames[base_no]))
+        {
+            zi->errCode = 109; /* Database unavailable */
+            zi->errString = basenames[base_no];
+        }
+        for (local_attr = attp->local_attributes; local_attr;
+             local_attr = local_attr->next)
+        {
+            int ord;
+
+            ord = zebTargetInfo_lookupSU (zi->zti, attp->attset_ordinal,
+                                          local_attr->local);
+            if (ord < 0)
+                continue;
+            if (prefix_len)
+                term_dict[prefix_len++] = '|';
+            else
+                term_dict[prefix_len++] = '(';
+            if ((ord >= 'A' && ord <= 'Z') || (ord >= 'a' && ord <= 'z'))
+                term_dict[prefix_len++] = ord;
+            else
+            {
+                term_dict[prefix_len++] = '\\';
+                term_dict[prefix_len++] = ord;
+            }
+        }
+        if (!prefix_len)
+        {
+            zi->errCode = 114;
+            return -1;
+        }
+        term_dict[prefix_len++] = ')';
+        term_dict[prefix_len] = '\0';
+        if (!relational_term (zi, zapt, term_sub, term_dict,
+                              attributeSet, grep_info, &max_pos))
+        {
+            const char *cp;
+
+            j = prefix_len;
+            switch (truncation_value)
+            {
+            case -1:         /* not specified */
+            case 100:        /* do not truncate */
+                term_dict[j++] = '(';
+                for (i = 0; term_sub[i]; i++)
+                    verbatim_char (term_sub[i], &j, term_dict);
+                strcpy (term_dict+j, ")");
+                r = dict_lookup_grep (zi->wordDict, term_dict, 0, grep_info,
+                                      &max_pos, 0, grep_handle);
+                if (r)
+                    logf (LOG_WARN, "dict_lookup_grep err, trunc=none:%d", r);
+                break;
+            case 1:          /* right truncation */
+                term_dict[j++] = '(';
+                for (i = 0; term_sub[i]; i++)
+                    verbatim_char (term_sub[i], &j, term_dict);
+                strcpy (term_dict+j, ".*)");
+                dict_lookup_grep (zi->wordDict, term_dict, 0, grep_info,
+                                  &max_pos, 0, grep_handle);
+                break;
+            case 2:          /* left truncation */
+            case 3:          /* left&right truncation */
+                zi->errCode = 120;
+                return -1;
+            case 101:        /* process # in term */
+                term_dict[j++] = '(';
+                for (i=0; term_sub[i]; i++)
+                    if (term_sub[i] == '#' && i > 2)
+                    {
+                        term_dict[j++] = '.';
+                        term_dict[j++] = '*';
+                    }
+                    else
+                        verbatim_char (term_sub[i], &j, term_dict);
+                strcpy (term_dict+j, ")");
+                r = dict_lookup_grep (zi->wordDict, term_dict, 0, grep_info,
+                                      &max_pos, 0, grep_handle);
+                if (r)
+                    logf (LOG_WARN, "dict_lookup_grep err, trunc=#: %d",
+                          r);
+                break;
+            case 102:        /* regular expression */
+               sprintf (term_dict + j, "(%s)", term_sub);
+                r = dict_lookup_grep (zi->wordDict, term_dict, 0, grep_info,
+                                      &max_pos, 0, grep_handle);
+                if (r)
+                    logf (LOG_WARN, "dict_lookup_grep err, trunc=regular: %d",
+                          r);
+                break;
+            case 103:        /* regular expression with error correction */
+                cp = term_sub;
+                r = 0;
+               if (*cp == '*' && cp[1] && cp[2])
+                {
+                    r = atoi (cp+1);
+                    cp += 2;
+                }
+               sprintf (term_dict + j, "(%s)", cp);
+                r = dict_lookup_grep (zi->wordDict, term_dict, r, grep_info,
+                                      &max_pos, j, grep_handle);
+                if (r)
+                    logf (LOG_WARN, "dict_lookup_grep err, trunc=eregular: %d",
+                          r);
+                break;
+            }
+        }
+#else
         int max_pos;
+#if 1
+        attent *attp;
+        data1_local_attribute *local_attr;
+        int prefix_len;
+
+        attp = att_getentbyatt (curAttributeSet, use_value);
+        if (!attp)
+        {
+            zi->errCode = 114;
+            return -1;
+        }
+        for (local_attr = attp->local_attributes; local_attr;
+             local_attr = local_attr->next)
+        {
+            prefix_len = index_word_prefix (term_dict, attp->attset_ordinal,
+                                            local_attr->local,
+                                            basenames[base_no]);
+            
+            if (!relational_term (zi, zapt, term_sub, term_dict,
+                                  attributeSet, grep_info, &max_pos))
+            {
+                const char *cp;
+                
+                j = prefix_len;
+                switch (truncation_value)
+                {
+                case -1:         /* not specified */
+                case 100:        /* do not truncate */
+                    term_dict[j++] = '(';
+                    for (i = 0; term_sub[i]; i++)
+                        verbatim_char (term_sub[i], &j, term_dict);
+                    strcpy (term_dict+j, ")");
+                    r = dict_lookup_grep (zi->wordDict, term_dict, 0, grep_info,
+                                          &max_pos, 0, grep_handle);
+                    if (r)
+                        logf (LOG_WARN, "dict_lookup_grep err, trunc=none:%d", r);
+                    break;
+                case 1:          /* right truncation */
+                    term_dict[j++] = '(';
+                    for (i = 0; term_sub[i]; i++)
+                        verbatim_char (term_sub[i], &j, term_dict);
+                    strcpy (term_dict+j, ".*)");
+                    dict_lookup_grep (zi->wordDict, term_dict, 0, grep_info,
+                                      &max_pos, 0, grep_handle);
+                    break;
+                case 2:          /* left truncation */
+                case 3:          /* left&right truncation */
+                    zi->errCode = 120;
+                    return -1;
+                case 101:        /* process # in term */
+                    term_dict[j++] = '(';
+                    for (i=0; term_sub[i]; i++)
+                        if (term_sub[i] == '#' && i > 2)
+                        {
+                            term_dict[j++] = '.';
+                            term_dict[j++] = '*';
+                        }
+                        else
+                            verbatim_char (term_sub[i], &j, term_dict);
+                    strcpy (term_dict+j, ")");
+                    r = dict_lookup_grep (zi->wordDict, term_dict, 0, grep_info,
+                                          &max_pos, 0, grep_handle);
+                    if (r)
+                        logf (LOG_WARN, "dict_lookup_grep err, trunc=#: %d",
+                              r);
+                    break;
+                case 102:        /* regular expression */
+                    sprintf (term_dict + j, "(%s)", term_sub);
+                    r = dict_lookup_grep (zi->wordDict, term_dict, 0, grep_info,
+                                          &max_pos, 0, grep_handle);
+                    if (r)
+                        logf (LOG_WARN, "dict_lookup_grep err, trunc=regular: %d",
+                              r);
+                    break;
+                case 103:        /* regular expression with error correction */
+                    cp = term_sub;
+                    r = 0;
+                    if (*cp == '*' && cp[1] && cp[2])
+                    {
+                        r = atoi (cp+1);
+                        cp += 2;
+                    }
+                    sprintf (term_dict + j, "(%s)", cp);
+                    r = dict_lookup_grep (zi->wordDict, term_dict, r, grep_info,
+                                          &max_pos, j, grep_handle);
+                    if (r)
+                        logf (LOG_WARN, "dict_lookup_grep err, trunc=eregular: %d",
+                              r);
+                    break;
+                }
+            }
+            if (max_pos <= strlen(basenames[base_no]))
+            {
+                zi->errCode = 109; /* Database unavailable */
+                zi->errString = basenames[base_no];
+                return -1;
+            }
+        }
+#else
         int prefix_len = index_word_prefix_map (term_dict, curAttributeSet,
                                                 use_value,
                                                 basenames[base_no]);
@@ -687,84 +918,102 @@ static int trunc_term (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
         if (!relational_term (zi, zapt, term_sub, term_dict,
                               attributeSet, grep_info, &max_pos))
         {
+            const char *cp;
+
+            j = prefix_len;
             switch (truncation_value)
             {
             case -1:         /* not specified */
             case 100:        /* do not truncate */
-                strcat (term_dict, "(");
-                strcat (term_dict, term_sub);
-                strcat (term_dict, ")");
-                logf (LOG_DEBUG, "dict_lookup_grep: %s", term_dict);
+                term_dict[j++] = '(';
+                for (i = 0; term_sub[i]; i++)
+                    verbatim_char (term_sub[i], &j, term_dict);
+                strcpy (term_dict+j, ")");
                 r = dict_lookup_grep (zi->wordDict, term_dict, 0, grep_info,
-                                      &max_pos, grep_handle);
+                                      &max_pos, 0, grep_handle);
                 if (r)
                     logf (LOG_WARN, "dict_lookup_grep err, trunc=none:%d", r);
                 break;
             case 1:          /* right truncation */
-                strcat (term_dict, term_sub);
-                strcat (term_dict, ".*");
+                term_dict[j++] = '(';
+                for (i = 0; term_sub[i]; i++)
+                    verbatim_char (term_sub[i], &j, term_dict);
+                strcpy (term_dict+j, ".*)");
                 dict_lookup_grep (zi->wordDict, term_dict, 0, grep_info,
-                                  &max_pos, grep_handle);
+                                  &max_pos, 0, grep_handle);
                 break;
             case 2:          /* left truncation */
             case 3:          /* left&right truncation */
                 zi->errCode = 120;
                 return -1;
             case 101:        /* process # in term */
-                for (j = strlen(term_dict), i = 0; term_sub[i] && i < 2; i++)
-                    term_dict[j++] = term_sub[i];
-                for (; term_sub[i]; i++)
-                    if (term_sub[i] == '#')
+                term_dict[j++] = '(';
+                for (i=0; term_sub[i]; i++)
+                    if (term_sub[i] == '#' && i > 2)
                     {
                         term_dict[j++] = '.';
                         term_dict[j++] = '*';
                     }
                     else
-                        term_dict[j++] = term_sub[i];
-                term_dict[j] = '\0';
+                        verbatim_char (term_sub[i], &j, term_dict);
+                strcpy (term_dict+j, ")");
                 r = dict_lookup_grep (zi->wordDict, term_dict, 0, grep_info,
-                                      &max_pos, grep_handle);
+                                      &max_pos, 0, grep_handle);
                 if (r)
                     logf (LOG_WARN, "dict_lookup_grep err, trunc=#: %d",
                           r);
                 break;
             case 102:        /* regular expression */
-                strcat (term_dict, "(");
-                strcat (term_dict, term_sub);
-                strcat (term_dict, ")");
-                logf (LOG_DEBUG, "dict_lookup_grep: %s", term_dict);
+               sprintf (term_dict + j, "(%s)", term_sub);
                 r = dict_lookup_grep (zi->wordDict, term_dict, 0, grep_info,
-                                      &max_pos, grep_handle);
+                                      &max_pos, 0, grep_handle);
                 if (r)
                     logf (LOG_WARN, "dict_lookup_grep err, trunc=regular: %d",
                           r);
                 break;
+            case 103:        /* regular expression with error correction */
+                cp = term_sub;
+                r = 0;
+               if (*cp == '*' && cp[1] && cp[2])
+                {
+                    r = atoi (cp+1);
+                    cp += 2;
+                }
+               sprintf (term_dict + j, "(%s)", cp);
+                r = dict_lookup_grep (zi->wordDict, term_dict, r, grep_info,
+                                      &max_pos, j, grep_handle);
+                if (r)
+                    logf (LOG_WARN, "dict_lookup_grep err, trunc=eregular: %d",
+                          r);
+                break;
             }
         }
-        logf (LOG_DEBUG, "max_pos = %d", max_pos);
         if (max_pos <= strlen(basenames[base_no]))
         {
             zi->errCode = 109; /* Database unavailable */
             zi->errString = basenames[base_no];
             return -1;
         }
+#endif
+#endif
     }
     logf (LOG_DEBUG, "%d positions", grep_info->isam_p_indx);
     return 0;
 }
 
 static void trans_term (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
-                        char *termz)
+                        int regType, char *termz)
 {
     size_t i, sizez;
     Z_Term *term = zapt->term;
 
     sizez = term->u.general->len;
-    if (sizez > IT_MAX_WORD)
-        sizez = IT_MAX_WORD;
+    if (sizez > IT_MAX_WORD-1)
+        sizez = IT_MAX_WORD-1;
+    termz[0] = regType;
     for (i = 0; i < sizez; i++)
-        termz[i] = index_char_cvt (term->u.general->buf[i]);
-    termz[i] = '\0';
+        termz[i+1] = index_char_cvt (term->u.general->buf[i]);
+    termz[i+1] = '\0';
 }
 
 static RSET rpn_search_APT_relevance (ZServerInfo *zi, 
@@ -789,7 +1038,8 @@ static RSET rpn_search_APT_relevance (ZServerInfo *zi,
         zi->errCode = 124;
         return NULL;
     }
-    trans_term (zi, zapt, termz);
+    trans_term (zi, zapt, 'w', termz);
+
     grep_info.isam_p_indx = 0;
     grep_info.isam_p_size = 0;
     grep_info.isam_p_buf = NULL;
@@ -802,7 +1052,7 @@ static RSET rpn_search_APT_relevance (ZServerInfo *zi,
         }
         else
             strcpy (term_sub, p0);
-        if (trunc_term (zi, zapt, term_sub, attributeSet, &grep_info,
+        if (field_term (zi, zapt, term_sub, attributeSet, &grep_info,
                         num_bases, basenames))
             return NULL;
         if (!p1)
@@ -821,6 +1071,44 @@ static RSET rpn_search_APT_relevance (ZServerInfo *zi,
     return result;
 }
 
+static RSET rpn_search_APT_cphrase (ZServerInfo *zi,
+                                    Z_AttributesPlusTerm *zapt,
+                                    oid_value attributeSet,
+                                    int num_bases, char **basenames)
+{
+    rset_isam_parms parms;
+    char termz[IT_MAX_WORD+1];
+    struct grep_info grep_info;
+    RSET result;
+
+    if (zapt->term->which != Z_Term_general)
+    {
+        zi->errCode = 124;
+        return NULL;
+    }
+    trans_term (zi, zapt, 'p', termz);
+
+    grep_info.isam_p_indx = 0;
+    grep_info.isam_p_size = 0;
+    grep_info.isam_p_buf = NULL;
+
+    if (field_term (zi, zapt, termz, attributeSet, &grep_info,
+                    num_bases, basenames))
+        return NULL;
+    if (grep_info.isam_p_indx < 1)
+        result = rset_create (rset_kind_null, NULL);
+    else if (grep_info.isam_p_indx == 1)
+    {
+        parms.is = zi->wordIsam;
+        parms.pos = *grep_info.isam_p_buf;
+        result = rset_create (rset_kind_isam, &parms);
+    }
+    else
+        result = rset_trunc (zi->wordIsam, grep_info.isam_p_buf,
+                             grep_info.isam_p_indx);
+    xfree (grep_info.isam_p_buf);
+    return result;
+}
 static RSET rpn_search_APT_word (ZServerInfo *zi,
                                  Z_AttributesPlusTerm *zapt,
                                  oid_value attributeSet,
@@ -836,13 +1124,13 @@ static RSET rpn_search_APT_word (ZServerInfo *zi,
         zi->errCode = 124;
         return NULL;
     }
-    trans_term (zi, zapt, termz);
+    trans_term (zi, zapt, 'w', termz);
 
     grep_info.isam_p_indx = 0;
     grep_info.isam_p_size = 0;
     grep_info.isam_p_buf = NULL;
 
-    if (trunc_term (zi, zapt, termz, attributeSet, &grep_info,
+    if (field_term (zi, zapt, termz, attributeSet, &grep_info,
                     num_bases, basenames))
         return NULL;
     if (grep_info.isam_p_indx < 1)
@@ -963,7 +1251,7 @@ static RSET rpn_search_APT_phrase (ZServerInfo *zi,
         zi->errCode = 124;
         return NULL;
     }
-    trans_term (zi, zapt, termz);
+    trans_term (zi, zapt, 'w', termz);
 
     grep_info.isam_p_size = 0;
     grep_info.isam_p_buf = NULL;
@@ -979,7 +1267,7 @@ static RSET rpn_search_APT_phrase (ZServerInfo *zi,
             strcpy (term_sub, p0);
 
         grep_info.isam_p_indx = 0;
-        if (trunc_term (zi, zapt, term_sub, attributeSet, &grep_info,
+        if (field_term (zi, zapt, term_sub, attributeSet, &grep_info,
                         num_bases, basenames))
             return NULL;
         if (grep_info.isam_p_indx == 0)
@@ -1035,8 +1323,9 @@ static RSET rpn_search_APT_local (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
     result = rset_create (rset_kind_temp, &parms);
     rsfd = rset_open (result, RSETF_WRITE|RSETF_SORT_SYSNO);
 
-    trans_term (zi, zapt, termz);
-    key.sysno = atoi (termz);
+    trans_term (zi, zapt, 'w', termz);
+
+    key.sysno = atoi (termz+1);
     if (key.sysno <= 0)
         key.sysno = 1;
     rset_write (result, rsfd, &key);
@@ -1050,25 +1339,34 @@ static RSET rpn_search_APT (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
 {
     AttrType relation;
     AttrType structure;
-    int relation_value, structure_value;
+    AttrType completeness;
+    int relation_value, structure_value, completeness_value;
 
     attr_init (&relation, zapt, 2);
     attr_init (&structure, zapt, 4);
+    attr_init (&completeness, zapt, 6);
     
     relation_value = attr_find (&relation, NULL);
     structure_value = attr_find (&structure, NULL);
+    completeness_value = attr_find (&completeness, NULL);
     switch (structure_value)
     {
     case -1:
         if (relation_value == 102) /* relevance relation */
             return rpn_search_APT_relevance (zi, zapt, attributeSet,
                                              num_bases, basenames);
+        if (completeness_value == 2 || completeness_value == 3)
+            return rpn_search_APT_cphrase (zi, zapt, attributeSet,
+                                           num_bases, basenames);
         return rpn_search_APT_phrase (zi, zapt, attributeSet,
                                       num_bases, basenames);
     case 1: /* phrase */
         if (relation_value == 102) /* relevance relation */
             return rpn_search_APT_relevance (zi, zapt, attributeSet,
                                              num_bases, basenames);
+        if (completeness_value == 2 || completeness_value == 3)
+            return rpn_search_APT_cphrase (zi, zapt, attributeSet,
+                                           num_bases, basenames);
         return rpn_search_APT_phrase (zi, zapt, attributeSet,
                                       num_bases, basenames);
         break;
@@ -1132,12 +1430,15 @@ static RSET rpn_search_structure (ZServerInfo *zi, Z_RPNStructure *zs,
     if (zs->which == Z_RPNStructure_complex)
     {
         rset_bool_parms bool_parms;
+        int soft = 0;
 
         bool_parms.rset_l = rpn_search_structure (zi, zs->u.complex->s1,
                                                   attributeSet,
                                                   num_bases, basenames);
         if (bool_parms.rset_l == NULL)
             return NULL;
+        if (rset_is_ranked(bool_parms.rset_l))
+            soft = 1;
         bool_parms.rset_r = rpn_search_structure (zi, zs->u.complex->s2,
                                                   attributeSet,
                                                   num_bases, basenames);
@@ -1146,19 +1447,21 @@ static RSET rpn_search_structure (ZServerInfo *zi, Z_RPNStructure *zs,
             rset_delete (bool_parms.rset_l);
             return NULL;
         }
+        if (rset_is_ranked(bool_parms.rset_r))
+            soft = 1;
         bool_parms.key_size = sizeof(struct it_key);
         bool_parms.cmp = key_compare;
 
-        switch (zs->u.complex->operator->which)
+        switch (zs->u.complex->roperator->which)
         {
         case Z_Operator_and:
-            r = rset_create (rset_kind_and, &bool_parms);
+            r = rset_create (soft ? rset_kind_sand:rset_kind_and, &bool_parms);
             break;
         case Z_Operator_or:
-            r = rset_create (rset_kind_or, &bool_parms);
+            r = rset_create (soft ? rset_kind_sor:rset_kind_or, &bool_parms);
             break;
         case Z_Operator_and_not:
-            r = rset_create (rset_kind_not, &bool_parms);
+            r = rset_create (soft ? rset_kind_snot:rset_kind_not, &bool_parms);
             break;
         default:
             assert (0);
@@ -1189,13 +1492,46 @@ static RSET rpn_search_structure (ZServerInfo *zi, Z_RPNStructure *zs,
     return r;
 }
 
+void count_set_save (RSET *r, int *count)
+{
+    int psysno = 0;
+    int kno = 0;
+    struct it_key key;
+    RSFD rfd, wfd;
+    RSET w;
+    rset_temp_parms parms;
+
+    logf (LOG_DEBUG, "count_set_save");
+    *count = 0;
+    parms.key_size = sizeof(struct it_key);
+    w = rset_create (rset_kind_temp, &parms);
+    wfd = rset_open (w, RSETF_WRITE|RSETF_SORT_SYSNO);
+    rfd = rset_open (*r, RSETF_READ|RSETF_SORT_SYSNO);
+    while (rset_read (*r, rfd, &key))
+    {
+        if (key.sysno != psysno)
+        {
+            rset_write (w, wfd, &key);
+            psysno = key.sysno;
+            (*count)++;
+        }
+        kno++;
+    }
+    rset_close (*r, rfd);
+    rset_delete (*r);
+    rset_close (w, wfd);
+    *r = w;
+    logf (LOG_DEBUG, "%d keys, %d distinct sysnos", kno, *count);
+}
+
 static void count_set (RSET r, int *count)
 {
     int psysno = 0;
+    int kno = 0;
     struct it_key key;
     RSFD rfd;
 
-    logf (LOG_DEBUG, "rpn_save_set");
+    logf (LOG_DEBUG, "count_set");
     *count = 0;
     rfd = rset_open (r, RSETF_READ|RSETF_SORT_SYSNO);
     while (rset_read (r, rfd, &key))
@@ -1205,9 +1541,10 @@ static void count_set (RSET r, int *count)
             psysno = key.sysno;
             (*count)++;
         }
+        kno++;
     }
     rset_close (r, rfd);
-    logf (LOG_DEBUG, "%d distinct sysnos", *count);
+    logf (LOG_DEBUG, "%d keys, %d distinct sysnos", kno, *count);
 }
 
 int rpn_search (ZServerInfo *zi,
@@ -1218,17 +1555,21 @@ int rpn_search (ZServerInfo *zi,
     oident *attrset;
     oid_value attributeSet;
 
+    zlog_rpn (rpn);
+
     zi->errCode = 0;
     zi->errString = NULL;
-    
+
     attrset = oid_getentbyoid (rpn->attributeSetId);
     attributeSet = attrset->value;
-
     rset = rpn_search_structure (zi, rpn->RPNStructure, attributeSet,
                                  num_bases, basenames);
     if (!rset)
         return zi->errCode;
-    count_set (rset, hits);
+    if (rset_is_volatile(rset))
+        count_set_save(&rset,hits);
+    else
+        count_set (rset, hits);
     resultSetAdd (zi, setname, 1, rset);
     if (zi->errCode)
         logf (LOG_DEBUG, "search error: %d", zi->errCode);
@@ -1243,8 +1584,7 @@ struct scan_info {
     char prefix[20];
 };
 
-static int scan_handle (Dict_char *name, const char *info, int pos, 
-                        void *client)
+static int scan_handle (char *name, const char *info, int pos, void *client)
 {
     int len_prefix, idx;
     ISAM_P isam_p;
@@ -1279,7 +1619,7 @@ static int scan_handle (Dict_char *name, const char *info, int pos,
 }
 
 
-static int dummy_handle (Dict_char *name, const char *info, void *p)
+static int dummy_handle (char *name, const char *info, void *p)
 {
     return 0;
 }
@@ -1323,7 +1663,7 @@ int rpn_scan (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
         use_value = 1016;
     i = index_word_prefix (termz, 1, use_value, *basenames);
 
-    dict_lookup_grep (zi->wordDict, termz, 0, NULL, &max_pos,
+    dict_lookup_grep (zi->wordDict, termz, 0, NULL, &max_pos, 0,
                       dummy_handle);
     if (max_pos <= strlen(*basenames))
     {