index/zrpn.c

   1 /* $Id: zrpn.c,v 1.174 2005-04-14 09:03:24 adam Exp $
   2    Copyright (C) 1995-2005
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with Zebra; see the file LICENSE.zebra.  If not, write to the
  19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  20 02111-1307, USA.
  21 */
  22
  23
  24 #include <stdio.h>
  25 #include <assert.h>
  26 #ifdef WIN32
  27 #include <io.h>
  28 #else
  29 #include <unistd.h>
  30 #endif
  31 #include <ctype.h>
  32
  33 #include "index.h"
  34 #include <zebra_xpath.h>
  35
  36 #include <charmap.h>
  37 #include <rset.h>
  38
  39 /* maximum number of terms in an and/or/phrase item */
  40 #define TERM_LIST_LENGTH_MAX 256
  41
  42 static const struct key_control it_ctrl =
  43 {
  44     sizeof(struct it_key),
  45     2, /* we have sysnos and seqnos in this key, nothing more */
  46     key_compare_it,
  47     key_logdump_txt,   /* FIXME  - clean up these functions */
  48     key_get_seq,
  49 };
  50
  51
  52 const struct key_control *key_it_ctrl = &it_ctrl;
  53
  54 struct rpn_char_map_info
  55 {
  56     ZebraMaps zm;
  57     int reg_type;
  58 };
  59
  60 typedef struct
  61 {
  62     int type;
  63     int major;
  64     int minor;
  65     Z_AttributesPlusTerm *zapt;
  66 } AttrType;
  67
  68
  69 static int log_level_set = 0;
  70 static int log_level_rpn = 0;
  71
  72 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  73 {
  74     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  75     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  76 #if 0
  77     if (out && *out)
  78     {
  79         const char *outp = *out;
  80         yaz_log(YLOG_LOG, "---");
  81         while (*outp)
  82         {
  83             yaz_log(YLOG_LOG, "%02X", *outp);
  84             outp++;
  85         }
  86     }
  87 #endif
  88     return out;
  89 }
  90
  91 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  92                                   struct rpn_char_map_info *map_info)
  93 {
  94     map_info->zm = reg->zebra_maps;
  95     map_info->reg_type = reg_type;
  96     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  97 }
  98
  99 static int attr_find_ex(AttrType *src, oid_value *attributeSetP,
 100                          const char **string_value)
 101 {
 102     int num_attributes;
 103
 104     num_attributes = src->zapt->attributes->num_attributes;
 105     while (src->major < num_attributes)
 106     {
 107         Z_AttributeElement *element;
 108
 109         element = src->zapt->attributes->attributes[src->major];
 110         if (src->type == *element->attributeType)
 111         {
 112             switch (element->which)
 113             {
 114             case Z_AttributeValue_numeric:
 115                 ++(src->major);
 116                 if (element->attributeSet && attributeSetP)
 117                 {
 118                     oident *attrset;
 119
 120                     attrset = oid_getentbyoid(element->attributeSet);
 121                     *attributeSetP = attrset->value;
 122                 }
 123                 return *element->value.numeric;
 124                 break;
 125             case Z_AttributeValue_complex:
 126                 if (src->minor >= element->value.complex->num_list)
 127                     break;
 128                 if (element->attributeSet && attributeSetP)
 129                 {
 130                     oident *attrset;
 131
 132                     attrset = oid_getentbyoid(element->attributeSet);
 133                     *attributeSetP = attrset->value;
 134                 }
 135                 if (element->value.complex->list[src->minor]->which ==
 136                     Z_StringOrNumeric_numeric)
 137                 {
 138                     ++(src->minor);
 139                     return
 140                         *element->value.complex->list[src->minor-1]->u.numeric;
 141                 }
 142                 else if (element->value.complex->list[src->minor]->which ==
 143                          Z_StringOrNumeric_string)
 144                 {
 145                     if (!string_value)
 146                         break;
 147                     ++(src->minor);
 148                     *string_value =
 149                         element->value.complex->list[src->minor-1]->u.string;
 150                     return -2;
 151                 }
 152                 else
 153                     break;
 154             default:
 155                 assert(0);
 156             }
 157         }
 158         ++(src->major);
 159     }
 160     return -1;
 161 }
 162
 163 static int attr_find(AttrType *src, oid_value *attributeSetP)
 164 {
 165     return attr_find_ex(src, attributeSetP, 0);
 166 }
 167
 168 static void attr_init(AttrType *src, Z_AttributesPlusTerm *zapt,
 169                        int type)
 170 {
 171     src->zapt = zapt;
 172     src->type = type;
 173     src->major = 0;
 174     src->minor = 0;
 175 }
 176
 177 #define TERM_COUNT
 178
 179 struct grep_info {
 180 #ifdef TERM_COUNT
 181     int *term_no;
 182 #endif
 183     ISAM_P *isam_p_buf;
 184     int isam_p_size;
 185     int isam_p_indx;
 186     ZebraHandle zh;
 187     int reg_type;
 188     ZebraSet termset;
 189 };
 190
 191 static void term_untrans(ZebraHandle zh, int reg_type,
 192                            char *dst, const char *src)
 193 {
 194     int len = 0;
 195     while (*src)
 196     {
 197         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
 198                                            reg_type, &src);
 199         if (!cp && len < IT_MAX_WORD-1)
 200             dst[len++] = *src++;
 201         else
 202             while (*cp && len < IT_MAX_WORD-1)
 203                 dst[len++] = *cp++;
 204     }
 205     dst[len] = '\0';
 206 }
 207
 208 static void add_isam_p(const char *name, const char *info,
 209                        struct grep_info *p)
 210 {
 211     if (!log_level_set)
 212     {
 213         log_level_rpn = yaz_log_module_level("rpn");
 214         log_level_set = 1;
 215     }
 216     if (p->isam_p_indx == p->isam_p_size)
 217     {
 218         ISAM_P *new_isam_p_buf;
 219 #ifdef TERM_COUNT
 220         int *new_term_no;
 221 #endif
 222         p->isam_p_size = 2*p->isam_p_size + 100;
 223         new_isam_p_buf = (ISAM_P *) xmalloc(sizeof(*new_isam_p_buf) *
 224                                             p->isam_p_size);
 225         if (p->isam_p_buf)
 226         {
 227             memcpy(new_isam_p_buf, p->isam_p_buf,
 228                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 229             xfree(p->isam_p_buf);
 230         }
 231         p->isam_p_buf = new_isam_p_buf;
 232
 233 #ifdef TERM_COUNT
 234         new_term_no = (int *) xmalloc(sizeof(*new_term_no) * p->isam_p_size);
 235         if (p->term_no)
 236         {
 237             memcpy(new_term_no, p->isam_p_buf,
 238                     p->isam_p_indx * sizeof(*p->term_no));
 239             xfree(p->term_no);
 240         }
 241         p->term_no = new_term_no;
 242 #endif
 243     }
 244     assert(*info == sizeof(*p->isam_p_buf));
 245     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 246
 247 #if 1
 248     if (p->termset)
 249     {
 250         const char *db;
 251         int set, use;
 252         char term_tmp[IT_MAX_WORD];
 253         int su_code = 0;
 254         int len = key_SU_decode (&su_code, name);
 255
 256         term_untrans  (p->zh, p->reg_type, term_tmp, name+len+1);
 257         yaz_log(log_level_rpn, "grep: %d %c %s", su_code, name[len], term_tmp);
 258         zebraExplain_lookup_ord (p->zh->reg->zei,
 259                                  su_code, &db, &set, &use);
 260         yaz_log(log_level_rpn, "grep:  set=%d use=%d db=%s", set, use, db);
 261
 262         resultSetAddTerm(p->zh, p->termset, name[len], db,
 263                          set, use, term_tmp);
 264     }
 265 #endif
 266     (p->isam_p_indx)++;
 267 }
 268
 269 static int grep_handle(char *name, const char *info, void *p)
 270 {
 271     add_isam_p(name, info, (struct grep_info *) p);
 272     return 0;
 273 }
 274
 275 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 276                      const char *ct1, const char *ct2, int first)
 277 {
 278     const char *s1, *s0 = *src;
 279     const char **map;
 280
 281     /* skip white space */
 282     while (*s0)
 283     {
 284         if (ct1 && strchr(ct1, *s0))
 285             break;
 286         if (ct2 && strchr(ct2, *s0))
 287             break;
 288         s1 = s0;
 289         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 290         if (**map != *CHR_SPACE)
 291             break;
 292         s0 = s1;
 293     }
 294     *src = s0;
 295     return *s0;
 296 }
 297
 298
 299 static void esc_str(char *out_buf, int out_size,
 300                     const char *in_buf, int in_size)
 301 {
 302     int k;
 303
 304     assert(out_buf);
 305     assert(in_buf);
 306     assert(out_size > 20);
 307     *out_buf = '\0';
 308     for (k = 0; k<in_size; k++)
 309     {
 310         int c = in_buf[k] & 0xff;
 311         int pc;
 312         if (c < 32 || c > 126)
 313             pc = '?';
 314         else
 315             pc = c;
 316         sprintf(out_buf +strlen(out_buf), "%02X:%c  ", c, pc);
 317         if (strlen(out_buf) > out_size-20)
 318         {
 319             strcat(out_buf, "..");
 320             break;
 321         }
 322     }
 323 }
 324
 325 #define REGEX_CHARS " []()|.*+?!"
 326
 327 /* term_100: handle term, where trunc = none(no operators at all) */
 328 static int term_100(ZebraMaps zebra_maps, int reg_type,
 329                      const char **src, char *dst, int space_split,
 330                      char *dst_term)
 331 {
 332     const char *s0;
 333     const char **map;
 334     int i = 0;
 335     int j = 0;
 336
 337     const char *space_start = 0;
 338     const char *space_end = 0;
 339
 340     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 341         return 0;
 342     s0 = *src;
 343     while (*s0)
 344     {
 345         const char *s1 = s0;
 346         int q_map_match = 0;
 347         map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 348                                 &q_map_match);
 349         if (space_split)
 350         {
 351             if (**map == *CHR_SPACE)
 352                 break;
 353         }
 354         else  /* complete subfield only. */
 355         {
 356             if (**map == *CHR_SPACE)
 357             {   /* save space mapping for later  .. */
 358                 space_start = s1;
 359                 space_end = s0;
 360                 continue;
 361             }
 362             else if (space_start)
 363             {   /* reload last space */
 364                 while (space_start < space_end)
 365                 {
 366                     if (strchr(REGEX_CHARS, *space_start))
 367                         dst[i++] = '\\';
 368                     dst_term[j++] = *space_start;
 369                     dst[i++] = *space_start++;
 370                 }
 371                 /* and reset */
 372                 space_start = space_end = 0;
 373             }
 374         }
 375         /* add non-space char */
 376         memcpy(dst_term+j, s1, s0 - s1);
 377         j += (s0 - s1);
 378         if (!q_map_match)
 379         {
 380             while (s1 < s0)
 381             {
 382                 if (strchr(REGEX_CHARS, *s1))
 383                     dst[i++] = '\\';
 384                 dst[i++] = *s1++;
 385             }
 386         }
 387         else
 388         {
 389             char tmpbuf[80];
 390             esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 391
 392             strcpy(dst + i, map[0]);
 393             i += strlen(map[0]);
 394         }
 395     }
 396     dst[i] = '\0';
 397     dst_term[j] = '\0';
 398     *src = s0;
 399     return i;
 400 }
 401
 402 /* term_101: handle term, where trunc = Process # */
 403 static int term_101(ZebraMaps zebra_maps, int reg_type,
 404                      const char **src, char *dst, int space_split,
 405                      char *dst_term)
 406 {
 407     const char *s0;
 408     const char **map;
 409     int i = 0;
 410     int j = 0;
 411
 412     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 413         return 0;
 414     s0 = *src;
 415     while (*s0)
 416     {
 417         if (*s0 == '#')
 418         {
 419             dst[i++] = '.';
 420             dst[i++] = '*';
 421             dst_term[j++] = *s0++;
 422         }
 423         else
 424         {
 425             const char *s1 = s0;
 426             int q_map_match = 0;
 427             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 428                                     &q_map_match);
 429             if (space_split && **map == *CHR_SPACE)
 430                 break;
 431
 432             /* add non-space char */
 433             memcpy(dst_term+j, s1, s0 - s1);
 434             j += (s0 - s1);
 435             if (!q_map_match)
 436             {
 437                 while (s1 < s0)
 438                 {
 439                     if (strchr(REGEX_CHARS, *s1))
 440                         dst[i++] = '\\';
 441                     dst[i++] = *s1++;
 442                 }
 443             }
 444             else
 445             {
 446                 char tmpbuf[80];
 447                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 448
 449                 strcpy(dst + i, map[0]);
 450                 i += strlen(map[0]);
 451             }
 452         }
 453     }
 454     dst[i] = '\0';
 455     dst_term[j++] = '\0';
 456     *src = s0;
 457     return i;
 458 }
 459
 460 /* term_103: handle term, where trunc = re-2 (regular expressions) */
 461 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 462                      char *dst, int *errors, int space_split,
 463                      char *dst_term)
 464 {
 465     int i = 0;
 466     int j = 0;
 467     const char *s0;
 468     const char **map;
 469
 470     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 471         return 0;
 472     s0 = *src;
 473     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 474         isdigit(((const unsigned char *)s0)[1]))
 475     {
 476         *errors = s0[1] - '0';
 477         s0 += 3;
 478         if (*errors > 3)
 479             *errors = 3;
 480     }
 481     while (*s0)
 482     {
 483         if (strchr("^\\()[].*+?|-", *s0))
 484         {
 485             dst_term[j++] = *s0;
 486             dst[i++] = *s0++;
 487         }
 488         else
 489         {
 490             const char *s1 = s0;
 491             int q_map_match = 0;
 492             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 493                                     &q_map_match);
 494             if (space_split && **map == *CHR_SPACE)
 495                 break;
 496
 497             /* add non-space char */
 498             memcpy(dst_term+j, s1, s0 - s1);
 499             j += (s0 - s1);
 500             if (!q_map_match)
 501             {
 502                 while (s1 < s0)
 503                 {
 504                     if (strchr(REGEX_CHARS, *s1))
 505                         dst[i++] = '\\';
 506                     dst[i++] = *s1++;
 507                 }
 508             }
 509             else
 510             {
 511                 char tmpbuf[80];
 512                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 513
 514                 strcpy(dst + i, map[0]);
 515                 i += strlen(map[0]);
 516             }
 517         }
 518     }
 519     dst[i] = '\0';
 520     dst_term[j] = '\0';
 521     *src = s0;
 522
 523     return i;
 524 }
 525
 526 /* term_103: handle term, where trunc = re-1 (regular expressions) */
 527 static int term_102 (ZebraMaps zebra_maps, int reg_type, const char **src,
 528                      char *dst, int space_split, char *dst_term)
 529 {
 530     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 531                      dst_term);
 532 }
 533
 534
 535 /* term_104: handle term, where trunc = Process # and ! */
 536 static int term_104(ZebraMaps zebra_maps, int reg_type,
 537                      const char **src, char *dst, int space_split,
 538                      char *dst_term)
 539 {
 540     const char *s0, *s1;
 541     const char **map;
 542     int i = 0;
 543     int j = 0;
 544
 545     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 546         return 0;
 547     s0 = *src;
 548     while (*s0)
 549     {
 550         if (*s0 == '?')
 551         {
 552             dst_term[j++] = *s0++;
 553             if (*s0 >= '0' && *s0 <= '9')
 554             {
 555                 int limit = 0;
 556                 while (*s0 >= '0' && *s0 <= '9')
 557                 {
 558                     limit = limit * 10 + (*s0 - '0');
 559                     dst_term[j++] = *s0++;
 560                 }
 561                 if (limit > 20)
 562                     limit = 20;
 563                 while (--limit >= 0)
 564                 {
 565                     dst[i++] = '.';
 566                     dst[i++] = '?';
 567                 }
 568             }
 569             else
 570             {
 571                 dst[i++] = '.';
 572                 dst[i++] = '*';
 573             }
 574         }
 575         else if (*s0 == '*')
 576         {
 577             dst[i++] = '.';
 578             dst[i++] = '*';
 579             dst_term[j++] = *s0++;
 580         }
 581         else if (*s0 == '#')
 582         {
 583             dst[i++] = '.';
 584             dst_term[j++] = *s0++;
 585         }
 586         else
 587         {
 588             s1 = s0;
 589             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 590             if (space_split && **map == *CHR_SPACE)
 591                 break;
 592             while (s1 < s0)
 593             {
 594                 if (strchr(REGEX_CHARS, *s1))
 595                     dst[i++] = '\\';
 596                 dst_term[j++] = *s1;
 597                 dst[i++] = *s1++;
 598             }
 599         }
 600     }
 601     dst[i] = '\0';
 602     dst_term[j++] = '\0';
 603     *src = s0;
 604     return i;
 605 }
 606
 607 /* term_105/106: handle term, where trunc = Process * and ! and right trunc */
 608 static int term_105 (ZebraMaps zebra_maps, int reg_type,
 609                      const char **src, char *dst, int space_split,
 610                      char *dst_term, int right_truncate)
 611 {
 612     const char *s0, *s1;
 613     const char **map;
 614     int i = 0;
 615     int j = 0;
 616
 617     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 618         return 0;
 619     s0 = *src;
 620     while (*s0)
 621     {
 622         if (*s0 == '*')
 623         {
 624             dst[i++] = '.';
 625             dst[i++] = '*';
 626             dst_term[j++] = *s0++;
 627         }
 628         else if (*s0 == '!')
 629         {
 630             dst[i++] = '.';
 631             dst_term[j++] = *s0++;
 632         }
 633         else
 634         {
 635             s1 = s0;
 636             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 637             if (space_split && **map == *CHR_SPACE)
 638                 break;
 639             while (s1 < s0)
 640             {
 641                 if (strchr(REGEX_CHARS, *s1))
 642                     dst[i++] = '\\';
 643                 dst_term[j++] = *s1;
 644                 dst[i++] = *s1++;
 645             }
 646         }
 647     }
 648     if (right_truncate)
 649     {
 650         dst[i++] = '.';
 651         dst[i++] = '*';
 652     }
 653     dst[i] = '\0';
 654
 655     dst_term[j++] = '\0';
 656     *src = s0;
 657     return i;
 658 }
 659
 660
 661 /* gen_regular_rel - generate regular expression from relation
 662  *  val:     border value (inclusive)
 663  *  islt:    1 if <=; 0 if >=.
 664  */
 665 static void gen_regular_rel(char *dst, int val, int islt)
 666 {
 667     int dst_p;
 668     int w, d, i;
 669     int pos = 0;
 670     char numstr[20];
 671
 672     yaz_log(YLOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 673     if (val >= 0)
 674     {
 675         if (islt)
 676             strcpy(dst, "(-[0-9]+|(");
 677         else
 678             strcpy(dst, "((");
 679     }
 680     else
 681     {
 682         if (!islt)
 683         {
 684             strcpy(dst, "([0-9]+|-(");
 685             dst_p = strlen(dst);
 686             islt = 1;
 687         }
 688         else
 689         {
 690             strcpy(dst, "(-(");
 691             islt = 0;
 692         }
 693         val = -val;
 694     }
 695     dst_p = strlen(dst);
 696     sprintf(numstr, "%d", val);
 697     for (w = strlen(numstr); --w >= 0; pos++)
 698     {
 699         d = numstr[w];
 700         if (pos > 0)
 701         {
 702             if (islt)
 703             {
 704                 if (d == '0')
 705                     continue;
 706                 d--;
 707             }
 708             else
 709             {
 710                 if (d == '9')
 711                     continue;
 712                 d++;
 713             }
 714         }
 715
 716         strcpy(dst + dst_p, numstr);
 717         dst_p = strlen(dst) - pos - 1;
 718
 719         if (islt)
 720         {
 721             if (d != '0')
 722             {
 723                 dst[dst_p++] = '[';
 724                 dst[dst_p++] = '0';
 725                 dst[dst_p++] = '-';
 726                 dst[dst_p++] = d;
 727                 dst[dst_p++] = ']';
 728             }
 729             else
 730                 dst[dst_p++] = d;
 731         }
 732         else
 733         {
 734             if (d != '9')
 735             {
 736                 dst[dst_p++] = '[';
 737                 dst[dst_p++] = d;
 738                 dst[dst_p++] = '-';
 739                 dst[dst_p++] = '9';
 740                 dst[dst_p++] = ']';
 741             }
 742             else
 743                 dst[dst_p++] = d;
 744         }
 745         for (i = 0; i<pos; i++)
 746         {
 747             dst[dst_p++] = '[';
 748             dst[dst_p++] = '0';
 749             dst[dst_p++] = '-';
 750             dst[dst_p++] = '9';
 751             dst[dst_p++] = ']';
 752         }
 753         dst[dst_p++] = '|';
 754     }
 755     dst[dst_p] = '\0';
 756     if (islt)
 757     {
 758         /* match everything less than 10^(pos-1) */
 759         strcat(dst, "0*");
 760         for (i = 1; i<pos; i++)
 761             strcat(dst, "[0-9]?");
 762     }
 763     else
 764     {
 765         /* match everything greater than 10^pos */
 766         for (i = 0; i <= pos; i++)
 767             strcat(dst, "[0-9]");
 768         strcat(dst, "[0-9]*");
 769     }
 770     strcat(dst, "))");
 771 }
 772
 773 void string_rel_add_char(char **term_p, const char *src, int *indx)
 774 {
 775     if (src[*indx] == '\\')
 776         *(*term_p)++ = src[(*indx)++];
 777     *(*term_p)++ = src[(*indx)++];
 778 }
 779
 780 /*
 781  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 782  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 783  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 784  *              ([^-a].*|a[^-b].*|ab[c-].*)
 785  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 786  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 787  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 788  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 789  */
 790 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 791                             const char **term_sub, char *term_dict,
 792                             oid_value attributeSet,
 793                             int reg_type, int space_split, char *term_dst)
 794 {
 795     AttrType relation;
 796     int relation_value;
 797     int i;
 798     char *term_tmp = term_dict + strlen(term_dict);
 799     char term_component[2*IT_MAX_WORD+20];
 800
 801     attr_init(&relation, zapt, 2);
 802     relation_value = attr_find(&relation, NULL);
 803
 804     yaz_log(YLOG_DEBUG, "string relation value=%d", relation_value);
 805     switch (relation_value)
 806     {
 807     case 1:
 808         if (!term_100 (zh->reg->zebra_maps, reg_type,
 809                        term_sub, term_component,
 810                        space_split, term_dst))
 811             return 0;
 812         yaz_log(log_level_rpn, "Relation <");
 813
 814         *term_tmp++ = '(';
 815         for (i = 0; term_component[i]; )
 816         {
 817             int j = 0;
 818
 819             if (i)
 820                 *term_tmp++ = '|';
 821             while (j < i)
 822                 string_rel_add_char (&term_tmp, term_component, &j);
 823
 824             *term_tmp++ = '[';
 825
 826             *term_tmp++ = '^';
 827             string_rel_add_char (&term_tmp, term_component, &i);
 828             *term_tmp++ = '-';
 829
 830             *term_tmp++ = ']';
 831             *term_tmp++ = '.';
 832             *term_tmp++ = '*';
 833
 834             if ((term_tmp - term_dict) > IT_MAX_WORD)
 835                 break;
 836         }
 837         *term_tmp++ = ')';
 838         *term_tmp = '\0';
 839         break;
 840     case 2:
 841         if (!term_100 (zh->reg->zebra_maps, reg_type,
 842                        term_sub, term_component,
 843                        space_split, term_dst))
 844             return 0;
 845         yaz_log(log_level_rpn, "Relation <=");
 846
 847         *term_tmp++ = '(';
 848         for (i = 0; term_component[i]; )
 849         {
 850             int j = 0;
 851
 852             while (j < i)
 853                 string_rel_add_char (&term_tmp, term_component, &j);
 854             *term_tmp++ = '[';
 855
 856             *term_tmp++ = '^';
 857             string_rel_add_char (&term_tmp, term_component, &i);
 858             *term_tmp++ = '-';
 859
 860             *term_tmp++ = ']';
 861             *term_tmp++ = '.';
 862             *term_tmp++ = '*';
 863
 864             *term_tmp++ = '|';
 865
 866             if ((term_tmp - term_dict) > IT_MAX_WORD)
 867                 break;
 868         }
 869         for (i = 0; term_component[i]; )
 870             string_rel_add_char (&term_tmp, term_component, &i);
 871         *term_tmp++ = ')';
 872         *term_tmp = '\0';
 873         break;
 874     case 5:
 875         if (!term_100 (zh->reg->zebra_maps, reg_type,
 876                        term_sub, term_component, space_split, term_dst))
 877             return 0;
 878         yaz_log(log_level_rpn, "Relation >");
 879
 880         *term_tmp++ = '(';
 881         for (i = 0; term_component[i];)
 882         {
 883             int j = 0;
 884
 885             while (j < i)
 886                 string_rel_add_char (&term_tmp, term_component, &j);
 887             *term_tmp++ = '[';
 888
 889             *term_tmp++ = '^';
 890             *term_tmp++ = '-';
 891             string_rel_add_char (&term_tmp, term_component, &i);
 892
 893             *term_tmp++ = ']';
 894             *term_tmp++ = '.';
 895             *term_tmp++ = '*';
 896
 897             *term_tmp++ = '|';
 898
 899             if ((term_tmp - term_dict) > IT_MAX_WORD)
 900                 break;
 901         }
 902         for (i = 0; term_component[i];)
 903             string_rel_add_char (&term_tmp, term_component, &i);
 904         *term_tmp++ = '.';
 905         *term_tmp++ = '+';
 906         *term_tmp++ = ')';
 907         *term_tmp = '\0';
 908         break;
 909     case 4:
 910         if (!term_100 (zh->reg->zebra_maps, reg_type, term_sub,
 911                        term_component, space_split, term_dst))
 912             return 0;
 913         yaz_log(log_level_rpn, "Relation >=");
 914
 915         *term_tmp++ = '(';
 916         for (i = 0; term_component[i];)
 917         {
 918             int j = 0;
 919
 920             if (i)
 921                 *term_tmp++ = '|';
 922             while (j < i)
 923                 string_rel_add_char (&term_tmp, term_component, &j);
 924             *term_tmp++ = '[';
 925
 926             if (term_component[i+1])
 927             {
 928                 *term_tmp++ = '^';
 929                 *term_tmp++ = '-';
 930                 string_rel_add_char (&term_tmp, term_component, &i);
 931             }
 932             else
 933             {
 934                 string_rel_add_char (&term_tmp, term_component, &i);
 935                 *term_tmp++ = '-';
 936             }
 937             *term_tmp++ = ']';
 938             *term_tmp++ = '.';
 939             *term_tmp++ = '*';
 940
 941             if ((term_tmp - term_dict) > IT_MAX_WORD)
 942                 break;
 943         }
 944         *term_tmp++ = ')';
 945         *term_tmp = '\0';
 946         break;
 947     case 3:
 948     default:
 949         yaz_log(log_level_rpn, "Relation =");
 950         if (!term_100 (zh->reg->zebra_maps, reg_type, term_sub,
 951                        term_component, space_split, term_dst))
 952             return 0;
 953         strcat(term_tmp, "(");
 954         strcat(term_tmp, term_component);
 955         strcat(term_tmp, ")");
 956     }
 957     return 1;
 958 }
 959
 960 static int string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 961                         const char **term_sub,
 962                         oid_value attributeSet, NMEM stream,
 963                         struct grep_info *grep_info,
 964                         int reg_type, int complete_flag,
 965                         int num_bases, char **basenames,
 966                         char *term_dst, int xpath_use);
 967
 968 static RSET term_trunc(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 969                         const char **term_sub,
 970                         oid_value attributeSet, NMEM stream,
 971                         struct grep_info *grep_info,
 972                         int reg_type, int complete_flag,
 973                         int num_bases, char **basenames,
 974                         char *term_dst,
 975                         const char *rank_type, int xpath_use,
 976                         NMEM rset_nmem)
 977 {
 978     int r;
 979     grep_info->isam_p_indx = 0;
 980     r = string_term(zh, zapt, term_sub, attributeSet, stream, grep_info,
 981                      reg_type, complete_flag, num_bases, basenames,
 982                      term_dst, xpath_use);
 983     if (r < 1)
 984         return 0;
 985     yaz_log(log_level_rpn, "term: %s", term_dst);
 986     return rset_trunc(zh, grep_info->isam_p_buf,
 987                        grep_info->isam_p_indx, term_dst,
 988                        strlen(term_dst), rank_type, 1 /* preserve pos */,
 989                        zapt->term->which, rset_nmem,
 990                        key_it_ctrl,key_it_ctrl->scope);
 991 }
 992 static char *nmem_strdup_i(NMEM nmem, int v)
 993 {
 994     char val_str[64];
 995     sprintf (val_str, "%d", v);
 996     return nmem_strdup(nmem, val_str);
 997 }
 998
 999 static int string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1000                        const char **term_sub,
1001                        oid_value attributeSet, NMEM stream,
1002                        struct grep_info *grep_info,
1003                        int reg_type, int complete_flag,
1004                        int num_bases, char **basenames,
1005                        char *term_dst, int xpath_use)
1006 {
1007     char term_dict[2*IT_MAX_WORD+4000];
1008     int j, r, base_no;
1009     AttrType truncation;
1010     int truncation_value;
1011     AttrType use;
1012     int use_value;
1013     const char *use_string = 0;
1014     oid_value curAttributeSet = attributeSet;
1015     const char *termp;
1016     struct rpn_char_map_info rcmi;
1017     int space_split = complete_flag ? 0 : 1;
1018
1019     int bases_ok = 0;     /* no of databases with OK attribute */
1020     int errCode = 0;      /* err code (if any is not OK) */
1021     char *errString = 0;  /* addinfo */
1022
1023     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1024     attr_init (&use, zapt, 1);
1025     use_value = attr_find_ex (&use, &curAttributeSet, &use_string);
1026     yaz_log(log_level_rpn, "string_term, use value %d", use_value);
1027     attr_init (&truncation, zapt, 5);
1028     truncation_value = attr_find (&truncation, NULL);
1029     yaz_log(log_level_rpn, "truncation value %d", truncation_value);
1030
1031     if (use_value == -1)    /* no attribute - assumy "any" */
1032         use_value = 1016;
1033     for (base_no = 0; base_no < num_bases; base_no++)
1034     {
1035         int ord = -1;
1036         int attr_ok = 0;
1037         int regex_range = 0;
1038         int init_pos = 0;
1039         attent attp;
1040         data1_local_attribute id_xpath_attr;
1041         data1_local_attribute *local_attr;
1042         int max_pos, prefix_len = 0;
1043
1044         termp = *term_sub;
1045
1046         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1047         {
1048             zh->errCode = 109; /* Database unavailable */
1049             zh->errString = basenames[base_no];
1050             return -1;
1051         }
1052         if (xpath_use > 0 && use_value == -2)
1053         {
1054             /* xpath mode and we have a string attribute */
1055             attp.local_attributes = &id_xpath_attr;
1056             attp.attset_ordinal = VAL_IDXPATH;
1057             id_xpath_attr.next = 0;
1058
1059             use_value = xpath_use;  /* xpath_use as use-attribute now */
1060             id_xpath_attr.local = use_value;
1061         }
1062         else if (curAttributeSet == VAL_IDXPATH && use_value >= 0)
1063         {
1064             /* X-Path attribute, use numeric value directly */
1065             attp.local_attributes = &id_xpath_attr;
1066             attp.attset_ordinal = VAL_IDXPATH;
1067             id_xpath_attr.next = 0;
1068             id_xpath_attr.local = use_value;
1069         }
1070         else if (use_string &&
1071                  (ord = zebraExplain_lookup_attr_str(zh->reg->zei,
1072                                                      use_string)) >= 0)
1073         {
1074             /* we have a match for a raw string attribute */
1075             char ord_buf[32];
1076             int i, ord_len;
1077
1078             if (prefix_len)
1079                 term_dict[prefix_len++] = '|';
1080             else
1081                 term_dict[prefix_len++] = '(';
1082
1083             ord_len = key_SU_encode (ord, ord_buf);
1084             for (i = 0; i<ord_len; i++)
1085             {
1086                 term_dict[prefix_len++] = 1;
1087                 term_dict[prefix_len++] = ord_buf[i];
1088             }
1089             attp.local_attributes = 0;  /* no more attributes */
1090         }
1091         else
1092         {
1093             /* lookup in the .att files . Allow string as well */
1094             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1095                                       use_string)))
1096             {
1097                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1098                         curAttributeSet, use_value, r);
1099                 if (r == -1)
1100                 {
1101                     /* set was found, but value wasn't defined */
1102                     errCode = 114;
1103                     if (use_string)
1104                         errString = nmem_strdup(stream, use_string);
1105                     else
1106                         errString = nmem_strdup_i (stream, use_value);
1107                 }
1108                 else
1109                 {
1110                     int oid[OID_SIZE];
1111                     struct oident oident;
1112
1113                     oident.proto = PROTO_Z3950;
1114                     oident.oclass = CLASS_ATTSET;
1115                     oident.value = curAttributeSet;
1116                     oid_ent_to_oid (&oident, oid);
1117
1118                     errCode = 121;
1119                     errString = nmem_strdup (stream, oident.desc);
1120                 }
1121                 continue;
1122             }
1123         }
1124         for (local_attr = attp.local_attributes; local_attr;
1125              local_attr = local_attr->next)
1126         {
1127             char ord_buf[32];
1128             int i, ord_len;
1129
1130             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1131                                               attp.attset_ordinal,
1132                                               local_attr->local);
1133             if (ord < 0)
1134                 continue;
1135             if (prefix_len)
1136                 term_dict[prefix_len++] = '|';
1137             else
1138                 term_dict[prefix_len++] = '(';
1139
1140             ord_len = key_SU_encode (ord, ord_buf);
1141             for (i = 0; i<ord_len; i++)
1142             {
1143                 term_dict[prefix_len++] = 1;
1144                 term_dict[prefix_len++] = ord_buf[i];
1145             }
1146         }
1147         if (!prefix_len)
1148         {
1149 #if 1
1150             bases_ok++;
1151 #else
1152             errCode = 114;
1153             errString = nmem_strdup_i(stream, use_value);
1154             continue;
1155 #endif
1156         }
1157         else
1158         {
1159             bases_ok++; /* this has OK attributes */
1160             attr_ok = 1;
1161         }
1162
1163         term_dict[prefix_len++] = ')';
1164         term_dict[prefix_len++] = 1;
1165         term_dict[prefix_len++] = reg_type;
1166         yaz_log(log_level_rpn, "reg_type = %d", term_dict[prefix_len-1]);
1167         term_dict[prefix_len] = '\0';
1168         j = prefix_len;
1169         switch (truncation_value)
1170         {
1171         case -1:         /* not specified */
1172         case 100:        /* do not truncate */
1173             if (!string_relation (zh, zapt, &termp, term_dict,
1174                                   attributeSet,
1175                                   reg_type, space_split, term_dst))
1176                 return 0;
1177             break;
1178         case 1:          /* right truncation */
1179             term_dict[j++] = '(';
1180             if (!term_100(zh->reg->zebra_maps, reg_type,
1181                           &termp, term_dict + j, space_split, term_dst))
1182                 return 0;
1183             strcat(term_dict, ".*)");
1184             break;
1185         case 2:          /* keft truncation */
1186             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1187             if (!term_100(zh->reg->zebra_maps, reg_type,
1188                           &termp, term_dict + j, space_split, term_dst))
1189                 return 0;
1190             strcat(term_dict, ")");
1191             break;
1192         case 3:          /* left&right truncation */
1193             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1194             if (!term_100(zh->reg->zebra_maps, reg_type,
1195                           &termp, term_dict + j, space_split, term_dst))
1196                 return 0;
1197             strcat(term_dict, ".*)");
1198             break;
1199         case 101:        /* process # in term */
1200             term_dict[j++] = '(';
1201             if (!term_101(zh->reg->zebra_maps, reg_type,
1202                           &termp, term_dict + j, space_split, term_dst))
1203                 return 0;
1204             strcat(term_dict, ")");
1205             break;
1206         case 102:        /* Regexp-1 */
1207             term_dict[j++] = '(';
1208             if (!term_102(zh->reg->zebra_maps, reg_type,
1209                           &termp, term_dict + j, space_split, term_dst))
1210                 return 0;
1211             strcat(term_dict, ")");
1212             break;
1213         case 103:       /* Regexp-2 */
1214             r = 1;
1215             term_dict[j++] = '(';
1216             init_pos = 2;
1217             if (!term_103 (zh->reg->zebra_maps, reg_type,
1218                            &termp, term_dict + j, &regex_range,
1219                            space_split, term_dst))
1220                 return 0;
1221             strcat(term_dict, ")");
1222         case 104:        /* process # and ! in term */
1223             term_dict[j++] = '(';
1224             if (!term_104 (zh->reg->zebra_maps, reg_type,
1225                            &termp, term_dict + j, space_split, term_dst))
1226                 return 0;
1227             strcat(term_dict, ")");
1228             break;
1229         case 105:        /* process * and ! in term */
1230             term_dict[j++] = '(';
1231             if (!term_105 (zh->reg->zebra_maps, reg_type,
1232                            &termp, term_dict + j, space_split, term_dst, 1))
1233                 return 0;
1234             strcat(term_dict, ")");
1235             break;
1236         case 106:        /* process * and ! in term */
1237             term_dict[j++] = '(';
1238             if (!term_105 (zh->reg->zebra_maps, reg_type,
1239                            &termp, term_dict + j, space_split, term_dst, 0))
1240                 return 0;
1241             strcat(term_dict, ")");
1242             break;
1243         default:
1244             zh->errCode = 120;
1245             zh->errString = nmem_strdup_i(stream, truncation_value);
1246             return -1;
1247         }
1248         if (attr_ok)
1249         {
1250             char buf[80];
1251             const char *input = term_dict + prefix_len;
1252             esc_str(buf, sizeof(buf), input, strlen(input));
1253         }
1254         if (attr_ok)
1255         {
1256             yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_dict+prefix_len);
1257             r = dict_lookup_grep(zh->reg->dict, term_dict, regex_range,
1258                                  grep_info, &max_pos, init_pos,
1259                                  grep_handle);
1260             if (r)
1261                 yaz_log(YLOG_WARN, "dict_lookup_grep fail %d", r);
1262         }
1263     }
1264     if (!bases_ok)
1265     {
1266         zh->errCode = errCode;
1267         zh->errString = errString;
1268         return -1;
1269     }
1270     *term_sub = termp;
1271     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1272     return 1;
1273 }
1274
1275
1276 /* convert APT search term to UTF8 */
1277 static int zapt_term_to_utf8 (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1278                               char *termz)
1279 {
1280     size_t sizez;
1281     Z_Term *term = zapt->term;
1282
1283     switch (term->which)
1284     {
1285     case Z_Term_general:
1286         if (zh->iconv_to_utf8 != 0)
1287         {
1288             char *inbuf = term->u.general->buf;
1289             size_t inleft = term->u.general->len;
1290             char *outbuf = termz;
1291             size_t outleft = IT_MAX_WORD-1;
1292             size_t ret;
1293
1294             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1295                         &outbuf, &outleft);
1296             if (ret == (size_t)(-1))
1297             {
1298                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1299                 zh->errCode = 125;
1300                 return -1;
1301             }
1302             *outbuf = 0;
1303         }
1304         else
1305         {
1306             sizez = term->u.general->len;
1307             if (sizez > IT_MAX_WORD-1)
1308                 sizez = IT_MAX_WORD-1;
1309             memcpy (termz, term->u.general->buf, sizez);
1310             termz[sizez] = '\0';
1311         }
1312         break;
1313     case Z_Term_characterString:
1314         sizez = strlen(term->u.characterString);
1315         if (sizez > IT_MAX_WORD-1)
1316             sizez = IT_MAX_WORD-1;
1317         memcpy (termz, term->u.characterString, sizez);
1318         termz[sizez] = '\0';
1319         break;
1320     default:
1321         zh->errCode = 124;
1322         return -1;
1323     }
1324     return 0;
1325 }
1326
1327 /* convert APT SCAN term to internal cmap */
1328 static int trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1329                             char *termz, int reg_type)
1330 {
1331     char termz0[IT_MAX_WORD];
1332
1333     if (zapt_term_to_utf8(zh, zapt, termz0))
1334         return -1;    /* error */
1335     else
1336     {
1337         const char **map;
1338         const char *cp = (const char *) termz0;
1339         const char *cp_end = cp + strlen(cp);
1340         const char *src;
1341         int i = 0;
1342         const char *space_map = NULL;
1343         int len;
1344
1345         while ((len = (cp_end - cp)) > 0)
1346         {
1347             map = zebra_maps_input (zh->reg->zebra_maps, reg_type, &cp, len, 0);
1348             if (**map == *CHR_SPACE)
1349                 space_map = *map;
1350             else
1351             {
1352                 if (i && space_map)
1353                     for (src = space_map; *src; src++)
1354                         termz[i++] = *src;
1355                 space_map = NULL;
1356                 for (src = *map; *src; src++)
1357                     termz[i++] = *src;
1358             }
1359         }
1360         termz[i] = '\0';
1361     }
1362     return 0;
1363 }
1364
1365 char *normalize_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1366                      const char *termz, NMEM stream, unsigned reg_id)
1367 {
1368     WRBUF wrbuf = 0;
1369     AttrType truncation;
1370     int truncation_value;
1371     char *ex_list = 0;
1372
1373     attr_init (&truncation, zapt, 5);
1374     truncation_value = attr_find (&truncation, NULL);
1375
1376     switch (truncation_value)
1377     {
1378     default:
1379         ex_list = "";
1380         break;
1381     case 101:
1382         ex_list = "#";
1383         break;
1384     case 102:
1385     case 103:
1386         ex_list = 0;
1387         break;
1388     case 104:
1389         ex_list = "!#";
1390         break;
1391     case 105:
1392         ex_list = "!*";
1393         break;
1394     }
1395     if (ex_list)
1396         wrbuf = zebra_replace(zh->reg->zebra_maps, reg_id, ex_list,
1397                               termz, strlen(termz));
1398     if (!wrbuf)
1399         return nmem_strdup(stream, termz);
1400     else
1401     {
1402         char *buf = (char*) nmem_malloc(stream, wrbuf_len(wrbuf)+1);
1403         memcpy (buf, wrbuf_buf(wrbuf), wrbuf_len(wrbuf));
1404         buf[wrbuf_len(wrbuf)] = '\0';
1405         return buf;
1406     }
1407 }
1408
1409 static void grep_info_delete (struct grep_info *grep_info)
1410 {
1411 #ifdef TERM_COUNT
1412     xfree(grep_info->term_no);
1413 #endif
1414     xfree (grep_info->isam_p_buf);
1415 }
1416
1417 static int grep_info_prepare (ZebraHandle zh,
1418                               Z_AttributesPlusTerm *zapt,
1419                               struct grep_info *grep_info,
1420                               int reg_type,
1421                               NMEM stream)
1422 {
1423     AttrType termset;
1424     int termset_value_numeric;
1425     const char *termset_value_string;
1426
1427 #ifdef TERM_COUNT
1428     grep_info->term_no = 0;
1429 #endif
1430     grep_info->isam_p_size = 0;
1431     grep_info->isam_p_buf = NULL;
1432     grep_info->zh = zh;
1433     grep_info->reg_type = reg_type;
1434     grep_info->termset = 0;
1435
1436     if (!zapt)
1437         return 0;
1438     attr_init (&termset, zapt, 8);
1439     termset_value_numeric =
1440         attr_find_ex (&termset, NULL, &termset_value_string);
1441     if (termset_value_numeric != -1)
1442     {
1443         char resname[32];
1444         const char *termset_name = 0;
1445         if (termset_value_numeric != -2)
1446         {
1447
1448             sprintf (resname, "%d", termset_value_numeric);
1449             termset_name = resname;
1450         }
1451         else
1452             termset_name = termset_value_string;
1453         yaz_log(log_level_rpn, "creating termset set %s", termset_name);
1454         grep_info->termset = resultSetAdd (zh, termset_name, 1);
1455         if (!grep_info->termset)
1456         {
1457             zh->errCode = 128;
1458             zh->errString = nmem_strdup (stream, termset_name);
1459             return -1;
1460         }
1461     }
1462     return 0;
1463 }
1464
1465
1466 static RSET rpn_search_APT_phrase (ZebraHandle zh,
1467                                    Z_AttributesPlusTerm *zapt,
1468                                    const char *termz_org,
1469                                    oid_value attributeSet,
1470                                    NMEM stream,
1471                                    int reg_type, int complete_flag,
1472                                    const char *rank_type, int xpath_use,
1473                                    int num_bases, char **basenames,
1474                                    NMEM rset_nmem)
1475 {
1476     char term_dst[IT_MAX_WORD+1];
1477     RSET rset[TERM_LIST_LENGTH_MAX], result;
1478     size_t rset_no = 0;
1479     struct grep_info grep_info;
1480     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1481     const char *termp = termz;
1482
1483     *term_dst = 0;
1484     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1485         return 0;
1486     for (; rset_no < sizeof(rset)/sizeof(*rset); rset_no++)
1487     {
1488         yaz_log(log_level_rpn, "APT_phrase termp=%s", termp);
1489         rset[rset_no] = term_trunc(zh, zapt, &termp, attributeSet,
1490                                     stream, &grep_info,
1491                                     reg_type, complete_flag,
1492                                     num_bases, basenames,
1493                                     term_dst, rank_type,
1494                                     xpath_use,rset_nmem);
1495         if (!rset[rset_no])
1496             break;
1497     }
1498     grep_info_delete (&grep_info);
1499     if (rset_no == 0)
1500         return rsnull_create (rset_nmem,key_it_ctrl);
1501     else if (rset_no == 1)
1502         return (rset[0]);
1503     else
1504         result = rsprox_create( rset_nmem, key_it_ctrl, key_it_ctrl->scope,
1505                        rset_no, rset,
1506                        1 /* ordered */, 0 /* exclusion */,
1507                        3 /* relation */, 1 /* distance */);
1508     return result;
1509 }
1510
1511 static RSET rpn_search_APT_or_list (ZebraHandle zh,
1512                                     Z_AttributesPlusTerm *zapt,
1513                                     const char *termz_org,
1514                                     oid_value attributeSet,
1515                                     NMEM stream,
1516                                     int reg_type, int complete_flag,
1517                                     const char *rank_type,
1518                                     int xpath_use,
1519                                     int num_bases, char **basenames,
1520                                     NMEM rset_nmem)
1521 {
1522     char term_dst[IT_MAX_WORD+1];
1523     RSET rset[TERM_LIST_LENGTH_MAX];
1524     size_t rset_no = 0;
1525     struct grep_info grep_info;
1526     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1527     const char *termp = termz;
1528
1529     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1530         return 0;
1531     for (; rset_no < sizeof(rset)/sizeof(*rset); rset_no++)
1532     {
1533         yaz_log(log_level_rpn, "APT_or_list termp=%s", termp);
1534         rset[rset_no] = term_trunc(zh, zapt, &termp, attributeSet,
1535                                     stream, &grep_info,
1536                                     reg_type, complete_flag,
1537                                     num_bases, basenames,
1538                                     term_dst, rank_type,
1539                                     xpath_use,rset_nmem);
1540         if (!rset[rset_no])
1541             break;
1542     }
1543     grep_info_delete (&grep_info);
1544     if (rset_no == 0)
1545         return rsnull_create (rset_nmem,key_it_ctrl);
1546     return rsmulti_or_create(rset_nmem, key_it_ctrl,key_it_ctrl->scope,
1547                              rset_no, rset);
1548 }
1549
1550 static RSET rpn_search_APT_and_list (ZebraHandle zh,
1551                                      Z_AttributesPlusTerm *zapt,
1552                                      const char *termz_org,
1553                                      oid_value attributeSet,
1554                                      NMEM stream,
1555                                      int reg_type, int complete_flag,
1556                                      const char *rank_type,
1557                                      int xpath_use,
1558                                      int num_bases, char **basenames,
1559                                      NMEM rset_nmem)
1560 {
1561     char term_dst[IT_MAX_WORD+1];
1562     RSET rset[TERM_LIST_LENGTH_MAX];
1563     size_t rset_no = 0;
1564     struct grep_info grep_info;
1565     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1566     const char *termp = termz;
1567
1568     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1569         return 0;
1570     for (; rset_no < sizeof(rset)/sizeof(*rset); rset_no++)
1571     {
1572         yaz_log(log_level_rpn, "APT_and_list termp=%s", termp);
1573         rset[rset_no] = term_trunc(zh, zapt, &termp, attributeSet,
1574                                     stream, &grep_info,
1575                                     reg_type, complete_flag,
1576                                     num_bases, basenames,
1577                                     term_dst, rank_type,
1578                                     xpath_use, rset_nmem);
1579         if (!rset[rset_no])
1580             break;
1581     }
1582     grep_info_delete (&grep_info);
1583     if (rset_no == 0)
1584         return rsnull_create(rset_nmem,key_it_ctrl);
1585
1586     return rsmulti_and_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
1587                               rset_no, rset);
1588 }
1589
1590 static int numeric_relation (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1591                              const char **term_sub,
1592                              char *term_dict,
1593                              oid_value attributeSet,
1594                              struct grep_info *grep_info,
1595                              int *max_pos,
1596                              int reg_type,
1597                              char *term_dst)
1598 {
1599     AttrType relation;
1600     int relation_value;
1601     int term_value;
1602     int r;
1603     char *term_tmp = term_dict + strlen(term_dict);
1604
1605     attr_init (&relation, zapt, 2);
1606     relation_value = attr_find (&relation, NULL);
1607
1608     yaz_log(log_level_rpn, "numeric relation value=%d", relation_value);
1609
1610     if (!term_100 (zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1611                    term_dst))
1612         return 0;
1613     term_value = atoi (term_tmp);
1614     switch (relation_value)
1615     {
1616     case 1:
1617         yaz_log(log_level_rpn, "Relation <");
1618         gen_regular_rel (term_tmp, term_value-1, 1);
1619         break;
1620     case 2:
1621         yaz_log(log_level_rpn, "Relation <=");
1622         gen_regular_rel (term_tmp, term_value, 1);
1623         break;
1624     case 4:
1625         yaz_log(log_level_rpn, "Relation >=");
1626         gen_regular_rel (term_tmp, term_value, 0);
1627         break;
1628     case 5:
1629         yaz_log(log_level_rpn, "Relation >");
1630         gen_regular_rel (term_tmp, term_value+1, 0);
1631         break;
1632     case 3:
1633     default:
1634         yaz_log(log_level_rpn, "Relation =");
1635         sprintf (term_tmp, "(0*%d)", term_value);
1636     }
1637     yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_tmp);
1638     r = dict_lookup_grep(zh->reg->dict, term_dict, 0, grep_info, max_pos,
1639                           0, grep_handle);
1640     if (r)
1641         yaz_log(YLOG_WARN, "dict_lookup_grep fail, rel = gt: %d", r);
1642     yaz_log(log_level_rpn, "%d positions", grep_info->isam_p_indx);
1643     return 1;
1644 }
1645
1646 static int numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1647                          const char **term_sub,
1648                          oid_value attributeSet, struct grep_info *grep_info,
1649                          int reg_type, int complete_flag,
1650                          int num_bases, char **basenames,
1651                          char *term_dst, int xpath_use, NMEM stream)
1652 {
1653     char term_dict[2*IT_MAX_WORD+2];
1654     int r, base_no;
1655     AttrType use;
1656     int use_value;
1657     const char *use_string = 0;
1658     oid_value curAttributeSet = attributeSet;
1659     const char *termp;
1660     struct rpn_char_map_info rcmi;
1661
1662     int bases_ok = 0;     /* no of databases with OK attribute */
1663     int errCode = 0;      /* err code (if any is not OK) */
1664     char *errString = 0;  /* addinfo */
1665
1666     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1667     attr_init (&use, zapt, 1);
1668     use_value = attr_find_ex (&use, &curAttributeSet, &use_string);
1669
1670     if (use_value == -1)
1671         use_value = 1016;
1672
1673     for (base_no = 0; base_no < num_bases; base_no++)
1674     {
1675         attent attp;
1676         data1_local_attribute id_xpath_attr;
1677         data1_local_attribute *local_attr;
1678         int max_pos, prefix_len = 0;
1679
1680         termp = *term_sub;
1681         if (use_value == -2)  /* string attribute (assume IDXPATH/any) */
1682         {
1683             use_value = xpath_use;
1684             attp.local_attributes = &id_xpath_attr;
1685             attp.attset_ordinal = VAL_IDXPATH;
1686             id_xpath_attr.next = 0;
1687             id_xpath_attr.local = use_value;
1688         }
1689         else if (curAttributeSet == VAL_IDXPATH)
1690         {
1691             attp.local_attributes = &id_xpath_attr;
1692             attp.attset_ordinal = VAL_IDXPATH;
1693             id_xpath_attr.next = 0;
1694             id_xpath_attr.local = use_value;
1695         }
1696         else
1697         {
1698             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1699                                             use_string)))
1700             {
1701                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1702                       curAttributeSet, use_value, r);
1703                 if (r == -1)
1704                 {
1705                     errCode = 114;
1706                     if (use_string)
1707                         errString = nmem_strdup(stream, use_string);
1708                     else
1709                         errString = nmem_strdup_i (stream, use_value);
1710                 }
1711                 else
1712                     errCode = 121;
1713                 continue;
1714             }
1715         }
1716         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1717         {
1718             zh->errCode = 109; /* Database unavailable */
1719             zh->errString = basenames[base_no];
1720             return -1;
1721         }
1722         for (local_attr = attp.local_attributes; local_attr;
1723              local_attr = local_attr->next)
1724         {
1725             int ord;
1726             char ord_buf[32];
1727             int i, ord_len;
1728
1729             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1730                                               attp.attset_ordinal,
1731                                               local_attr->local);
1732             if (ord < 0)
1733                 continue;
1734             if (prefix_len)
1735                 term_dict[prefix_len++] = '|';
1736             else
1737                 term_dict[prefix_len++] = '(';
1738
1739             ord_len = key_SU_encode (ord, ord_buf);
1740             for (i = 0; i<ord_len; i++)
1741             {
1742                 term_dict[prefix_len++] = 1;
1743                 term_dict[prefix_len++] = ord_buf[i];
1744             }
1745         }
1746         if (!prefix_len)
1747         {
1748             errCode = 114;
1749             errString = nmem_strdup_i(stream, use_value);
1750             continue;
1751         }
1752         bases_ok++;
1753         term_dict[prefix_len++] = ')';
1754         term_dict[prefix_len++] = 1;
1755         term_dict[prefix_len++] = reg_type;
1756         yaz_log(YLOG_DEBUG, "reg_type = %d", term_dict[prefix_len-1]);
1757         term_dict[prefix_len] = '\0';
1758         if (!numeric_relation (zh, zapt, &termp, term_dict,
1759                                attributeSet, grep_info, &max_pos, reg_type,
1760                                term_dst))
1761             return 0;
1762     }
1763     if (!bases_ok)
1764     {
1765         zh->errCode = errCode;
1766         zh->errString = errString;
1767         return -1;
1768     }
1769     *term_sub = termp;
1770     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1771     return 1;
1772 }
1773
1774 static RSET rpn_search_APT_numeric (ZebraHandle zh,
1775                                     Z_AttributesPlusTerm *zapt,
1776                                     const char *termz,
1777                                     oid_value attributeSet,
1778                                     NMEM stream,
1779                                     int reg_type, int complete_flag,
1780                                     const char *rank_type, int xpath_use,
1781                                     int num_bases, char **basenames,
1782                                     NMEM rset_nmem)
1783 {
1784     char term_dst[IT_MAX_WORD+1];
1785     const char *termp = termz;
1786     RSET rset[TERM_LIST_LENGTH_MAX];
1787     int  r;
1788     size_t rset_no = 0;
1789     struct grep_info grep_info;
1790
1791     yaz_log(log_level_rpn, "APT_numeric t='%s'",termz);
1792     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1793         return 0;
1794     for (; rset_no < sizeof(rset)/sizeof(*rset); rset_no++)
1795     {
1796         yaz_log(YLOG_DEBUG, "APT_numeric termp=%s", termp);
1797         grep_info.isam_p_indx = 0;
1798         r = numeric_term(zh, zapt, &termp, attributeSet, &grep_info,
1799                           reg_type, complete_flag, num_bases, basenames,
1800                           term_dst, xpath_use,
1801                           stream);
1802         if (r < 1)
1803             break;
1804         yaz_log(YLOG_DEBUG, "term: %s", term_dst);
1805         rset[rset_no] = rset_trunc(zh, grep_info.isam_p_buf,
1806                                     grep_info.isam_p_indx, term_dst,
1807                                     strlen(term_dst), rank_type,
1808                                     0 /* preserve position */,
1809                                     zapt->term->which, rset_nmem,
1810                                     key_it_ctrl,key_it_ctrl->scope);
1811         if (!rset[rset_no])
1812             break;
1813     }
1814     grep_info_delete (&grep_info);
1815     if (rset_no == 0)
1816         return rsnull_create(rset_nmem,key_it_ctrl);
1817     if (rset_no == 1)
1818         return rset[0];
1819     return rsmulti_and_create(rset_nmem,key_it_ctrl,key_it_ctrl->scope,
1820                               rset_no, rset);
1821 }
1822
1823 static RSET rpn_search_APT_local (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1824                                   const char *termz,
1825                                   oid_value attributeSet,
1826                                   NMEM stream,
1827                                   const char *rank_type, NMEM rset_nmem)
1828 {
1829     RSET result;
1830     RSFD rsfd;
1831     struct it_key key;
1832     int sys;
1833     result = rstemp_create( rset_nmem,key_it_ctrl,key_it_ctrl->scope,
1834                      res_get (zh->res, "setTmpDir"),0 );
1835     rsfd = rset_open (result, RSETF_WRITE);
1836
1837     sys = atoi(termz);
1838     if (sys <= 0)
1839         sys = 1;
1840     key.mem[0] = sys;
1841     key.mem[1] = 1;
1842     key.len = 2;
1843     rset_write (rsfd, &key);
1844     rset_close (rsfd);
1845     return result;
1846 }
1847
1848 static RSET rpn_sort_spec (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1849                            oid_value attributeSet, NMEM stream,
1850                            Z_SortKeySpecList *sort_sequence,
1851                            const char *rank_type)
1852 {
1853     int i;
1854     int sort_relation_value;
1855     AttrType sort_relation_type;
1856     int use_value;
1857     AttrType use_type;
1858     Z_SortKeySpec *sks;
1859     Z_SortKey *sk;
1860     Z_AttributeElement *ae;
1861     int oid[OID_SIZE];
1862     oident oe;
1863     char termz[20];
1864
1865     attr_init (&sort_relation_type, zapt, 7);
1866     sort_relation_value = attr_find (&sort_relation_type, &attributeSet);
1867
1868     attr_init (&use_type, zapt, 1);
1869     use_value = attr_find (&use_type, &attributeSet);
1870
1871     if (!sort_sequence->specs)
1872     {
1873         sort_sequence->num_specs = 10;
1874         sort_sequence->specs = (Z_SortKeySpec **)
1875             nmem_malloc(stream, sort_sequence->num_specs *
1876                          sizeof(*sort_sequence->specs));
1877         for (i = 0; i<sort_sequence->num_specs; i++)
1878             sort_sequence->specs[i] = 0;
1879     }
1880     if (zapt->term->which != Z_Term_general)
1881         i = 0;
1882     else
1883         i = atoi_n ((char *) zapt->term->u.general->buf,
1884                     zapt->term->u.general->len);
1885     if (i >= sort_sequence->num_specs)
1886         i = 0;
1887     sprintf (termz, "%d", i);
1888
1889     oe.proto = PROTO_Z3950;
1890     oe.oclass = CLASS_ATTSET;
1891     oe.value = attributeSet;
1892     if (!oid_ent_to_oid (&oe, oid))
1893         return 0;
1894
1895     sks = (Z_SortKeySpec *) nmem_malloc(stream, sizeof(*sks));
1896     sks->sortElement = (Z_SortElement *)
1897         nmem_malloc(stream, sizeof(*sks->sortElement));
1898     sks->sortElement->which = Z_SortElement_generic;
1899     sk = sks->sortElement->u.generic = (Z_SortKey *)
1900         nmem_malloc(stream, sizeof(*sk));
1901     sk->which = Z_SortKey_sortAttributes;
1902     sk->u.sortAttributes = (Z_SortAttributes *)
1903         nmem_malloc(stream, sizeof(*sk->u.sortAttributes));
1904
1905     sk->u.sortAttributes->id = oid;
1906     sk->u.sortAttributes->list = (Z_AttributeList *)
1907         nmem_malloc(stream, sizeof(*sk->u.sortAttributes->list));
1908     sk->u.sortAttributes->list->num_attributes = 1;
1909     sk->u.sortAttributes->list->attributes = (Z_AttributeElement **)
1910         nmem_malloc(stream, sizeof(*sk->u.sortAttributes->list->attributes));
1911     ae = *sk->u.sortAttributes->list->attributes = (Z_AttributeElement *)
1912         nmem_malloc(stream, sizeof(**sk->u.sortAttributes->list->attributes));
1913     ae->attributeSet = 0;
1914     ae->attributeType = (int *)
1915         nmem_malloc(stream, sizeof(*ae->attributeType));
1916     *ae->attributeType = 1;
1917     ae->which = Z_AttributeValue_numeric;
1918     ae->value.numeric = (int *)
1919         nmem_malloc(stream, sizeof(*ae->value.numeric));
1920     *ae->value.numeric = use_value;
1921
1922     sks->sortRelation = (int *)
1923         nmem_malloc(stream, sizeof(*sks->sortRelation));
1924     if (sort_relation_value == 1)
1925         *sks->sortRelation = Z_SortKeySpec_ascending;
1926     else if (sort_relation_value == 2)
1927         *sks->sortRelation = Z_SortKeySpec_descending;
1928     else
1929         *sks->sortRelation = Z_SortKeySpec_ascending;
1930
1931     sks->caseSensitivity = (int *)
1932         nmem_malloc(stream, sizeof(*sks->caseSensitivity));
1933     *sks->caseSensitivity = 0;
1934
1935     sks->which = Z_SortKeySpec_null;
1936     sks->u.null = odr_nullval ();
1937     sort_sequence->specs[i] = sks;
1938     return rsnull_create (NULL,key_it_ctrl);
1939         /* FIXME - nmem?? */
1940 }
1941
1942
1943 static int parse_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1944                        oid_value attributeSet,
1945                        struct xpath_location_step *xpath, int max, NMEM mem)
1946 {
1947     oid_value curAttributeSet = attributeSet;
1948     AttrType use;
1949     const char *use_string = 0;
1950
1951     attr_init (&use, zapt, 1);
1952     attr_find_ex (&use, &curAttributeSet, &use_string);
1953
1954     if (!use_string || *use_string != '/')
1955         return -1;
1956
1957     return zebra_parse_xpath_str(use_string, xpath, max, mem);
1958 }
1959
1960
1961
1962 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
1963                         int reg_type, const char *term, int use,
1964                         oid_value curAttributeSet, NMEM rset_nmem)
1965 {
1966     RSET rset;
1967     struct grep_info grep_info;
1968     char term_dict[2048];
1969     char ord_buf[32];
1970     int prefix_len = 0;
1971     int ord = zebraExplain_lookup_attr_su(zh->reg->zei, curAttributeSet, use);
1972     int ord_len, i, r, max_pos;
1973     int term_type = Z_Term_characterString;
1974     const char *flags = "void";
1975
1976     if (grep_info_prepare (zh, 0 /* zapt */, &grep_info, '0', stream))
1977         return rsnull_create (rset_nmem,key_it_ctrl);
1978
1979     if (ord < 0)
1980         return rsnull_create (rset_nmem,key_it_ctrl);
1981     if (prefix_len)
1982         term_dict[prefix_len++] = '|';
1983     else
1984         term_dict[prefix_len++] = '(';
1985
1986     ord_len = key_SU_encode (ord, ord_buf);
1987     for (i = 0; i<ord_len; i++)
1988     {
1989         term_dict[prefix_len++] = 1;
1990         term_dict[prefix_len++] = ord_buf[i];
1991     }
1992     term_dict[prefix_len++] = ')';
1993     term_dict[prefix_len++] = 1;
1994     term_dict[prefix_len++] = reg_type;
1995
1996     strcpy(term_dict+prefix_len, term);
1997
1998     grep_info.isam_p_indx = 0;
1999     r = dict_lookup_grep(zh->reg->dict, term_dict, 0,
2000                           &grep_info, &max_pos, 0, grep_handle);
2001     yaz_log (YLOG_DEBUG, "%s %d positions", term,
2002              grep_info.isam_p_indx);
2003     rset = rset_trunc(zh, grep_info.isam_p_buf,
2004                        grep_info.isam_p_indx, term, strlen(term),
2005                        flags, 1, term_type,rset_nmem,
2006                        key_it_ctrl, key_it_ctrl->scope);
2007     grep_info_delete (&grep_info);
2008     return rset;
2009 }
2010
2011 static RSET rpn_search_xpath (ZebraHandle zh,
2012                               oid_value attributeSet,
2013                               int num_bases, char **basenames,
2014                               NMEM stream, const char *rank_type, RSET rset,
2015                               int xpath_len, struct xpath_location_step *xpath,
2016                               NMEM rset_nmem)
2017 {
2018     oid_value curAttributeSet = attributeSet;
2019     int base_no;
2020     int i;
2021
2022     if (xpath_len < 0)
2023         return rset;
2024
2025     yaz_log (YLOG_DEBUG, "xpath len=%d", xpath_len);
2026     for (i = 0; i<xpath_len; i++)
2027     {
2028         yaz_log (log_level_rpn, "XPATH %d %s", i, xpath[i].part);
2029
2030     }
2031
2032     curAttributeSet = VAL_IDXPATH;
2033
2034     /*
2035       //a    ->    a/.*
2036       //a/b  ->    b/a/.*
2037       /a     ->    a/
2038       /a/b   ->    b/a/
2039
2040       /      ->    none
2041
2042    a[@attr = value]/b[@other = othervalue]
2043
2044  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
2045  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
2046  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
2047  /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y)
2048  /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y)
2049  /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x)
2050
2051     */
2052
2053     dict_grep_cmap (zh->reg->dict, 0, 0);
2054
2055     for (base_no = 0; base_no < num_bases; base_no++)
2056     {
2057         int level = xpath_len;
2058         int first_path = 1;
2059
2060         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2061         {
2062             zh->errCode = 109; /* Database unavailable */
2063             zh->errString = basenames[base_no];
2064             return rset;
2065         }
2066         while (--level >= 0)
2067         {
2068             char xpath_rev[128];
2069             int i, len;
2070             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
2071
2072             *xpath_rev = 0;
2073             len = 0;
2074             for (i = level; i >= 1; --i)
2075             {
2076                 const char *cp = xpath[i].part;
2077                 if (*cp)
2078                 {
2079                     for (;*cp; cp++)
2080                         if (*cp == '*')
2081                         {
2082                             memcpy (xpath_rev + len, "[^/]*", 5);
2083                             len += 5;
2084                         }
2085                         else if (*cp == ' ')
2086                         {
2087
2088                             xpath_rev[len++] = 1;
2089                             xpath_rev[len++] = ' ';
2090                         }
2091
2092                         else
2093                             xpath_rev[len++] = *cp;
2094                     xpath_rev[len++] = '/';
2095                 }
2096                 else if (i == 1)  /* // case */
2097                 {
2098                     xpath_rev[len++] = '.';
2099                     xpath_rev[len++] = '*';
2100                 }
2101             }
2102             xpath_rev[len] = 0;
2103
2104             if (xpath[level].predicate &&
2105                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
2106                 xpath[level].predicate->u.relation.name[0])
2107             {
2108                 WRBUF wbuf = wrbuf_alloc();
2109                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
2110                 if (xpath[level].predicate->u.relation.value)
2111                 {
2112                     const char *cp = xpath[level].predicate->u.relation.value;
2113                     wrbuf_putc(wbuf, '=');
2114
2115                     while (*cp)
2116                     {
2117                         if (strchr(REGEX_CHARS, *cp))
2118                             wrbuf_putc(wbuf, '\\');
2119                         wrbuf_putc(wbuf, *cp);
2120                         cp++;
2121                     }
2122                 }
2123                 wrbuf_puts(wbuf, "");
2124                 rset_attr = xpath_trunc(
2125                     zh, stream, '0', wrbuf_buf(wbuf), 3,
2126                     curAttributeSet,rset_nmem);
2127                 wrbuf_free(wbuf, 1);
2128             }
2129             else
2130             {
2131                 if (!first_path)
2132                     continue;
2133             }
2134             yaz_log (log_level_rpn, "xpath_rev (%d) = %s", level, xpath_rev);
2135             if (strlen(xpath_rev))
2136             {
2137                 rset_start_tag = xpath_trunc(zh, stream, '0',
2138                         xpath_rev, 1, curAttributeSet, rset_nmem);
2139
2140                 rset_end_tag = xpath_trunc(zh, stream, '0',
2141                         xpath_rev, 2, curAttributeSet, rset_nmem);
2142
2143                 rset = rsbetween_create(rset_nmem, key_it_ctrl,
2144                                         key_it_ctrl->scope,
2145                                         rset_start_tag, rset,
2146                                         rset_end_tag, rset_attr);
2147             }
2148             first_path = 0;
2149         }
2150     }
2151
2152     return rset;
2153 }
2154
2155
2156
2157 static RSET rpn_search_APT (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2158                             oid_value attributeSet, NMEM stream,
2159                             Z_SortKeySpecList *sort_sequence,
2160                             int num_bases, char **basenames,
2161                             NMEM rset_nmem)
2162 {
2163     unsigned reg_id;
2164     char *search_type = NULL;
2165     char rank_type[128];
2166     int complete_flag;
2167     int sort_flag;
2168     char termz[IT_MAX_WORD+1];
2169     RSET rset = 0;
2170     int xpath_len;
2171     int xpath_use = 0;
2172     struct xpath_location_step xpath[10];
2173
2174     if (!log_level_set)
2175     {
2176         log_level_rpn = yaz_log_module_level("rpn");
2177         log_level_set = 1;
2178     }
2179     zebra_maps_attr (zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2180                      rank_type, &complete_flag, &sort_flag);
2181
2182     yaz_log(YLOG_DEBUG, "reg_id=%c", reg_id);
2183     yaz_log(YLOG_DEBUG, "complete_flag=%d", complete_flag);
2184     yaz_log(YLOG_DEBUG, "search_type=%s", search_type);
2185     yaz_log(YLOG_DEBUG, "rank_type=%s", rank_type);
2186
2187     if (zapt_term_to_utf8(zh, zapt, termz))
2188         return 0;
2189
2190     if (sort_flag)
2191         return rpn_sort_spec (zh, zapt, attributeSet, stream, sort_sequence,
2192                               rank_type);
2193     xpath_len = parse_xpath(zh, zapt, attributeSet, xpath, 10, stream);
2194     if (xpath_len >= 0)
2195     {
2196         xpath_use = 1016;
2197         if (xpath[xpath_len-1].part[0] == '@')
2198             xpath_use = 1015;
2199     }
2200
2201     if (!strcmp (search_type, "phrase"))
2202     {
2203         rset = rpn_search_APT_phrase (zh, zapt, termz, attributeSet, stream,
2204                                       reg_id, complete_flag, rank_type,
2205                                       xpath_use,
2206                                       num_bases, basenames, rset_nmem);
2207     }
2208     else if (!strcmp (search_type, "and-list"))
2209     {
2210         rset = rpn_search_APT_and_list (zh, zapt, termz, attributeSet, stream,
2211                                         reg_id, complete_flag, rank_type,
2212                                         xpath_use,
2213                                         num_bases, basenames, rset_nmem);
2214     }
2215     else if (!strcmp (search_type, "or-list"))
2216     {
2217         rset = rpn_search_APT_or_list (zh, zapt, termz, attributeSet, stream,
2218                                        reg_id, complete_flag, rank_type,
2219                                        xpath_use,
2220                                        num_bases, basenames, rset_nmem);
2221     }
2222     else if (!strcmp (search_type, "local"))
2223     {
2224         rset = rpn_search_APT_local (zh, zapt, termz, attributeSet, stream,
2225                                      rank_type, rset_nmem);
2226     }
2227     else if (!strcmp (search_type, "numeric"))
2228     {
2229         rset = rpn_search_APT_numeric (zh, zapt, termz, attributeSet, stream,
2230                                        reg_id, complete_flag, rank_type,
2231                                        xpath_use,
2232                                        num_bases, basenames, rset_nmem);
2233     }
2234     else if (!strcmp (search_type, "always"))
2235     {
2236         rset = 0;
2237     }
2238     else
2239         zh->errCode = 118;
2240     return rpn_search_xpath (zh, attributeSet, num_bases, basenames,
2241                              stream, rank_type, rset,
2242                              xpath_len, xpath, rset_nmem);
2243 }
2244
2245 static RSET rpn_search_structure (ZebraHandle zh, Z_RPNStructure *zs,
2246                                   oid_value attributeSet,
2247                                   NMEM stream, NMEM rset_nmem,
2248                                   Z_SortKeySpecList *sort_sequence,
2249                                   int num_bases, char **basenames)
2250 {
2251     RSET r = NULL;
2252     if (zs->which == Z_RPNStructure_complex)
2253     {
2254         Z_Operator *zop = zs->u.complex->roperator;
2255         RSET rsets[2]; /* l and r argument */
2256
2257         rsets[0]=rpn_search_structure (zh, zs->u.complex->s1,
2258                                        attributeSet, stream, rset_nmem,
2259                                        sort_sequence,
2260                                        num_bases, basenames);
2261         if (rsets[0] == NULL)
2262             return NULL;
2263         rsets[1]=rpn_search_structure (zh, zs->u.complex->s2,
2264                                        attributeSet, stream, rset_nmem,
2265                                        sort_sequence,
2266                                        num_bases, basenames);
2267         if (rsets[1] == NULL)
2268         {
2269             rset_delete (rsets[0]);
2270             return NULL;
2271         }
2272
2273         switch (zop->which)
2274         {
2275         case Z_Operator_and:
2276             r = rsmulti_and_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
2277                                    2, rsets);
2278             break;
2279         case Z_Operator_or:
2280             r = rsmulti_or_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
2281                                   2, rsets);
2282             break;
2283         case Z_Operator_and_not:
2284             r = rsbool_create_not(rset_nmem,key_it_ctrl, key_it_ctrl->scope,
2285                     rsets[0],rsets[1]);
2286             break;
2287         case Z_Operator_prox:
2288             if (zop->u.prox->which != Z_ProximityOperator_known)
2289             {
2290                 zh->errCode = 132;
2291                 return NULL;
2292             }
2293             if (*zop->u.prox->u.known != Z_ProxUnit_word)
2294             {
2295                 char *val = (char *) nmem_malloc(stream, 16);
2296                 zh->errCode = 132;
2297                 zh->errString = val;
2298                 sprintf (val, "%d", *zop->u.prox->u.known);
2299                 return NULL;
2300             }
2301             else
2302             {
2303                 /* new / old prox */
2304                 r = rsprox_create(rset_nmem,key_it_ctrl,key_it_ctrl->scope,
2305                          2, rsets,
2306                          *zop->u.prox->ordered,
2307                          (!zop->u.prox->exclusion ?
2308                               0 : *zop->u.prox->exclusion),
2309                          *zop->u.prox->relationType,
2310                          *zop->u.prox->distance );
2311             }
2312             break;
2313         default:
2314             zh->errCode = 110;
2315             return NULL;
2316         }
2317     }
2318     else if (zs->which == Z_RPNStructure_simple)
2319     {
2320         if (zs->u.simple->which == Z_Operand_APT)
2321         {
2322             yaz_log(YLOG_DEBUG, "rpn_search_APT");
2323             r = rpn_search_APT (zh, zs->u.simple->u.attributesPlusTerm,
2324                                 attributeSet, stream, sort_sequence,
2325                                 num_bases, basenames,rset_nmem);
2326         }
2327         else if (zs->u.simple->which == Z_Operand_resultSetId)
2328         {
2329             yaz_log(YLOG_DEBUG, "rpn_search_ref");
2330             r = resultSetRef (zh, zs->u.simple->u.resultSetId);
2331             if (!r)
2332             {
2333                 r = rsnull_create (rset_nmem,key_it_ctrl);
2334                 zh->errCode = 30;
2335                 zh->errString =
2336                     nmem_strdup (stream, zs->u.simple->u.resultSetId);
2337                 return 0;
2338             }
2339             else
2340                 rset_dup(r);
2341         }
2342         else
2343         {
2344             zh->errCode = 3;
2345             return 0;
2346         }
2347     }
2348     else
2349     {
2350         zh->errCode = 3;
2351         return 0;
2352     }
2353     return r;
2354 }
2355
2356
2357 RSET rpn_search(ZebraHandle zh, NMEM nmem, NMEM rset_nmem,
2358                 Z_RPNQuery *rpn, int num_bases, char **basenames,
2359                 const char *setname,
2360                 ZebraSet sset)
2361 {
2362     RSET rset;
2363     oident *attrset;
2364     oid_value attributeSet;
2365     Z_SortKeySpecList *sort_sequence;
2366     int sort_status, i;
2367
2368     zh->errCode = 0;
2369     zh->errString = NULL;
2370     zh->hits = 0;
2371
2372     sort_sequence = (Z_SortKeySpecList *)
2373         nmem_malloc(nmem, sizeof(*sort_sequence));
2374     sort_sequence->num_specs = 10; /* FIXME - Hard-coded number */
2375     sort_sequence->specs = (Z_SortKeySpec **)
2376         nmem_malloc(nmem, sort_sequence->num_specs *
2377                      sizeof(*sort_sequence->specs));
2378     for (i = 0; i<sort_sequence->num_specs; i++)
2379         sort_sequence->specs[i] = 0;
2380
2381     attrset = oid_getentbyoid (rpn->attributeSetId);
2382     attributeSet = attrset->value;
2383     rset = rpn_search_structure (zh, rpn->RPNStructure, attributeSet,
2384                                  nmem, rset_nmem,
2385                                  sort_sequence, num_bases, basenames);
2386     if (!rset)
2387         return 0;
2388
2389     if (zh->errCode)
2390         yaz_log(YLOG_DEBUG, "search error: %d", zh->errCode);
2391
2392     for (i = 0; sort_sequence->specs[i]; i++)
2393         ;
2394     sort_sequence->num_specs = i;
2395     if (!i)
2396         resultSetRank (zh, sset, rset, rset_nmem);
2397     else
2398     {
2399         yaz_log(YLOG_DEBUG, "resultSetSortSingle in rpn_search");
2400         resultSetSortSingle (zh, nmem, sset, rset,
2401                              sort_sequence, &sort_status);
2402         if (zh->errCode)
2403         {
2404             yaz_log(YLOG_DEBUG, "resultSetSortSingle status = %d", zh->errCode);
2405         }
2406     }
2407     return rset;
2408 }
2409
2410 struct scan_info_entry {
2411     char *term;
2412     ISAM_P isam_p;
2413 };
2414
2415 struct scan_info {
2416     struct scan_info_entry *list;
2417     ODR odr;
2418     int before, after;
2419     char prefix[20];
2420 };
2421
2422 static int scan_handle (char *name, const char *info, int pos, void *client)
2423 {
2424     int len_prefix, idx;
2425     struct scan_info *scan_info = (struct scan_info *) client;
2426
2427     len_prefix = strlen(scan_info->prefix);
2428     if (memcmp (name, scan_info->prefix, len_prefix))
2429         return 1;
2430     if (pos > 0)        idx = scan_info->after - pos + scan_info->before;
2431     else
2432         idx = - pos - 1;
2433     scan_info->list[idx].term = (char *)
2434         odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
2435     strcpy(scan_info->list[idx].term, name + len_prefix);
2436     assert (*info == sizeof(ISAM_P));
2437     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
2438     return 0;
2439 }
2440
2441 static void scan_term_untrans (ZebraHandle zh, NMEM stream, int reg_type,
2442                                char **dst, const char *src)
2443 {
2444     char term_src[IT_MAX_WORD];
2445     char term_dst[IT_MAX_WORD];
2446
2447     term_untrans (zh, reg_type, term_src, src);
2448
2449     if (zh->iconv_from_utf8 != 0)
2450     {
2451         int len;
2452         char *inbuf = term_src;
2453         size_t inleft = strlen(term_src);
2454         char *outbuf = term_dst;
2455         size_t outleft = sizeof(term_dst)-1;
2456         size_t ret;
2457
2458         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2459                          &outbuf, &outleft);
2460         if (ret == (size_t)(-1))
2461             len = 0;
2462         else
2463             len = outbuf - term_dst;
2464         *dst = nmem_malloc(stream, len + 1);
2465         if (len > 0)
2466             memcpy (*dst, term_dst, len);
2467         (*dst)[len] = '\0';
2468     }
2469     else
2470         *dst = nmem_strdup(stream, term_src);
2471 }
2472
2473 static void count_set (RSET r, int *count)
2474 {
2475     zint psysno = 0;
2476     int kno = 0;
2477     struct it_key key;
2478     RSFD rfd;
2479
2480     yaz_log(YLOG_DEBUG, "count_set");
2481
2482     *count = 0;
2483     rfd = rset_open (r, RSETF_READ);
2484     while (rset_read (rfd, &key,0 /* never mind terms */))
2485     {
2486         if (key.mem[0] != psysno)
2487         {
2488             psysno = key.mem[0];
2489             (*count)++;
2490         }
2491         kno++;
2492     }
2493     rset_close (rfd);
2494     yaz_log(YLOG_DEBUG, "%d keys, %d records", kno, *count);
2495 }
2496
2497 void rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2498                oid_value attributeset,
2499                int num_bases, char **basenames,
2500                int *position, int *num_entries, ZebraScanEntry **list,
2501                int *is_partial, RSET limit_set, int return_zero)
2502 {
2503     int i;
2504     int pos = *position;
2505     int num = *num_entries;
2506     int before;
2507     int after;
2508     int base_no;
2509     char termz[IT_MAX_WORD+20];
2510     AttrType use;
2511     int use_value;
2512     const char *use_string = 0;
2513     struct scan_info *scan_info_array;
2514     ZebraScanEntry *glist;
2515     int ords[32], ord_no = 0;
2516     int ptr[32];
2517
2518     int bases_ok = 0;     /* no of databases with OK attribute */
2519     int errCode = 0;      /* err code (if any is not OK) */
2520     char *errString = 0;  /* addinfo */
2521
2522     unsigned reg_id;
2523     char *search_type = NULL;
2524     char rank_type[128];
2525     int complete_flag;
2526     int sort_flag;
2527     NMEM rset_nmem = NULL;
2528
2529     *list = 0;
2530     *is_partial = 0;
2531
2532     if (attributeset == VAL_NONE)
2533         attributeset = VAL_BIB1;
2534
2535     if (!limit_set)
2536     {
2537         AttrType termset;
2538         int termset_value_numeric;
2539         const char *termset_value_string;
2540         attr_init (&termset, zapt, 8);
2541         termset_value_numeric =
2542             attr_find_ex (&termset, NULL, &termset_value_string);
2543         if (termset_value_numeric != -1)
2544         {
2545             char resname[32];
2546             const char *termset_name = 0;
2547
2548             if (termset_value_numeric != -2)
2549             {
2550
2551                 sprintf (resname, "%d", termset_value_numeric);
2552                 termset_name = resname;
2553             }
2554             else
2555                 termset_name = termset_value_string;
2556
2557             limit_set = resultSetRef (zh, termset_name);
2558         }
2559     }
2560
2561     yaz_log (YLOG_DEBUG, "position = %d, num = %d set=%d",
2562              pos, num, attributeset);
2563
2564     attr_init (&use, zapt, 1);
2565     use_value = attr_find_ex (&use, &attributeset, &use_string);
2566
2567     if (zebra_maps_attr (zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2568                          rank_type, &complete_flag, &sort_flag))
2569     {
2570         *num_entries = 0;
2571         zh->errCode = 113;
2572         return ;
2573     }
2574     yaz_log (YLOG_DEBUG, "use_value = %d", use_value);
2575
2576     if (use_value == -1)
2577         use_value = 1016;
2578     for (base_no = 0; base_no < num_bases && ord_no < 32; base_no++)
2579     {
2580         data1_local_attribute *local_attr;
2581         attent attp;
2582         int ord;
2583
2584         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2585         {
2586             zh->errString = basenames[base_no];
2587             zh->errCode = 109; /* Database unavailable */
2588             *num_entries = 0;
2589             return;
2590         }
2591
2592         if (use_string &&
2593             (ord = zebraExplain_lookup_attr_str(zh->reg->zei,
2594                                                 use_string)) >= 0)
2595         {
2596             /* we have a match for a raw string attribute */
2597             if (ord > 0)
2598                 ords[ord_no++] = ord;
2599             attp.local_attributes = 0;  /* no more attributes */
2600         }
2601         else
2602         {
2603             int r;
2604
2605             if ((r = att_getentbyatt (zh, &attp, attributeset, use_value,
2606                                       use_string)))
2607             {
2608                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
2609                         attributeset, use_value);
2610                 if (r == -1)
2611                 {
2612                     errCode = 114;
2613                     if (use_string)
2614                         errString = odr_strdup(stream, use_string);
2615                     else
2616                     {
2617                         char val_str[32];
2618                         sprintf (val_str, "%d", use_value);
2619                         errString = odr_strdup(stream, val_str);
2620                     }
2621                 }
2622                 else
2623                     errCode = 121;
2624                 continue;
2625             }
2626         }
2627         bases_ok++;
2628         for (local_attr = attp.local_attributes; local_attr && ord_no < 32;
2629              local_attr = local_attr->next)
2630         {
2631             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
2632                                               attp.attset_ordinal,
2633                                               local_attr->local);
2634             if (ord > 0)
2635                 ords[ord_no++] = ord;
2636         }
2637     }
2638     if (!bases_ok && errCode)
2639     {
2640         zh->errCode = errCode;
2641         zh->errString = errString;
2642         *num_entries = 0;
2643         return;
2644     }
2645     if (ord_no == 0)
2646     {
2647         *num_entries = 0;
2648         return;
2649     }
2650     /* prepare dictionary scanning */
2651     before = pos-1;
2652     after = 1+num-pos;
2653     scan_info_array = (struct scan_info *)
2654         odr_malloc(stream, ord_no * sizeof(*scan_info_array));
2655     for (i = 0; i < ord_no; i++)
2656     {
2657         int j, prefix_len = 0;
2658         int before_tmp = before, after_tmp = after;
2659         struct scan_info *scan_info = scan_info_array + i;
2660         struct rpn_char_map_info rcmi;
2661
2662         rpn_char_map_prepare (zh->reg, reg_id, &rcmi);
2663
2664         scan_info->before = before;
2665         scan_info->after = after;
2666         scan_info->odr = stream;
2667
2668         scan_info->list = (struct scan_info_entry *)
2669             odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
2670         for (j = 0; j<before+after; j++)
2671             scan_info->list[j].term = NULL;
2672
2673         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2674         termz[prefix_len++] = reg_id;
2675         termz[prefix_len] = 0;
2676         strcpy(scan_info->prefix, termz);
2677
2678         if (trans_scan_term(zh, zapt, termz+prefix_len, reg_id))
2679             return ;
2680
2681         dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
2682                   scan_info, scan_handle);
2683     }
2684     glist = (ZebraScanEntry *)
2685         odr_malloc(stream, (before+after)*sizeof(*glist));
2686
2687     rset_nmem = nmem_create();
2688
2689     /* consider terms after main term */
2690     for (i = 0; i < ord_no; i++)
2691         ptr[i] = before;
2692
2693     *is_partial = 0;
2694     for (i = 0; i<after; i++)
2695     {
2696         int j, j0 = -1;
2697         const char *mterm = NULL;
2698         const char *tst;
2699         RSET rset;
2700
2701         for (j = 0; j < ord_no; j++)
2702         {
2703             if (ptr[j] < before+after &&
2704                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2705                 (!mterm || strcmp (tst, mterm) < 0))
2706             {
2707                 j0 = j;
2708                 mterm = tst;
2709             }
2710         }
2711         if (j0 == -1)
2712             break;
2713         scan_term_untrans (zh, stream->mem, reg_id,
2714                            &glist[i+before].term, mterm);
2715         rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2716                           glist[i+before].term, strlen(glist[i+before].term),
2717                           NULL, 0, zapt->term->which, rset_nmem,
2718                           key_it_ctrl,key_it_ctrl->scope);
2719         ptr[j0]++;
2720         for (j = j0+1; j<ord_no; j++)
2721         {
2722             if (ptr[j] < before+after &&
2723                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2724                 !strcmp (tst, mterm))
2725             {
2726                 RSET rsets[2];
2727
2728                 rsets[0] = rset;
2729                 rsets[1] =
2730                     rset_trunc(zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
2731                                glist[i+before].term,
2732                                strlen(glist[i+before].term), NULL, 0,
2733                                zapt->term->which,rset_nmem,
2734                                key_it_ctrl, key_it_ctrl->scope);
2735                 rset = rsmulti_or_create(rset_nmem, key_it_ctrl,
2736                                          2, key_it_ctrl->scope, rsets);
2737                 ptr[j]++;
2738             }
2739         }
2740         if (limit_set)
2741         {
2742             RSET rsets[2];
2743             rsets[0] = rset;
2744             rsets[1] = rset_dup(limit_set);
2745
2746             rset = rsmulti_and_create(rset_nmem, key_it_ctrl,
2747                                       key_it_ctrl->scope, 2, rsets);
2748         }
2749         count_set(rset, &glist[i+before].occurrences);
2750         rset_delete(rset);
2751     }
2752     if (i < after)
2753     {
2754         *num_entries -= (after-i);
2755         *is_partial = 1;
2756     }
2757
2758     /* consider terms before main term */
2759     for (i = 0; i<ord_no; i++)
2760         ptr[i] = 0;
2761
2762     for (i = 0; i<before; i++)
2763     {
2764         int j, j0 = -1;
2765         const char *mterm = NULL;
2766         const char *tst;
2767         RSET rset;
2768
2769         for (j = 0; j <ord_no; j++)
2770         {
2771             if (ptr[j] < before &&
2772                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2773                 (!mterm || strcmp (tst, mterm) > 0))
2774             {
2775                 j0 = j;
2776                     mterm = tst;
2777             }
2778         }
2779         if (j0 == -1)
2780             break;
2781
2782         scan_term_untrans (zh, stream->mem, reg_id,
2783                            &glist[before-1-i].term, mterm);
2784
2785         rset = rset_trunc
2786             (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
2787              glist[before-1-i].term, strlen(glist[before-1-i].term),
2788              NULL, 0, zapt->term->which,rset_nmem,
2789              key_it_ctrl,key_it_ctrl->scope);
2790
2791         ptr[j0]++;
2792
2793         for (j = j0+1; j<ord_no; j++)
2794         {
2795             if (ptr[j] < before &&
2796                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2797                 !strcmp (tst, mterm))
2798             {
2799                 RSET rsets[2];
2800
2801                 rsets[0] = rset;
2802                 rsets[1] = rset_trunc(
2803                     zh,
2804                     &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
2805                     glist[before-1-i].term,
2806                     strlen(glist[before-1-i].term), NULL, 0,
2807                     zapt->term->which, rset_nmem,
2808                     key_it_ctrl, key_it_ctrl->scope);
2809                 rset = rsmulti_or_create(rset_nmem, key_it_ctrl,
2810                                          2, key_it_ctrl->scope, rsets);
2811
2812                 ptr[j]++;
2813             }
2814         }
2815         if (limit_set)
2816         {
2817             RSET rsets[2];
2818             rsets[0] = rset;
2819             rsets[1] = rset_dup(limit_set);
2820
2821             rset = rsmulti_and_create(rset_nmem, key_it_ctrl,
2822                                       key_it_ctrl->scope, 2, rsets);
2823         }
2824         count_set (rset, &glist[before-1-i].occurrences);
2825         rset_delete (rset);
2826     }
2827     i = before-i;
2828     if (i)
2829     {
2830         *is_partial = 1;
2831         *position -= i;
2832         *num_entries -= i;
2833     }
2834
2835     nmem_destroy(rset_nmem);
2836     *list = glist + i;               /* list is set to first 'real' entry */
2837
2838     yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
2839             *position, *num_entries);
2840     if (zh->errCode)
2841         yaz_log(YLOG_DEBUG, "scan error: %d", zh->errCode);
2842 }
2843