index/zrpn.c

   1 /* $Id: zrpn.c,v 1.176 2005-04-20 10:17:14 adam Exp $
   2    Copyright (C) 1995-2005
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with Zebra; see the file LICENSE.zebra.  If not, write to the
  19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  20 02111-1307, USA.
  21 */
  22
  23
  24 #include <stdio.h>
  25 #include <assert.h>
  26 #ifdef WIN32
  27 #include <io.h>
  28 #else
  29 #include <unistd.h>
  30 #endif
  31 #include <ctype.h>
  32
  33 #include "index.h"
  34 #include <zebra_xpath.h>
  35
  36 #include <charmap.h>
  37 #include <rset.h>
  38
  39 /* maximum number of terms in an and/or/phrase item */
  40 #define TERM_LIST_LENGTH_MAX 256
  41
  42 static const struct key_control it_ctrl =
  43 {
  44     sizeof(struct it_key),
  45     2, /* we have sysnos and seqnos in this key, nothing more */
  46     key_compare_it,
  47     key_logdump_txt,   /* FIXME  - clean up these functions */
  48     key_get_seq,
  49 };
  50
  51
  52 const struct key_control *key_it_ctrl = &it_ctrl;
  53
  54 struct rpn_char_map_info
  55 {
  56     ZebraMaps zm;
  57     int reg_type;
  58 };
  59
  60 typedef struct
  61 {
  62     int type;
  63     int major;
  64     int minor;
  65     Z_AttributesPlusTerm *zapt;
  66 } AttrType;
  67
  68
  69 static int log_level_set = 0;
  70 static int log_level_rpn = 0;
  71
  72 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  73 {
  74     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  75     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  76 #if 0
  77     if (out && *out)
  78     {
  79         const char *outp = *out;
  80         yaz_log(YLOG_LOG, "---");
  81         while (*outp)
  82         {
  83             yaz_log(YLOG_LOG, "%02X", *outp);
  84             outp++;
  85         }
  86     }
  87 #endif
  88     return out;
  89 }
  90
  91 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  92                                   struct rpn_char_map_info *map_info)
  93 {
  94     map_info->zm = reg->zebra_maps;
  95     map_info->reg_type = reg_type;
  96     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  97 }
  98
  99 static int attr_find_ex(AttrType *src, oid_value *attributeSetP,
 100                          const char **string_value)
 101 {
 102     int num_attributes;
 103
 104     num_attributes = src->zapt->attributes->num_attributes;
 105     while (src->major < num_attributes)
 106     {
 107         Z_AttributeElement *element;
 108
 109         element = src->zapt->attributes->attributes[src->major];
 110         if (src->type == *element->attributeType)
 111         {
 112             switch (element->which)
 113             {
 114             case Z_AttributeValue_numeric:
 115                 ++(src->major);
 116                 if (element->attributeSet && attributeSetP)
 117                 {
 118                     oident *attrset;
 119
 120                     attrset = oid_getentbyoid(element->attributeSet);
 121                     *attributeSetP = attrset->value;
 122                 }
 123                 return *element->value.numeric;
 124                 break;
 125             case Z_AttributeValue_complex:
 126                 if (src->minor >= element->value.complex->num_list)
 127                     break;
 128                 if (element->attributeSet && attributeSetP)
 129                 {
 130                     oident *attrset;
 131
 132                     attrset = oid_getentbyoid(element->attributeSet);
 133                     *attributeSetP = attrset->value;
 134                 }
 135                 if (element->value.complex->list[src->minor]->which ==
 136                     Z_StringOrNumeric_numeric)
 137                 {
 138                     ++(src->minor);
 139                     return
 140                         *element->value.complex->list[src->minor-1]->u.numeric;
 141                 }
 142                 else if (element->value.complex->list[src->minor]->which ==
 143                          Z_StringOrNumeric_string)
 144                 {
 145                     if (!string_value)
 146                         break;
 147                     ++(src->minor);
 148                     *string_value =
 149                         element->value.complex->list[src->minor-1]->u.string;
 150                     return -2;
 151                 }
 152                 else
 153                     break;
 154             default:
 155                 assert(0);
 156             }
 157         }
 158         ++(src->major);
 159     }
 160     return -1;
 161 }
 162
 163 static int attr_find(AttrType *src, oid_value *attributeSetP)
 164 {
 165     return attr_find_ex(src, attributeSetP, 0);
 166 }
 167
 168 static void attr_init(AttrType *src, Z_AttributesPlusTerm *zapt,
 169                        int type)
 170 {
 171     src->zapt = zapt;
 172     src->type = type;
 173     src->major = 0;
 174     src->minor = 0;
 175 }
 176
 177 #define TERM_COUNT
 178
 179 struct grep_info {
 180 #ifdef TERM_COUNT
 181     int *term_no;
 182 #endif
 183     ISAM_P *isam_p_buf;
 184     int isam_p_size;
 185     int isam_p_indx;
 186     ZebraHandle zh;
 187     int reg_type;
 188     ZebraSet termset;
 189 };
 190
 191 static void term_untrans(ZebraHandle zh, int reg_type,
 192                            char *dst, const char *src)
 193 {
 194     int len = 0;
 195     while (*src)
 196     {
 197         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
 198                                            reg_type, &src);
 199         if (!cp && len < IT_MAX_WORD-1)
 200             dst[len++] = *src++;
 201         else
 202             while (*cp && len < IT_MAX_WORD-1)
 203                 dst[len++] = *cp++;
 204     }
 205     dst[len] = '\0';
 206 }
 207
 208 static void add_isam_p(const char *name, const char *info,
 209                        struct grep_info *p)
 210 {
 211     if (!log_level_set)
 212     {
 213         log_level_rpn = yaz_log_module_level("rpn");
 214         log_level_set = 1;
 215     }
 216     if (p->isam_p_indx == p->isam_p_size)
 217     {
 218         ISAM_P *new_isam_p_buf;
 219 #ifdef TERM_COUNT
 220         int *new_term_no;
 221 #endif
 222         p->isam_p_size = 2*p->isam_p_size + 100;
 223         new_isam_p_buf = (ISAM_P *) xmalloc(sizeof(*new_isam_p_buf) *
 224                                             p->isam_p_size);
 225         if (p->isam_p_buf)
 226         {
 227             memcpy(new_isam_p_buf, p->isam_p_buf,
 228                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 229             xfree(p->isam_p_buf);
 230         }
 231         p->isam_p_buf = new_isam_p_buf;
 232
 233 #ifdef TERM_COUNT
 234         new_term_no = (int *) xmalloc(sizeof(*new_term_no) * p->isam_p_size);
 235         if (p->term_no)
 236         {
 237             memcpy(new_term_no, p->isam_p_buf,
 238                     p->isam_p_indx * sizeof(*p->term_no));
 239             xfree(p->term_no);
 240         }
 241         p->term_no = new_term_no;
 242 #endif
 243     }
 244     assert(*info == sizeof(*p->isam_p_buf));
 245     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 246
 247 #if 1
 248     if (p->termset)
 249     {
 250         const char *db;
 251         int set, use;
 252         char term_tmp[IT_MAX_WORD];
 253         int su_code = 0;
 254         int len = key_SU_decode (&su_code, name);
 255
 256         term_untrans  (p->zh, p->reg_type, term_tmp, name+len+1);
 257         yaz_log(log_level_rpn, "grep: %d %c %s", su_code, name[len], term_tmp);
 258         zebraExplain_lookup_ord (p->zh->reg->zei,
 259                                  su_code, &db, &set, &use);
 260         yaz_log(log_level_rpn, "grep:  set=%d use=%d db=%s", set, use, db);
 261
 262         resultSetAddTerm(p->zh, p->termset, name[len], db,
 263                          set, use, term_tmp);
 264     }
 265 #endif
 266     (p->isam_p_indx)++;
 267 }
 268
 269 static int grep_handle(char *name, const char *info, void *p)
 270 {
 271     add_isam_p(name, info, (struct grep_info *) p);
 272     return 0;
 273 }
 274
 275 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 276                      const char *ct1, const char *ct2, int first)
 277 {
 278     const char *s1, *s0 = *src;
 279     const char **map;
 280
 281     /* skip white space */
 282     while (*s0)
 283     {
 284         if (ct1 && strchr(ct1, *s0))
 285             break;
 286         if (ct2 && strchr(ct2, *s0))
 287             break;
 288         s1 = s0;
 289         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 290         if (**map != *CHR_SPACE)
 291             break;
 292         s0 = s1;
 293     }
 294     *src = s0;
 295     return *s0;
 296 }
 297
 298
 299 static void esc_str(char *out_buf, int out_size,
 300                     const char *in_buf, int in_size)
 301 {
 302     int k;
 303
 304     assert(out_buf);
 305     assert(in_buf);
 306     assert(out_size > 20);
 307     *out_buf = '\0';
 308     for (k = 0; k<in_size; k++)
 309     {
 310         int c = in_buf[k] & 0xff;
 311         int pc;
 312         if (c < 32 || c > 126)
 313             pc = '?';
 314         else
 315             pc = c;
 316         sprintf(out_buf +strlen(out_buf), "%02X:%c  ", c, pc);
 317         if (strlen(out_buf) > out_size-20)
 318         {
 319             strcat(out_buf, "..");
 320             break;
 321         }
 322     }
 323 }
 324
 325 #define REGEX_CHARS " []()|.*+?!"
 326
 327 /* term_100: handle term, where trunc = none(no operators at all) */
 328 static int term_100(ZebraMaps zebra_maps, int reg_type,
 329                      const char **src, char *dst, int space_split,
 330                      char *dst_term)
 331 {
 332     const char *s0;
 333     const char **map;
 334     int i = 0;
 335     int j = 0;
 336
 337     const char *space_start = 0;
 338     const char *space_end = 0;
 339
 340     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 341         return 0;
 342     s0 = *src;
 343     while (*s0)
 344     {
 345         const char *s1 = s0;
 346         int q_map_match = 0;
 347         map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 348                                 &q_map_match);
 349         if (space_split)
 350         {
 351             if (**map == *CHR_SPACE)
 352                 break;
 353         }
 354         else  /* complete subfield only. */
 355         {
 356             if (**map == *CHR_SPACE)
 357             {   /* save space mapping for later  .. */
 358                 space_start = s1;
 359                 space_end = s0;
 360                 continue;
 361             }
 362             else if (space_start)
 363             {   /* reload last space */
 364                 while (space_start < space_end)
 365                 {
 366                     if (strchr(REGEX_CHARS, *space_start))
 367                         dst[i++] = '\\';
 368                     dst_term[j++] = *space_start;
 369                     dst[i++] = *space_start++;
 370                 }
 371                 /* and reset */
 372                 space_start = space_end = 0;
 373             }
 374         }
 375         /* add non-space char */
 376         memcpy(dst_term+j, s1, s0 - s1);
 377         j += (s0 - s1);
 378         if (!q_map_match)
 379         {
 380             while (s1 < s0)
 381             {
 382                 if (strchr(REGEX_CHARS, *s1))
 383                     dst[i++] = '\\';
 384                 dst[i++] = *s1++;
 385             }
 386         }
 387         else
 388         {
 389             char tmpbuf[80];
 390             esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 391
 392             strcpy(dst + i, map[0]);
 393             i += strlen(map[0]);
 394         }
 395     }
 396     dst[i] = '\0';
 397     dst_term[j] = '\0';
 398     *src = s0;
 399     return i;
 400 }
 401
 402 /* term_101: handle term, where trunc = Process # */
 403 static int term_101(ZebraMaps zebra_maps, int reg_type,
 404                      const char **src, char *dst, int space_split,
 405                      char *dst_term)
 406 {
 407     const char *s0;
 408     const char **map;
 409     int i = 0;
 410     int j = 0;
 411
 412     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 413         return 0;
 414     s0 = *src;
 415     while (*s0)
 416     {
 417         if (*s0 == '#')
 418         {
 419             dst[i++] = '.';
 420             dst[i++] = '*';
 421             dst_term[j++] = *s0++;
 422         }
 423         else
 424         {
 425             const char *s1 = s0;
 426             int q_map_match = 0;
 427             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 428                                     &q_map_match);
 429             if (space_split && **map == *CHR_SPACE)
 430                 break;
 431
 432             /* add non-space char */
 433             memcpy(dst_term+j, s1, s0 - s1);
 434             j += (s0 - s1);
 435             if (!q_map_match)
 436             {
 437                 while (s1 < s0)
 438                 {
 439                     if (strchr(REGEX_CHARS, *s1))
 440                         dst[i++] = '\\';
 441                     dst[i++] = *s1++;
 442                 }
 443             }
 444             else
 445             {
 446                 char tmpbuf[80];
 447                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 448
 449                 strcpy(dst + i, map[0]);
 450                 i += strlen(map[0]);
 451             }
 452         }
 453     }
 454     dst[i] = '\0';
 455     dst_term[j++] = '\0';
 456     *src = s0;
 457     return i;
 458 }
 459
 460 /* term_103: handle term, where trunc = re-2 (regular expressions) */
 461 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 462                      char *dst, int *errors, int space_split,
 463                      char *dst_term)
 464 {
 465     int i = 0;
 466     int j = 0;
 467     const char *s0;
 468     const char **map;
 469
 470     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 471         return 0;
 472     s0 = *src;
 473     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 474         isdigit(((const unsigned char *)s0)[1]))
 475     {
 476         *errors = s0[1] - '0';
 477         s0 += 3;
 478         if (*errors > 3)
 479             *errors = 3;
 480     }
 481     while (*s0)
 482     {
 483         if (strchr("^\\()[].*+?|-", *s0))
 484         {
 485             dst_term[j++] = *s0;
 486             dst[i++] = *s0++;
 487         }
 488         else
 489         {
 490             const char *s1 = s0;
 491             int q_map_match = 0;
 492             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 493                                     &q_map_match);
 494             if (space_split && **map == *CHR_SPACE)
 495                 break;
 496
 497             /* add non-space char */
 498             memcpy(dst_term+j, s1, s0 - s1);
 499             j += (s0 - s1);
 500             if (!q_map_match)
 501             {
 502                 while (s1 < s0)
 503                 {
 504                     if (strchr(REGEX_CHARS, *s1))
 505                         dst[i++] = '\\';
 506                     dst[i++] = *s1++;
 507                 }
 508             }
 509             else
 510             {
 511                 char tmpbuf[80];
 512                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 513
 514                 strcpy(dst + i, map[0]);
 515                 i += strlen(map[0]);
 516             }
 517         }
 518     }
 519     dst[i] = '\0';
 520     dst_term[j] = '\0';
 521     *src = s0;
 522
 523     return i;
 524 }
 525
 526 /* term_103: handle term, where trunc = re-1 (regular expressions) */
 527 static int term_102 (ZebraMaps zebra_maps, int reg_type, const char **src,
 528                      char *dst, int space_split, char *dst_term)
 529 {
 530     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 531                      dst_term);
 532 }
 533
 534
 535 /* term_104: handle term, where trunc = Process # and ! */
 536 static int term_104(ZebraMaps zebra_maps, int reg_type,
 537                      const char **src, char *dst, int space_split,
 538                      char *dst_term)
 539 {
 540     const char *s0, *s1;
 541     const char **map;
 542     int i = 0;
 543     int j = 0;
 544
 545     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 546         return 0;
 547     s0 = *src;
 548     while (*s0)
 549     {
 550         if (*s0 == '?')
 551         {
 552             dst_term[j++] = *s0++;
 553             if (*s0 >= '0' && *s0 <= '9')
 554             {
 555                 int limit = 0;
 556                 while (*s0 >= '0' && *s0 <= '9')
 557                 {
 558                     limit = limit * 10 + (*s0 - '0');
 559                     dst_term[j++] = *s0++;
 560                 }
 561                 if (limit > 20)
 562                     limit = 20;
 563                 while (--limit >= 0)
 564                 {
 565                     dst[i++] = '.';
 566                     dst[i++] = '?';
 567                 }
 568             }
 569             else
 570             {
 571                 dst[i++] = '.';
 572                 dst[i++] = '*';
 573             }
 574         }
 575         else if (*s0 == '*')
 576         {
 577             dst[i++] = '.';
 578             dst[i++] = '*';
 579             dst_term[j++] = *s0++;
 580         }
 581         else if (*s0 == '#')
 582         {
 583             dst[i++] = '.';
 584             dst_term[j++] = *s0++;
 585         }
 586         else
 587         {
 588             s1 = s0;
 589             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 590             if (space_split && **map == *CHR_SPACE)
 591                 break;
 592             while (s1 < s0)
 593             {
 594                 if (strchr(REGEX_CHARS, *s1))
 595                     dst[i++] = '\\';
 596                 dst_term[j++] = *s1;
 597                 dst[i++] = *s1++;
 598             }
 599         }
 600     }
 601     dst[i] = '\0';
 602     dst_term[j++] = '\0';
 603     *src = s0;
 604     return i;
 605 }
 606
 607 /* term_105/106: handle term, where trunc = Process * and ! and right trunc */
 608 static int term_105 (ZebraMaps zebra_maps, int reg_type,
 609                      const char **src, char *dst, int space_split,
 610                      char *dst_term, int right_truncate)
 611 {
 612     const char *s0, *s1;
 613     const char **map;
 614     int i = 0;
 615     int j = 0;
 616
 617     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 618         return 0;
 619     s0 = *src;
 620     while (*s0)
 621     {
 622         if (*s0 == '*')
 623         {
 624             dst[i++] = '.';
 625             dst[i++] = '*';
 626             dst_term[j++] = *s0++;
 627         }
 628         else if (*s0 == '!')
 629         {
 630             dst[i++] = '.';
 631             dst_term[j++] = *s0++;
 632         }
 633         else
 634         {
 635             s1 = s0;
 636             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 637             if (space_split && **map == *CHR_SPACE)
 638                 break;
 639             while (s1 < s0)
 640             {
 641                 if (strchr(REGEX_CHARS, *s1))
 642                     dst[i++] = '\\';
 643                 dst_term[j++] = *s1;
 644                 dst[i++] = *s1++;
 645             }
 646         }
 647     }
 648     if (right_truncate)
 649     {
 650         dst[i++] = '.';
 651         dst[i++] = '*';
 652     }
 653     dst[i] = '\0';
 654
 655     dst_term[j++] = '\0';
 656     *src = s0;
 657     return i;
 658 }
 659
 660
 661 /* gen_regular_rel - generate regular expression from relation
 662  *  val:     border value (inclusive)
 663  *  islt:    1 if <=; 0 if >=.
 664  */
 665 static void gen_regular_rel(char *dst, int val, int islt)
 666 {
 667     int dst_p;
 668     int w, d, i;
 669     int pos = 0;
 670     char numstr[20];
 671
 672     yaz_log(YLOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 673     if (val >= 0)
 674     {
 675         if (islt)
 676             strcpy(dst, "(-[0-9]+|(");
 677         else
 678             strcpy(dst, "((");
 679     }
 680     else
 681     {
 682         if (!islt)
 683         {
 684             strcpy(dst, "([0-9]+|-(");
 685             dst_p = strlen(dst);
 686             islt = 1;
 687         }
 688         else
 689         {
 690             strcpy(dst, "(-(");
 691             islt = 0;
 692         }
 693         val = -val;
 694     }
 695     dst_p = strlen(dst);
 696     sprintf(numstr, "%d", val);
 697     for (w = strlen(numstr); --w >= 0; pos++)
 698     {
 699         d = numstr[w];
 700         if (pos > 0)
 701         {
 702             if (islt)
 703             {
 704                 if (d == '0')
 705                     continue;
 706                 d--;
 707             }
 708             else
 709             {
 710                 if (d == '9')
 711                     continue;
 712                 d++;
 713             }
 714         }
 715
 716         strcpy(dst + dst_p, numstr);
 717         dst_p = strlen(dst) - pos - 1;
 718
 719         if (islt)
 720         {
 721             if (d != '0')
 722             {
 723                 dst[dst_p++] = '[';
 724                 dst[dst_p++] = '0';
 725                 dst[dst_p++] = '-';
 726                 dst[dst_p++] = d;
 727                 dst[dst_p++] = ']';
 728             }
 729             else
 730                 dst[dst_p++] = d;
 731         }
 732         else
 733         {
 734             if (d != '9')
 735             {
 736                 dst[dst_p++] = '[';
 737                 dst[dst_p++] = d;
 738                 dst[dst_p++] = '-';
 739                 dst[dst_p++] = '9';
 740                 dst[dst_p++] = ']';
 741             }
 742             else
 743                 dst[dst_p++] = d;
 744         }
 745         for (i = 0; i<pos; i++)
 746         {
 747             dst[dst_p++] = '[';
 748             dst[dst_p++] = '0';
 749             dst[dst_p++] = '-';
 750             dst[dst_p++] = '9';
 751             dst[dst_p++] = ']';
 752         }
 753         dst[dst_p++] = '|';
 754     }
 755     dst[dst_p] = '\0';
 756     if (islt)
 757     {
 758         /* match everything less than 10^(pos-1) */
 759         strcat(dst, "0*");
 760         for (i = 1; i<pos; i++)
 761             strcat(dst, "[0-9]?");
 762     }
 763     else
 764     {
 765         /* match everything greater than 10^pos */
 766         for (i = 0; i <= pos; i++)
 767             strcat(dst, "[0-9]");
 768         strcat(dst, "[0-9]*");
 769     }
 770     strcat(dst, "))");
 771 }
 772
 773 void string_rel_add_char(char **term_p, const char *src, int *indx)
 774 {
 775     if (src[*indx] == '\\')
 776         *(*term_p)++ = src[(*indx)++];
 777     *(*term_p)++ = src[(*indx)++];
 778 }
 779
 780 /*
 781  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 782  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 783  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 784  *              ([^-a].*|a[^-b].*|ab[c-].*)
 785  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 786  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 787  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 788  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 789  */
 790 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 791                            const char **term_sub, char *term_dict,
 792                            oid_value attributeSet,
 793                            int reg_type, int space_split, char *term_dst,
 794                            int *error_code)
 795 {
 796     AttrType relation;
 797     int relation_value;
 798     int i;
 799     char *term_tmp = term_dict + strlen(term_dict);
 800     char term_component[2*IT_MAX_WORD+20];
 801
 802     attr_init(&relation, zapt, 2);
 803     relation_value = attr_find(&relation, NULL);
 804
 805     *error_code = 0;
 806     yaz_log(YLOG_DEBUG, "string relation value=%d", relation_value);
 807     switch (relation_value)
 808     {
 809     case 1:
 810         if (!term_100 (zh->reg->zebra_maps, reg_type,
 811                        term_sub, term_component,
 812                        space_split, term_dst))
 813             return 0;
 814         yaz_log(log_level_rpn, "Relation <");
 815
 816         *term_tmp++ = '(';
 817         for (i = 0; term_component[i]; )
 818         {
 819             int j = 0;
 820
 821             if (i)
 822                 *term_tmp++ = '|';
 823             while (j < i)
 824                 string_rel_add_char (&term_tmp, term_component, &j);
 825
 826             *term_tmp++ = '[';
 827
 828             *term_tmp++ = '^';
 829             string_rel_add_char (&term_tmp, term_component, &i);
 830             *term_tmp++ = '-';
 831
 832             *term_tmp++ = ']';
 833             *term_tmp++ = '.';
 834             *term_tmp++ = '*';
 835
 836             if ((term_tmp - term_dict) > IT_MAX_WORD)
 837                 break;
 838         }
 839         *term_tmp++ = ')';
 840         *term_tmp = '\0';
 841         break;
 842     case 2:
 843         if (!term_100 (zh->reg->zebra_maps, reg_type,
 844                        term_sub, term_component,
 845                        space_split, term_dst))
 846             return 0;
 847         yaz_log(log_level_rpn, "Relation <=");
 848
 849         *term_tmp++ = '(';
 850         for (i = 0; term_component[i]; )
 851         {
 852             int j = 0;
 853
 854             while (j < i)
 855                 string_rel_add_char (&term_tmp, term_component, &j);
 856             *term_tmp++ = '[';
 857
 858             *term_tmp++ = '^';
 859             string_rel_add_char (&term_tmp, term_component, &i);
 860             *term_tmp++ = '-';
 861
 862             *term_tmp++ = ']';
 863             *term_tmp++ = '.';
 864             *term_tmp++ = '*';
 865
 866             *term_tmp++ = '|';
 867
 868             if ((term_tmp - term_dict) > IT_MAX_WORD)
 869                 break;
 870         }
 871         for (i = 0; term_component[i]; )
 872             string_rel_add_char (&term_tmp, term_component, &i);
 873         *term_tmp++ = ')';
 874         *term_tmp = '\0';
 875         break;
 876     case 5:
 877         if (!term_100 (zh->reg->zebra_maps, reg_type,
 878                        term_sub, term_component, space_split, term_dst))
 879             return 0;
 880         yaz_log(log_level_rpn, "Relation >");
 881
 882         *term_tmp++ = '(';
 883         for (i = 0; term_component[i];)
 884         {
 885             int j = 0;
 886
 887             while (j < i)
 888                 string_rel_add_char (&term_tmp, term_component, &j);
 889             *term_tmp++ = '[';
 890
 891             *term_tmp++ = '^';
 892             *term_tmp++ = '-';
 893             string_rel_add_char (&term_tmp, term_component, &i);
 894
 895             *term_tmp++ = ']';
 896             *term_tmp++ = '.';
 897             *term_tmp++ = '*';
 898
 899             *term_tmp++ = '|';
 900
 901             if ((term_tmp - term_dict) > IT_MAX_WORD)
 902                 break;
 903         }
 904         for (i = 0; term_component[i];)
 905             string_rel_add_char (&term_tmp, term_component, &i);
 906         *term_tmp++ = '.';
 907         *term_tmp++ = '+';
 908         *term_tmp++ = ')';
 909         *term_tmp = '\0';
 910         break;
 911     case 4:
 912         if (!term_100 (zh->reg->zebra_maps, reg_type, term_sub,
 913                        term_component, space_split, term_dst))
 914             return 0;
 915         yaz_log(log_level_rpn, "Relation >=");
 916
 917         *term_tmp++ = '(';
 918         for (i = 0; term_component[i];)
 919         {
 920             int j = 0;
 921
 922             if (i)
 923                 *term_tmp++ = '|';
 924             while (j < i)
 925                 string_rel_add_char (&term_tmp, term_component, &j);
 926             *term_tmp++ = '[';
 927
 928             if (term_component[i+1])
 929             {
 930                 *term_tmp++ = '^';
 931                 *term_tmp++ = '-';
 932                 string_rel_add_char (&term_tmp, term_component, &i);
 933             }
 934             else
 935             {
 936                 string_rel_add_char (&term_tmp, term_component, &i);
 937                 *term_tmp++ = '-';
 938             }
 939             *term_tmp++ = ']';
 940             *term_tmp++ = '.';
 941             *term_tmp++ = '*';
 942
 943             if ((term_tmp - term_dict) > IT_MAX_WORD)
 944                 break;
 945         }
 946         *term_tmp++ = ')';
 947         *term_tmp = '\0';
 948         break;
 949     case 3:
 950     case 102:
 951     case -1:
 952         yaz_log(log_level_rpn, "Relation =");
 953         if (!term_100 (zh->reg->zebra_maps, reg_type, term_sub,
 954                        term_component, space_split, term_dst))
 955             return 0;
 956         strcat(term_tmp, "(");
 957         strcat(term_tmp, term_component);
 958         strcat(term_tmp, ")");
 959         break;
 960     default:
 961         *error_code = 117;
 962         return 0;
 963     }
 964     return 1;
 965 }
 966
 967 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 968                              const char **term_sub,
 969                              oid_value attributeSet, NMEM stream,
 970                              struct grep_info *grep_info,
 971                              int reg_type, int complete_flag,
 972                              int num_bases, char **basenames,
 973                              char *term_dst, int xpath_use);
 974
 975 static RSET term_trunc(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 976                        const char **term_sub,
 977                        oid_value attributeSet, NMEM stream,
 978                        struct grep_info *grep_info,
 979                        int reg_type, int complete_flag,
 980                        int num_bases, char **basenames,
 981                        char *term_dst,
 982                        const char *rank_type, int xpath_use,
 983                        NMEM rset_nmem)
 984 {
 985     ZEBRA_RES res;
 986     grep_info->isam_p_indx = 0;
 987     res = string_term(zh, zapt, term_sub, attributeSet, stream, grep_info,
 988                       reg_type, complete_flag, num_bases, basenames,
 989                       term_dst, xpath_use);
 990     if (res != ZEBRA_OK)
 991         return 0;
 992     if (!*term_sub)  /* no more terms ? */
 993         return 0;
 994     yaz_log(log_level_rpn, "term: %s", term_dst);
 995     return rset_trunc(zh, grep_info->isam_p_buf,
 996                       grep_info->isam_p_indx, term_dst,
 997                       strlen(term_dst), rank_type, 1 /* preserve pos */,
 998                       zapt->term->which, rset_nmem,
 999                       key_it_ctrl, key_it_ctrl->scope);
1000 }
1001 static char *nmem_strdup_i(NMEM nmem, int v)
1002 {
1003     char val_str[64];
1004     sprintf (val_str, "%d", v);
1005     return nmem_strdup(nmem, val_str);
1006 }
1007
1008 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1009                              const char **term_sub,
1010                              oid_value attributeSet, NMEM stream,
1011                              struct grep_info *grep_info,
1012                              int reg_type, int complete_flag,
1013                              int num_bases, char **basenames,
1014                              char *term_dst, int xpath_use)
1015 {
1016     char term_dict[2*IT_MAX_WORD+4000];
1017     int j, r, base_no;
1018     AttrType truncation;
1019     int truncation_value;
1020     AttrType use;
1021     int use_value;
1022     const char *use_string = 0;
1023     oid_value curAttributeSet = attributeSet;
1024     const char *termp;
1025     struct rpn_char_map_info rcmi;
1026     int space_split = complete_flag ? 0 : 1;
1027
1028     int bases_ok = 0;     /* no of databases with OK attribute */
1029     int errCode = 0;      /* err code (if any is not OK) */
1030     char *errString = 0;  /* addinfo */
1031
1032     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1033     attr_init (&use, zapt, 1);
1034     use_value = attr_find_ex (&use, &curAttributeSet, &use_string);
1035     yaz_log(log_level_rpn, "string_term, use value %d", use_value);
1036     attr_init (&truncation, zapt, 5);
1037     truncation_value = attr_find (&truncation, NULL);
1038     yaz_log(log_level_rpn, "truncation value %d", truncation_value);
1039
1040     if (use_value == -1)    /* no attribute - assumy "any" */
1041         use_value = 1016;
1042     for (base_no = 0; base_no < num_bases; base_no++)
1043     {
1044         int ord = -1;
1045         int attr_ok = 0;
1046         int regex_range = 0;
1047         int init_pos = 0;
1048         attent attp;
1049         data1_local_attribute id_xpath_attr;
1050         data1_local_attribute *local_attr;
1051         int max_pos, prefix_len = 0;
1052         int relation_error;
1053
1054         termp = *term_sub;
1055
1056         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1057         {
1058             zh->errCode = 109; /* Database unavailable */
1059             zh->errString = basenames[base_no];
1060             return ZEBRA_FAIL;
1061         }
1062         if (xpath_use > 0 && use_value == -2)
1063         {
1064             /* xpath mode and we have a string attribute */
1065             attp.local_attributes = &id_xpath_attr;
1066             attp.attset_ordinal = VAL_IDXPATH;
1067             id_xpath_attr.next = 0;
1068
1069             use_value = xpath_use;  /* xpath_use as use-attribute now */
1070             id_xpath_attr.local = use_value;
1071         }
1072         else if (curAttributeSet == VAL_IDXPATH && use_value >= 0)
1073         {
1074             /* X-Path attribute, use numeric value directly */
1075             attp.local_attributes = &id_xpath_attr;
1076             attp.attset_ordinal = VAL_IDXPATH;
1077             id_xpath_attr.next = 0;
1078             id_xpath_attr.local = use_value;
1079         }
1080         else if (use_string &&
1081                  (ord = zebraExplain_lookup_attr_str(zh->reg->zei,
1082                                                      use_string)) >= 0)
1083         {
1084             /* we have a match for a raw string attribute */
1085             char ord_buf[32];
1086             int i, ord_len;
1087
1088             if (prefix_len)
1089                 term_dict[prefix_len++] = '|';
1090             else
1091                 term_dict[prefix_len++] = '(';
1092
1093             ord_len = key_SU_encode (ord, ord_buf);
1094             for (i = 0; i<ord_len; i++)
1095             {
1096                 term_dict[prefix_len++] = 1;
1097                 term_dict[prefix_len++] = ord_buf[i];
1098             }
1099             attp.local_attributes = 0;  /* no more attributes */
1100         }
1101         else
1102         {
1103             /* lookup in the .att files . Allow string as well */
1104             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1105                                       use_string)))
1106             {
1107                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1108                         curAttributeSet, use_value, r);
1109                 if (r == -1)
1110                 {
1111                     /* set was found, but value wasn't defined */
1112                     errCode = 114;
1113                     if (use_string)
1114                         errString = nmem_strdup(stream, use_string);
1115                     else
1116                         errString = nmem_strdup_i (stream, use_value);
1117                 }
1118                 else
1119                 {
1120                     int oid[OID_SIZE];
1121                     struct oident oident;
1122
1123                     oident.proto = PROTO_Z3950;
1124                     oident.oclass = CLASS_ATTSET;
1125                     oident.value = curAttributeSet;
1126                     oid_ent_to_oid (&oident, oid);
1127
1128                     errCode = 121;
1129                     errString = nmem_strdup (stream, oident.desc);
1130                 }
1131                 continue;
1132             }
1133         }
1134         for (local_attr = attp.local_attributes; local_attr;
1135              local_attr = local_attr->next)
1136         {
1137             char ord_buf[32];
1138             int i, ord_len;
1139
1140             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1141                                               attp.attset_ordinal,
1142                                               local_attr->local);
1143             if (ord < 0)
1144                 continue;
1145             if (prefix_len)
1146                 term_dict[prefix_len++] = '|';
1147             else
1148                 term_dict[prefix_len++] = '(';
1149
1150             ord_len = key_SU_encode (ord, ord_buf);
1151             for (i = 0; i<ord_len; i++)
1152             {
1153                 term_dict[prefix_len++] = 1;
1154                 term_dict[prefix_len++] = ord_buf[i];
1155             }
1156         }
1157         bases_ok++;
1158         if (prefix_len)
1159             attr_ok = 1;
1160
1161         term_dict[prefix_len++] = ')';
1162         term_dict[prefix_len++] = 1;
1163         term_dict[prefix_len++] = reg_type;
1164         yaz_log(log_level_rpn, "reg_type = %d", term_dict[prefix_len-1]);
1165         term_dict[prefix_len] = '\0';
1166         j = prefix_len;
1167         switch (truncation_value)
1168         {
1169         case -1:         /* not specified */
1170         case 100:        /* do not truncate */
1171             if (!string_relation (zh, zapt, &termp, term_dict,
1172                                   attributeSet,
1173                                   reg_type, space_split, term_dst,
1174                                   &relation_error))
1175             {
1176                 if (relation_error)
1177                 {
1178                     zh->errCode = relation_error;
1179                     return ZEBRA_FAIL;
1180                 }
1181                 *term_sub = 0;
1182                 return ZEBRA_OK;
1183             }
1184             break;
1185         case 1:          /* right truncation */
1186             term_dict[j++] = '(';
1187             if (!term_100(zh->reg->zebra_maps, reg_type,
1188                           &termp, term_dict + j, space_split, term_dst))
1189             {
1190                 *term_sub = 0;
1191                 return ZEBRA_OK;
1192             }
1193             strcat(term_dict, ".*)");
1194             break;
1195         case 2:          /* keft truncation */
1196             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1197             if (!term_100(zh->reg->zebra_maps, reg_type,
1198                           &termp, term_dict + j, space_split, term_dst))
1199             {
1200                 *term_sub = 0;
1201                 return ZEBRA_OK;
1202             }
1203             strcat(term_dict, ")");
1204             break;
1205         case 3:          /* left&right truncation */
1206             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1207             if (!term_100(zh->reg->zebra_maps, reg_type,
1208                           &termp, term_dict + j, space_split, term_dst))
1209             {
1210                 *term_sub = 0;
1211                 return ZEBRA_OK;
1212             }
1213             strcat(term_dict, ".*)");
1214             break;
1215         case 101:        /* process # in term */
1216             term_dict[j++] = '(';
1217             if (!term_101(zh->reg->zebra_maps, reg_type,
1218                           &termp, term_dict + j, space_split, term_dst))
1219             {
1220                 *term_sub = 0;
1221                 return ZEBRA_OK;
1222             }
1223             strcat(term_dict, ")");
1224             break;
1225         case 102:        /* Regexp-1 */
1226             term_dict[j++] = '(';
1227             if (!term_102(zh->reg->zebra_maps, reg_type,
1228                           &termp, term_dict + j, space_split, term_dst))
1229             {
1230                 *term_sub = 0;
1231                 return ZEBRA_OK;
1232             }
1233             strcat(term_dict, ")");
1234             break;
1235         case 103:       /* Regexp-2 */
1236             r = 1;
1237             term_dict[j++] = '(';
1238             init_pos = 2;
1239             if (!term_103 (zh->reg->zebra_maps, reg_type,
1240                            &termp, term_dict + j, &regex_range,
1241                            space_split, term_dst))
1242             {
1243                 *term_sub = 0;
1244                 return ZEBRA_OK;
1245             }
1246             strcat(term_dict, ")");
1247         case 104:        /* process # and ! in term */
1248             term_dict[j++] = '(';
1249             if (!term_104 (zh->reg->zebra_maps, reg_type,
1250                            &termp, term_dict + j, space_split, term_dst))
1251             {
1252                 *term_sub = 0;
1253                 return ZEBRA_OK;
1254             }
1255             strcat(term_dict, ")");
1256             break;
1257         case 105:        /* process * and ! in term */
1258             term_dict[j++] = '(';
1259             if (!term_105 (zh->reg->zebra_maps, reg_type,
1260                            &termp, term_dict + j, space_split, term_dst, 1))
1261             {
1262                 *term_sub = 0;
1263                 return ZEBRA_OK;
1264             }
1265             strcat(term_dict, ")");
1266             break;
1267         case 106:        /* process * and ! in term */
1268             term_dict[j++] = '(';
1269             if (!term_105 (zh->reg->zebra_maps, reg_type,
1270                            &termp, term_dict + j, space_split, term_dst, 0))
1271             {
1272                 *term_sub = 0;
1273                 return ZEBRA_OK;
1274             }
1275             strcat(term_dict, ")");
1276             break;
1277         default:
1278             zh->errCode = 120;
1279             zh->errString = nmem_strdup_i(stream, truncation_value);
1280             return ZEBRA_FAIL;
1281         }
1282         if (attr_ok)
1283         {
1284             char buf[80];
1285             const char *input = term_dict + prefix_len;
1286             esc_str(buf, sizeof(buf), input, strlen(input));
1287         }
1288         if (attr_ok)
1289         {
1290             yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_dict+prefix_len);
1291             r = dict_lookup_grep(zh->reg->dict, term_dict, regex_range,
1292                                  grep_info, &max_pos, init_pos,
1293                                  grep_handle);
1294             if (r)
1295                 yaz_log(YLOG_WARN, "dict_lookup_grep fail %d", r);
1296         }
1297     }
1298     if (!bases_ok)
1299     {
1300         zh->errCode = errCode;
1301         zh->errString = errString;
1302         return ZEBRA_FAIL;
1303     }
1304     *term_sub = termp;
1305     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1306     return ZEBRA_OK;
1307 }
1308
1309
1310 /* convert APT search term to UTF8 */
1311 static int zapt_term_to_utf8 (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1312                               char *termz)
1313 {
1314     size_t sizez;
1315     Z_Term *term = zapt->term;
1316
1317     switch (term->which)
1318     {
1319     case Z_Term_general:
1320         if (zh->iconv_to_utf8 != 0)
1321         {
1322             char *inbuf = term->u.general->buf;
1323             size_t inleft = term->u.general->len;
1324             char *outbuf = termz;
1325             size_t outleft = IT_MAX_WORD-1;
1326             size_t ret;
1327
1328             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1329                         &outbuf, &outleft);
1330             if (ret == (size_t)(-1))
1331             {
1332                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1333                 zh->errCode = 125;
1334                 return -1;
1335             }
1336             *outbuf = 0;
1337         }
1338         else
1339         {
1340             sizez = term->u.general->len;
1341             if (sizez > IT_MAX_WORD-1)
1342                 sizez = IT_MAX_WORD-1;
1343             memcpy (termz, term->u.general->buf, sizez);
1344             termz[sizez] = '\0';
1345         }
1346         break;
1347     case Z_Term_characterString:
1348         sizez = strlen(term->u.characterString);
1349         if (sizez > IT_MAX_WORD-1)
1350             sizez = IT_MAX_WORD-1;
1351         memcpy (termz, term->u.characterString, sizez);
1352         termz[sizez] = '\0';
1353         break;
1354     default:
1355         zh->errCode = 124;
1356         return -1;
1357     }
1358     return 0;
1359 }
1360
1361 /* convert APT SCAN term to internal cmap */
1362 static int trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1363                             char *termz, int reg_type)
1364 {
1365     char termz0[IT_MAX_WORD];
1366
1367     if (zapt_term_to_utf8(zh, zapt, termz0))
1368         return -1;    /* error */
1369     else
1370     {
1371         const char **map;
1372         const char *cp = (const char *) termz0;
1373         const char *cp_end = cp + strlen(cp);
1374         const char *src;
1375         int i = 0;
1376         const char *space_map = NULL;
1377         int len;
1378
1379         while ((len = (cp_end - cp)) > 0)
1380         {
1381             map = zebra_maps_input (zh->reg->zebra_maps, reg_type, &cp, len, 0);
1382             if (**map == *CHR_SPACE)
1383                 space_map = *map;
1384             else
1385             {
1386                 if (i && space_map)
1387                     for (src = space_map; *src; src++)
1388                         termz[i++] = *src;
1389                 space_map = NULL;
1390                 for (src = *map; *src; src++)
1391                     termz[i++] = *src;
1392             }
1393         }
1394         termz[i] = '\0';
1395     }
1396     return 0;
1397 }
1398
1399 char *normalize_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1400                      const char *termz, NMEM stream, unsigned reg_id)
1401 {
1402     WRBUF wrbuf = 0;
1403     AttrType truncation;
1404     int truncation_value;
1405     char *ex_list = 0;
1406
1407     attr_init (&truncation, zapt, 5);
1408     truncation_value = attr_find (&truncation, NULL);
1409
1410     switch (truncation_value)
1411     {
1412     default:
1413         ex_list = "";
1414         break;
1415     case 101:
1416         ex_list = "#";
1417         break;
1418     case 102:
1419     case 103:
1420         ex_list = 0;
1421         break;
1422     case 104:
1423         ex_list = "!#";
1424         break;
1425     case 105:
1426         ex_list = "!*";
1427         break;
1428     }
1429     if (ex_list)
1430         wrbuf = zebra_replace(zh->reg->zebra_maps, reg_id, ex_list,
1431                               termz, strlen(termz));
1432     if (!wrbuf)
1433         return nmem_strdup(stream, termz);
1434     else
1435     {
1436         char *buf = (char*) nmem_malloc(stream, wrbuf_len(wrbuf)+1);
1437         memcpy (buf, wrbuf_buf(wrbuf), wrbuf_len(wrbuf));
1438         buf[wrbuf_len(wrbuf)] = '\0';
1439         return buf;
1440     }
1441 }
1442
1443 static void grep_info_delete (struct grep_info *grep_info)
1444 {
1445 #ifdef TERM_COUNT
1446     xfree(grep_info->term_no);
1447 #endif
1448     xfree (grep_info->isam_p_buf);
1449 }
1450
1451 static int grep_info_prepare (ZebraHandle zh,
1452                               Z_AttributesPlusTerm *zapt,
1453                               struct grep_info *grep_info,
1454                               int reg_type,
1455                               NMEM stream)
1456 {
1457     AttrType termset;
1458     int termset_value_numeric;
1459     const char *termset_value_string;
1460
1461 #ifdef TERM_COUNT
1462     grep_info->term_no = 0;
1463 #endif
1464     grep_info->isam_p_size = 0;
1465     grep_info->isam_p_buf = NULL;
1466     grep_info->zh = zh;
1467     grep_info->reg_type = reg_type;
1468     grep_info->termset = 0;
1469
1470     if (!zapt)
1471         return 0;
1472     attr_init (&termset, zapt, 8);
1473     termset_value_numeric =
1474         attr_find_ex (&termset, NULL, &termset_value_string);
1475     if (termset_value_numeric != -1)
1476     {
1477         char resname[32];
1478         const char *termset_name = 0;
1479         if (termset_value_numeric != -2)
1480         {
1481
1482             sprintf (resname, "%d", termset_value_numeric);
1483             termset_name = resname;
1484         }
1485         else
1486             termset_name = termset_value_string;
1487         yaz_log(log_level_rpn, "creating termset set %s", termset_name);
1488         grep_info->termset = resultSetAdd (zh, termset_name, 1);
1489         if (!grep_info->termset)
1490         {
1491             zh->errCode = 128;
1492             zh->errString = nmem_strdup (stream, termset_name);
1493             return -1;
1494         }
1495     }
1496     return 0;
1497 }
1498
1499
1500 static RSET rpn_search_APT_phrase(ZebraHandle zh,
1501                                   Z_AttributesPlusTerm *zapt,
1502                                   const char *termz_org,
1503                                   oid_value attributeSet,
1504                                   NMEM stream,
1505                                   int reg_type, int complete_flag,
1506                                   const char *rank_type, int xpath_use,
1507                                   int num_bases, char **basenames,
1508                                   NMEM rset_nmem)
1509 {
1510     char term_dst[IT_MAX_WORD+1];
1511     RSET rset[TERM_LIST_LENGTH_MAX], result;
1512     size_t rset_no = 0;
1513     struct grep_info grep_info;
1514     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1515     const char *termp = termz;
1516
1517     *term_dst = 0;
1518     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1519         return 0;
1520     for (; rset_no < sizeof(rset)/sizeof(*rset); rset_no++)
1521     {
1522         yaz_log(log_level_rpn, "APT_phrase termp=%s", termp);
1523         rset[rset_no] = term_trunc(zh, zapt, &termp, attributeSet,
1524                                    stream, &grep_info,
1525                                    reg_type, complete_flag,
1526                                    num_bases, basenames,
1527                                    term_dst, rank_type,
1528                                    xpath_use, rset_nmem);
1529         if (!rset[rset_no])  /* error or no more terms */
1530             break;
1531     }
1532     grep_info_delete (&grep_info);
1533     if (termp)
1534     {
1535         int i;
1536         for (i = 0; i<rset_no; i++)
1537             rset_delete(rset[i]);
1538         return 0;
1539     }
1540     if (rset_no == 0)
1541         return rsnull_create (rset_nmem,key_it_ctrl);
1542     else if (rset_no == 1)
1543         return (rset[0]);
1544     else
1545         result = rsprox_create( rset_nmem, key_it_ctrl, key_it_ctrl->scope,
1546                        rset_no, rset,
1547                        1 /* ordered */, 0 /* exclusion */,
1548                        3 /* relation */, 1 /* distance */);
1549     return result;
1550 }
1551
1552 static RSET rpn_search_APT_or_list(ZebraHandle zh,
1553                                    Z_AttributesPlusTerm *zapt,
1554                                    const char *termz_org,
1555                                    oid_value attributeSet,
1556                                    NMEM stream,
1557                                    int reg_type, int complete_flag,
1558                                    const char *rank_type,
1559                                    int xpath_use,
1560                                    int num_bases, char **basenames,
1561                                    NMEM rset_nmem)
1562 {
1563     char term_dst[IT_MAX_WORD+1];
1564     RSET rset[TERM_LIST_LENGTH_MAX];
1565     size_t rset_no = 0;
1566     struct grep_info grep_info;
1567     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1568     const char *termp = termz;
1569
1570     if (grep_info_prepare(zh, zapt, &grep_info, reg_type, stream))
1571         return 0;
1572     for (; rset_no < sizeof(rset)/sizeof(*rset); rset_no++)
1573     {
1574         yaz_log(log_level_rpn, "APT_or_list termp=%s", termp);
1575         rset[rset_no] = term_trunc(zh, zapt, &termp, attributeSet,
1576                                    stream, &grep_info,
1577                                    reg_type, complete_flag,
1578                                    num_bases, basenames,
1579                                    term_dst, rank_type,
1580                                    xpath_use, rset_nmem);
1581         if (!rset[rset_no])  /* error or no more terms */
1582             break;
1583     }
1584     grep_info_delete (&grep_info);
1585     if (termp)
1586     {
1587         int i;
1588         for (i = 0; i<rset_no; i++)
1589             rset_delete(rset[i]);
1590         return 0;
1591     }
1592     if (rset_no == 0)
1593         return rsnull_create (rset_nmem,key_it_ctrl);
1594     return rsmulti_or_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
1595                              rset_no, rset);
1596 }
1597
1598 static RSET rpn_search_APT_and_list(ZebraHandle zh,
1599                                     Z_AttributesPlusTerm *zapt,
1600                                     const char *termz_org,
1601                                     oid_value attributeSet,
1602                                     NMEM stream,
1603                                     int reg_type, int complete_flag,
1604                                     const char *rank_type,
1605                                     int xpath_use,
1606                                     int num_bases, char **basenames,
1607                                     NMEM rset_nmem)
1608 {
1609     char term_dst[IT_MAX_WORD+1];
1610     RSET rset[TERM_LIST_LENGTH_MAX];
1611     size_t rset_no = 0;
1612     struct grep_info grep_info;
1613     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1614     const char *termp = termz;
1615
1616     if (grep_info_prepare(zh, zapt, &grep_info, reg_type, stream))
1617         return 0;
1618     for (; rset_no < sizeof(rset)/sizeof(*rset); rset_no++)
1619     {
1620         yaz_log(log_level_rpn, "APT_and_list termp=%s", termp);
1621         rset[rset_no] = term_trunc(zh, zapt, &termp, attributeSet,
1622                                    stream, &grep_info,
1623                                    reg_type, complete_flag,
1624                                    num_bases, basenames,
1625                                    term_dst, rank_type,
1626                                    xpath_use, rset_nmem);
1627         if (!rset[rset_no])  /* error or no more terms */
1628             break;
1629     }
1630     grep_info_delete (&grep_info);
1631     if (termp)
1632     {
1633         int i;
1634         for (i = 0; i<rset_no; i++)
1635             rset_delete(rset[i]);
1636         return 0;
1637     }
1638     if (rset_no == 0)
1639         return rsnull_create(rset_nmem, key_it_ctrl);
1640
1641     return rsmulti_and_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
1642                               rset_no, rset);
1643 }
1644
1645 static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1646                             const char **term_sub,
1647                             char *term_dict,
1648                             oid_value attributeSet,
1649                             struct grep_info *grep_info,
1650                             int *max_pos,
1651                             int reg_type,
1652                             char *term_dst,
1653                             int *error_code)
1654 {
1655     AttrType relation;
1656     int relation_value;
1657     int term_value;
1658     int r;
1659     char *term_tmp = term_dict + strlen(term_dict);
1660
1661     *error_code = 0;
1662     attr_init (&relation, zapt, 2);
1663     relation_value = attr_find (&relation, NULL);
1664
1665     yaz_log(log_level_rpn, "numeric relation value=%d", relation_value);
1666
1667     if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1668                   term_dst))
1669         return 0;
1670     term_value = atoi (term_tmp);
1671     switch (relation_value)
1672     {
1673     case 1:
1674         yaz_log(log_level_rpn, "Relation <");
1675         gen_regular_rel(term_tmp, term_value-1, 1);
1676         break;
1677     case 2:
1678         yaz_log(log_level_rpn, "Relation <=");
1679         gen_regular_rel(term_tmp, term_value, 1);
1680         break;
1681     case 4:
1682         yaz_log(log_level_rpn, "Relation >=");
1683         gen_regular_rel(term_tmp, term_value, 0);
1684         break;
1685     case 5:
1686         yaz_log(log_level_rpn, "Relation >");
1687         gen_regular_rel(term_tmp, term_value+1, 0);
1688         break;
1689     case -1:
1690     case 3:
1691         yaz_log(log_level_rpn, "Relation =");
1692         sprintf(term_tmp, "(0*%d)", term_value);
1693         break;
1694     default:
1695         *error_code = 117;
1696         return 0;
1697     }
1698     yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_tmp);
1699     r = dict_lookup_grep(zh->reg->dict, term_dict, 0, grep_info, max_pos,
1700                           0, grep_handle);
1701     if (r)
1702         yaz_log(YLOG_WARN, "dict_lookup_grep fail, rel = gt: %d", r);
1703     yaz_log(log_level_rpn, "%d positions", grep_info->isam_p_indx);
1704     return 1;
1705 }
1706
1707 static ZEBRA_RES numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1708                               const char **term_sub,
1709                               oid_value attributeSet,
1710                               struct grep_info *grep_info,
1711                               int reg_type, int complete_flag,
1712                               int num_bases, char **basenames,
1713                               char *term_dst, int xpath_use, NMEM stream)
1714 {
1715     char term_dict[2*IT_MAX_WORD+2];
1716     int r, base_no;
1717     AttrType use;
1718     int use_value;
1719     const char *use_string = 0;
1720     oid_value curAttributeSet = attributeSet;
1721     const char *termp;
1722     struct rpn_char_map_info rcmi;
1723
1724     int bases_ok = 0;     /* no of databases with OK attribute */
1725     int errCode = 0;      /* err code (if any is not OK) */
1726     char *errString = 0;  /* addinfo */
1727
1728     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1729     attr_init (&use, zapt, 1);
1730     use_value = attr_find_ex (&use, &curAttributeSet, &use_string);
1731
1732     if (use_value == -1)
1733         use_value = 1016;
1734
1735     for (base_no = 0; base_no < num_bases; base_no++)
1736     {
1737         attent attp;
1738         data1_local_attribute id_xpath_attr;
1739         data1_local_attribute *local_attr;
1740         int max_pos, prefix_len = 0;
1741         int relation_error = 0;
1742
1743         termp = *term_sub;
1744         if (use_value == -2)  /* string attribute (assume IDXPATH/any) */
1745         {
1746             use_value = xpath_use;
1747             attp.local_attributes = &id_xpath_attr;
1748             attp.attset_ordinal = VAL_IDXPATH;
1749             id_xpath_attr.next = 0;
1750             id_xpath_attr.local = use_value;
1751         }
1752         else if (curAttributeSet == VAL_IDXPATH)
1753         {
1754             attp.local_attributes = &id_xpath_attr;
1755             attp.attset_ordinal = VAL_IDXPATH;
1756             id_xpath_attr.next = 0;
1757             id_xpath_attr.local = use_value;
1758         }
1759         else
1760         {
1761             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1762                                             use_string)))
1763             {
1764                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1765                       curAttributeSet, use_value, r);
1766                 if (r == -1)
1767                 {
1768                     errCode = 114;
1769                     if (use_string)
1770                         errString = nmem_strdup(stream, use_string);
1771                     else
1772                         errString = nmem_strdup_i (stream, use_value);
1773                 }
1774                 else
1775                     errCode = 121;
1776                 continue;
1777             }
1778         }
1779         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1780         {
1781             zh->errCode = 109; /* Database unavailable */
1782             zh->errString = basenames[base_no];
1783             return -1;
1784         }
1785         for (local_attr = attp.local_attributes; local_attr;
1786              local_attr = local_attr->next)
1787         {
1788             int ord;
1789             char ord_buf[32];
1790             int i, ord_len;
1791
1792             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1793                                               attp.attset_ordinal,
1794                                               local_attr->local);
1795             if (ord < 0)
1796                 continue;
1797             if (prefix_len)
1798                 term_dict[prefix_len++] = '|';
1799             else
1800                 term_dict[prefix_len++] = '(';
1801
1802             ord_len = key_SU_encode (ord, ord_buf);
1803             for (i = 0; i<ord_len; i++)
1804             {
1805                 term_dict[prefix_len++] = 1;
1806                 term_dict[prefix_len++] = ord_buf[i];
1807             }
1808         }
1809         if (!prefix_len)
1810         {
1811             errCode = 114;
1812             errString = nmem_strdup_i(stream, use_value);
1813             continue;
1814         }
1815         bases_ok++;
1816         term_dict[prefix_len++] = ')';
1817         term_dict[prefix_len++] = 1;
1818         term_dict[prefix_len++] = reg_type;
1819         yaz_log(YLOG_DEBUG, "reg_type = %d", term_dict[prefix_len-1]);
1820         term_dict[prefix_len] = '\0';
1821         if (!numeric_relation(zh, zapt, &termp, term_dict,
1822                               attributeSet, grep_info, &max_pos, reg_type,
1823                               term_dst, &relation_error))
1824         {
1825             if (relation_error)
1826             {
1827                 zh->errCode = relation_error;
1828                 zh->errString = 0;
1829                 return ZEBRA_FAIL;
1830             }
1831             *term_sub = 0;
1832             return ZEBRA_OK;
1833         }
1834     }
1835     if (!bases_ok)
1836     {
1837         zh->errCode = errCode;
1838         zh->errString = errString;
1839         return ZEBRA_FAIL;
1840     }
1841     *term_sub = termp;
1842     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1843     return ZEBRA_OK;
1844 }
1845
1846 static RSET rpn_search_APT_numeric(ZebraHandle zh,
1847                                    Z_AttributesPlusTerm *zapt,
1848                                    const char *termz,
1849                                    oid_value attributeSet,
1850                                    NMEM stream,
1851                                    int reg_type, int complete_flag,
1852                                    const char *rank_type, int xpath_use,
1853                                    int num_bases, char **basenames,
1854                                    NMEM rset_nmem)
1855 {
1856     char term_dst[IT_MAX_WORD+1];
1857     const char *termp = termz;
1858     RSET rset[TERM_LIST_LENGTH_MAX];
1859     ZEBRA_RES  r;
1860     size_t rset_no = 0;
1861     struct grep_info grep_info;
1862
1863     yaz_log(log_level_rpn, "APT_numeric t='%s'",termz);
1864     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1865         return 0;
1866     for (; rset_no < sizeof(rset)/sizeof(*rset); rset_no++)
1867     {
1868         yaz_log(YLOG_DEBUG, "APT_numeric termp=%s", termp);
1869         grep_info.isam_p_indx = 0;
1870         r = numeric_term(zh, zapt, &termp, attributeSet, &grep_info,
1871                          reg_type, complete_flag, num_bases, basenames,
1872                          term_dst, xpath_use,
1873                          stream);
1874         if (r == ZEBRA_FAIL || termp == 0)
1875             break;
1876         yaz_log(YLOG_DEBUG, "term: %s", term_dst);
1877         rset[rset_no] = rset_trunc(zh, grep_info.isam_p_buf,
1878                                    grep_info.isam_p_indx, term_dst,
1879                                    strlen(term_dst), rank_type,
1880                                    0 /* preserve position */,
1881                                    zapt->term->which, rset_nmem,
1882                                    key_it_ctrl,key_it_ctrl->scope);
1883         if (!rset[rset_no])
1884             break;
1885     }
1886     grep_info_delete (&grep_info);
1887     if (termp)
1888     {
1889         int i;
1890         for (i = 0; i<rset_no; i++)
1891             rset_delete(rset[i]);
1892         return 0;
1893     }
1894     if (rset_no == 0)
1895         return rsnull_create(rset_nmem,key_it_ctrl);
1896     if (rset_no == 1)
1897         return rset[0];
1898     return rsmulti_and_create(rset_nmem,key_it_ctrl,key_it_ctrl->scope,
1899                               rset_no, rset);
1900 }
1901
1902 static RSET rpn_search_APT_local (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1903                                   const char *termz,
1904                                   oid_value attributeSet,
1905                                   NMEM stream,
1906                                   const char *rank_type, NMEM rset_nmem)
1907 {
1908     RSET result;
1909     RSFD rsfd;
1910     struct it_key key;
1911     int sys;
1912     result = rstemp_create( rset_nmem,key_it_ctrl,key_it_ctrl->scope,
1913                      res_get (zh->res, "setTmpDir"),0 );
1914     rsfd = rset_open (result, RSETF_WRITE);
1915
1916     sys = atoi(termz);
1917     if (sys <= 0)
1918         sys = 1;
1919     key.mem[0] = sys;
1920     key.mem[1] = 1;
1921     key.len = 2;
1922     rset_write (rsfd, &key);
1923     rset_close (rsfd);
1924     return result;
1925 }
1926
1927 static RSET rpn_sort_spec (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1928                            oid_value attributeSet, NMEM stream,
1929                            Z_SortKeySpecList *sort_sequence,
1930                            const char *rank_type)
1931 {
1932     int i;
1933     int sort_relation_value;
1934     AttrType sort_relation_type;
1935     int use_value;
1936     AttrType use_type;
1937     Z_SortKeySpec *sks;
1938     Z_SortKey *sk;
1939     Z_AttributeElement *ae;
1940     int oid[OID_SIZE];
1941     oident oe;
1942     char termz[20];
1943
1944     attr_init (&sort_relation_type, zapt, 7);
1945     sort_relation_value = attr_find (&sort_relation_type, &attributeSet);
1946
1947     attr_init (&use_type, zapt, 1);
1948     use_value = attr_find (&use_type, &attributeSet);
1949
1950     if (!sort_sequence->specs)
1951     {
1952         sort_sequence->num_specs = 10;
1953         sort_sequence->specs = (Z_SortKeySpec **)
1954             nmem_malloc(stream, sort_sequence->num_specs *
1955                          sizeof(*sort_sequence->specs));
1956         for (i = 0; i<sort_sequence->num_specs; i++)
1957             sort_sequence->specs[i] = 0;
1958     }
1959     if (zapt->term->which != Z_Term_general)
1960         i = 0;
1961     else
1962         i = atoi_n ((char *) zapt->term->u.general->buf,
1963                     zapt->term->u.general->len);
1964     if (i >= sort_sequence->num_specs)
1965         i = 0;
1966     sprintf (termz, "%d", i);
1967
1968     oe.proto = PROTO_Z3950;
1969     oe.oclass = CLASS_ATTSET;
1970     oe.value = attributeSet;
1971     if (!oid_ent_to_oid (&oe, oid))
1972         return 0;
1973
1974     sks = (Z_SortKeySpec *) nmem_malloc(stream, sizeof(*sks));
1975     sks->sortElement = (Z_SortElement *)
1976         nmem_malloc(stream, sizeof(*sks->sortElement));
1977     sks->sortElement->which = Z_SortElement_generic;
1978     sk = sks->sortElement->u.generic = (Z_SortKey *)
1979         nmem_malloc(stream, sizeof(*sk));
1980     sk->which = Z_SortKey_sortAttributes;
1981     sk->u.sortAttributes = (Z_SortAttributes *)
1982         nmem_malloc(stream, sizeof(*sk->u.sortAttributes));
1983
1984     sk->u.sortAttributes->id = oid;
1985     sk->u.sortAttributes->list = (Z_AttributeList *)
1986         nmem_malloc(stream, sizeof(*sk->u.sortAttributes->list));
1987     sk->u.sortAttributes->list->num_attributes = 1;
1988     sk->u.sortAttributes->list->attributes = (Z_AttributeElement **)
1989         nmem_malloc(stream, sizeof(*sk->u.sortAttributes->list->attributes));
1990     ae = *sk->u.sortAttributes->list->attributes = (Z_AttributeElement *)
1991         nmem_malloc(stream, sizeof(**sk->u.sortAttributes->list->attributes));
1992     ae->attributeSet = 0;
1993     ae->attributeType = (int *)
1994         nmem_malloc(stream, sizeof(*ae->attributeType));
1995     *ae->attributeType = 1;
1996     ae->which = Z_AttributeValue_numeric;
1997     ae->value.numeric = (int *)
1998         nmem_malloc(stream, sizeof(*ae->value.numeric));
1999     *ae->value.numeric = use_value;
2000
2001     sks->sortRelation = (int *)
2002         nmem_malloc(stream, sizeof(*sks->sortRelation));
2003     if (sort_relation_value == 1)
2004         *sks->sortRelation = Z_SortKeySpec_ascending;
2005     else if (sort_relation_value == 2)
2006         *sks->sortRelation = Z_SortKeySpec_descending;
2007     else
2008         *sks->sortRelation = Z_SortKeySpec_ascending;
2009
2010     sks->caseSensitivity = (int *)
2011         nmem_malloc(stream, sizeof(*sks->caseSensitivity));
2012     *sks->caseSensitivity = 0;
2013
2014     sks->which = Z_SortKeySpec_null;
2015     sks->u.null = odr_nullval ();
2016     sort_sequence->specs[i] = sks;
2017     return rsnull_create (NULL,key_it_ctrl);
2018         /* FIXME - nmem?? */
2019 }
2020
2021
2022 static int parse_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2023                        oid_value attributeSet,
2024                        struct xpath_location_step *xpath, int max, NMEM mem)
2025 {
2026     oid_value curAttributeSet = attributeSet;
2027     AttrType use;
2028     const char *use_string = 0;
2029
2030     attr_init (&use, zapt, 1);
2031     attr_find_ex (&use, &curAttributeSet, &use_string);
2032
2033     if (!use_string || *use_string != '/')
2034         return -1;
2035
2036     return zebra_parse_xpath_str(use_string, xpath, max, mem);
2037 }
2038
2039
2040
2041 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
2042                         int reg_type, const char *term, int use,
2043                         oid_value curAttributeSet, NMEM rset_nmem)
2044 {
2045     RSET rset;
2046     struct grep_info grep_info;
2047     char term_dict[2048];
2048     char ord_buf[32];
2049     int prefix_len = 0;
2050     int ord = zebraExplain_lookup_attr_su(zh->reg->zei, curAttributeSet, use);
2051     int ord_len, i, r, max_pos;
2052     int term_type = Z_Term_characterString;
2053     const char *flags = "void";
2054
2055     if (grep_info_prepare (zh, 0 /* zapt */, &grep_info, '0', stream))
2056         return rsnull_create (rset_nmem,key_it_ctrl);
2057
2058     if (ord < 0)
2059         return rsnull_create (rset_nmem,key_it_ctrl);
2060     if (prefix_len)
2061         term_dict[prefix_len++] = '|';
2062     else
2063         term_dict[prefix_len++] = '(';
2064
2065     ord_len = key_SU_encode (ord, ord_buf);
2066     for (i = 0; i<ord_len; i++)
2067     {
2068         term_dict[prefix_len++] = 1;
2069         term_dict[prefix_len++] = ord_buf[i];
2070     }
2071     term_dict[prefix_len++] = ')';
2072     term_dict[prefix_len++] = 1;
2073     term_dict[prefix_len++] = reg_type;
2074
2075     strcpy(term_dict+prefix_len, term);
2076
2077     grep_info.isam_p_indx = 0;
2078     r = dict_lookup_grep(zh->reg->dict, term_dict, 0,
2079                           &grep_info, &max_pos, 0, grep_handle);
2080     yaz_log (YLOG_DEBUG, "%s %d positions", term,
2081              grep_info.isam_p_indx);
2082     rset = rset_trunc(zh, grep_info.isam_p_buf,
2083                       grep_info.isam_p_indx, term, strlen(term),
2084                       flags, 1, term_type,rset_nmem,
2085                       key_it_ctrl, key_it_ctrl->scope);
2086     grep_info_delete (&grep_info);
2087     return rset;
2088 }
2089
2090 static RSET rpn_search_xpath (ZebraHandle zh,
2091                               oid_value attributeSet,
2092                               int num_bases, char **basenames,
2093                               NMEM stream, const char *rank_type, RSET rset,
2094                               int xpath_len, struct xpath_location_step *xpath,
2095                               NMEM rset_nmem)
2096 {
2097     oid_value curAttributeSet = attributeSet;
2098     int base_no;
2099     int i;
2100
2101     if (xpath_len < 0)
2102         return rset;
2103
2104     yaz_log (YLOG_DEBUG, "xpath len=%d", xpath_len);
2105     for (i = 0; i<xpath_len; i++)
2106     {
2107         yaz_log (log_level_rpn, "XPATH %d %s", i, xpath[i].part);
2108
2109     }
2110
2111     curAttributeSet = VAL_IDXPATH;
2112
2113     /*
2114       //a    ->    a/.*
2115       //a/b  ->    b/a/.*
2116       /a     ->    a/
2117       /a/b   ->    b/a/
2118
2119       /      ->    none
2120
2121    a[@attr = value]/b[@other = othervalue]
2122
2123  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
2124  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
2125  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
2126  /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y)
2127  /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y)
2128  /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x)
2129
2130     */
2131
2132     dict_grep_cmap (zh->reg->dict, 0, 0);
2133
2134     for (base_no = 0; base_no < num_bases; base_no++)
2135     {
2136         int level = xpath_len;
2137         int first_path = 1;
2138
2139         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2140         {
2141             zh->errCode = 109; /* Database unavailable */
2142             zh->errString = basenames[base_no];
2143             return rset;
2144         }
2145         while (--level >= 0)
2146         {
2147             char xpath_rev[128];
2148             int i, len;
2149             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
2150
2151             *xpath_rev = 0;
2152             len = 0;
2153             for (i = level; i >= 1; --i)
2154             {
2155                 const char *cp = xpath[i].part;
2156                 if (*cp)
2157                 {
2158                     for (;*cp; cp++)
2159                         if (*cp == '*')
2160                         {
2161                             memcpy (xpath_rev + len, "[^/]*", 5);
2162                             len += 5;
2163                         }
2164                         else if (*cp == ' ')
2165                         {
2166
2167                             xpath_rev[len++] = 1;
2168                             xpath_rev[len++] = ' ';
2169                         }
2170
2171                         else
2172                             xpath_rev[len++] = *cp;
2173                     xpath_rev[len++] = '/';
2174                 }
2175                 else if (i == 1)  /* // case */
2176                 {
2177                     xpath_rev[len++] = '.';
2178                     xpath_rev[len++] = '*';
2179                 }
2180             }
2181             xpath_rev[len] = 0;
2182
2183             if (xpath[level].predicate &&
2184                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
2185                 xpath[level].predicate->u.relation.name[0])
2186             {
2187                 WRBUF wbuf = wrbuf_alloc();
2188                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
2189                 if (xpath[level].predicate->u.relation.value)
2190                 {
2191                     const char *cp = xpath[level].predicate->u.relation.value;
2192                     wrbuf_putc(wbuf, '=');
2193
2194                     while (*cp)
2195                     {
2196                         if (strchr(REGEX_CHARS, *cp))
2197                             wrbuf_putc(wbuf, '\\');
2198                         wrbuf_putc(wbuf, *cp);
2199                         cp++;
2200                     }
2201                 }
2202                 wrbuf_puts(wbuf, "");
2203                 rset_attr = xpath_trunc(
2204                     zh, stream, '0', wrbuf_buf(wbuf), 3,
2205                     curAttributeSet,rset_nmem);
2206                 wrbuf_free(wbuf, 1);
2207             }
2208             else
2209             {
2210                 if (!first_path)
2211                     continue;
2212             }
2213             yaz_log (log_level_rpn, "xpath_rev (%d) = %s", level, xpath_rev);
2214             if (strlen(xpath_rev))
2215             {
2216                 rset_start_tag = xpath_trunc(zh, stream, '0',
2217                         xpath_rev, 1, curAttributeSet, rset_nmem);
2218
2219                 rset_end_tag = xpath_trunc(zh, stream, '0',
2220                         xpath_rev, 2, curAttributeSet, rset_nmem);
2221
2222                 rset = rsbetween_create(rset_nmem, key_it_ctrl,
2223                                         key_it_ctrl->scope,
2224                                         rset_start_tag, rset,
2225                                         rset_end_tag, rset_attr);
2226             }
2227             first_path = 0;
2228         }
2229     }
2230
2231     return rset;
2232 }
2233
2234
2235
2236 static RSET rpn_search_APT(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2237                            oid_value attributeSet, NMEM stream,
2238                            Z_SortKeySpecList *sort_sequence,
2239                            int num_bases, char **basenames,
2240                            NMEM rset_nmem)
2241 {
2242     unsigned reg_id;
2243     char *search_type = NULL;
2244     char rank_type[128];
2245     int complete_flag;
2246     int sort_flag;
2247     char termz[IT_MAX_WORD+1];
2248     RSET rset = 0;
2249     int xpath_len;
2250     int xpath_use = 0;
2251     struct xpath_location_step xpath[10];
2252
2253     if (!log_level_set)
2254     {
2255         log_level_rpn = yaz_log_module_level("rpn");
2256         log_level_set = 1;
2257     }
2258     zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2259                     rank_type, &complete_flag, &sort_flag);
2260
2261     yaz_log(YLOG_DEBUG, "reg_id=%c", reg_id);
2262     yaz_log(YLOG_DEBUG, "complete_flag=%d", complete_flag);
2263     yaz_log(YLOG_DEBUG, "search_type=%s", search_type);
2264     yaz_log(YLOG_DEBUG, "rank_type=%s", rank_type);
2265
2266     if (zapt_term_to_utf8(zh, zapt, termz))
2267         return 0;
2268
2269     if (sort_flag)
2270         return rpn_sort_spec(zh, zapt, attributeSet, stream, sort_sequence,
2271                              rank_type);
2272     xpath_len = parse_xpath(zh, zapt, attributeSet, xpath, 10, stream);
2273     if (xpath_len >= 0)
2274     {
2275         xpath_use = 1016;
2276         if (xpath[xpath_len-1].part[0] == '@')
2277             xpath_use = 1015;
2278     }
2279
2280     if (!strcmp (search_type, "phrase"))
2281     {
2282         rset = rpn_search_APT_phrase(zh, zapt, termz, attributeSet, stream,
2283                                      reg_id, complete_flag, rank_type,
2284                                      xpath_use,
2285                                      num_bases, basenames, rset_nmem);
2286     }
2287     else if (!strcmp (search_type, "and-list"))
2288     {
2289         rset = rpn_search_APT_and_list(zh, zapt, termz, attributeSet, stream,
2290                                        reg_id, complete_flag, rank_type,
2291                                        xpath_use,
2292                                        num_bases, basenames, rset_nmem);
2293     }
2294     else if (!strcmp (search_type, "or-list"))
2295     {
2296         rset = rpn_search_APT_or_list(zh, zapt, termz, attributeSet, stream,
2297                                       reg_id, complete_flag, rank_type,
2298                                       xpath_use,
2299                                       num_bases, basenames, rset_nmem);
2300     }
2301     else if (!strcmp (search_type, "local"))
2302     {
2303         rset = rpn_search_APT_local(zh, zapt, termz, attributeSet, stream,
2304                                     rank_type, rset_nmem);
2305     }
2306     else if (!strcmp (search_type, "numeric"))
2307     {
2308         rset = rpn_search_APT_numeric(zh, zapt, termz, attributeSet, stream,
2309                                       reg_id, complete_flag, rank_type,
2310                                       xpath_use,
2311                                       num_bases, basenames, rset_nmem);
2312     }
2313     else if (!strcmp (search_type, "always"))
2314     {
2315         rset = 0;
2316     }
2317     else
2318     {
2319         zh->errCode = 118;
2320         return 0;
2321     }
2322     return rpn_search_xpath (zh, attributeSet, num_bases, basenames,
2323                              stream, rank_type, rset,
2324                              xpath_len, xpath, rset_nmem);
2325 }
2326
2327 RSET rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2328                           oid_value attributeSet,
2329                           NMEM stream, NMEM rset_nmem,
2330                           Z_SortKeySpecList *sort_sequence,
2331                           int num_bases, char **basenames)
2332 {
2333     RSET r = NULL;
2334     if (zs->which == Z_RPNStructure_complex)
2335     {
2336         Z_Operator *zop = zs->u.complex->roperator;
2337         RSET rsets[2]; /* l and r argument */
2338
2339         rsets[0] = rpn_search_structure(zh, zs->u.complex->s1,
2340                                         attributeSet, stream, rset_nmem,
2341                                         sort_sequence,
2342                                         num_bases, basenames);
2343         if (rsets[0] == NULL)
2344             return NULL;
2345         rsets[1] = rpn_search_structure(zh, zs->u.complex->s2,
2346                                         attributeSet, stream, rset_nmem,
2347                                         sort_sequence,
2348                                         num_bases, basenames);
2349         if (rsets[1] == NULL)
2350         {
2351             rset_delete(rsets[0]);
2352             return NULL;
2353         }
2354
2355         switch (zop->which)
2356         {
2357         case Z_Operator_and:
2358             r = rsmulti_and_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
2359                                    2, rsets);
2360             break;
2361         case Z_Operator_or:
2362             r = rsmulti_or_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
2363                                   2, rsets);
2364             break;
2365         case Z_Operator_and_not:
2366             r = rsbool_create_not(rset_nmem,key_it_ctrl, key_it_ctrl->scope,
2367                     rsets[0], rsets[1]);
2368             break;
2369         case Z_Operator_prox:
2370             if (zop->u.prox->which != Z_ProximityOperator_known)
2371             {
2372                 zh->errCode = 132;
2373                 return NULL;
2374             }
2375             if (*zop->u.prox->u.known != Z_ProxUnit_word)
2376             {
2377                 char *val = (char *) nmem_malloc(stream, 16);
2378                 zh->errCode = 132;
2379                 zh->errString = val;
2380                 sprintf (val, "%d", *zop->u.prox->u.known);
2381                 return NULL;
2382             }
2383             else
2384             {
2385                 /* new / old prox */
2386                 r = rsprox_create(rset_nmem,key_it_ctrl,key_it_ctrl->scope,
2387                          2, rsets,
2388                          *zop->u.prox->ordered,
2389                          (!zop->u.prox->exclusion ?
2390                               0 : *zop->u.prox->exclusion),
2391                          *zop->u.prox->relationType,
2392                          *zop->u.prox->distance );
2393             }
2394             break;
2395         default:
2396             zh->errCode = 110;
2397             return NULL;
2398         }
2399     }
2400     else if (zs->which == Z_RPNStructure_simple)
2401     {
2402         if (zs->u.simple->which == Z_Operand_APT)
2403         {
2404             yaz_log(YLOG_DEBUG, "rpn_search_APT");
2405             r = rpn_search_APT(zh, zs->u.simple->u.attributesPlusTerm,
2406                                attributeSet, stream, sort_sequence,
2407                                num_bases, basenames,rset_nmem);
2408         }
2409         else if (zs->u.simple->which == Z_Operand_resultSetId)
2410         {
2411             yaz_log(YLOG_DEBUG, "rpn_search_ref");
2412             r = resultSetRef (zh, zs->u.simple->u.resultSetId);
2413             if (!r)
2414             {
2415                 zh->errCode = 30;
2416                 zh->errString =
2417                     nmem_strdup (stream, zs->u.simple->u.resultSetId);
2418                 return 0;
2419             }
2420             rset_dup(r);
2421         }
2422         else
2423         {
2424             zh->errCode = 3;
2425             return 0;
2426         }
2427     }
2428     else
2429     {
2430         zh->errCode = 3;
2431         return 0;
2432     }
2433     return r;
2434 }
2435
2436 struct scan_info_entry {
2437     char *term;
2438     ISAM_P isam_p;
2439 };
2440
2441 struct scan_info {
2442     struct scan_info_entry *list;
2443     ODR odr;
2444     int before, after;
2445     char prefix[20];
2446 };
2447
2448 static int scan_handle (char *name, const char *info, int pos, void *client)
2449 {
2450     int len_prefix, idx;
2451     struct scan_info *scan_info = (struct scan_info *) client;
2452
2453     len_prefix = strlen(scan_info->prefix);
2454     if (memcmp (name, scan_info->prefix, len_prefix))
2455         return 1;
2456     if (pos > 0)        idx = scan_info->after - pos + scan_info->before;
2457     else
2458         idx = - pos - 1;
2459     scan_info->list[idx].term = (char *)
2460         odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
2461     strcpy(scan_info->list[idx].term, name + len_prefix);
2462     assert (*info == sizeof(ISAM_P));
2463     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
2464     return 0;
2465 }
2466
2467 static void scan_term_untrans (ZebraHandle zh, NMEM stream, int reg_type,
2468                                char **dst, const char *src)
2469 {
2470     char term_src[IT_MAX_WORD];
2471     char term_dst[IT_MAX_WORD];
2472
2473     term_untrans (zh, reg_type, term_src, src);
2474
2475     if (zh->iconv_from_utf8 != 0)
2476     {
2477         int len;
2478         char *inbuf = term_src;
2479         size_t inleft = strlen(term_src);
2480         char *outbuf = term_dst;
2481         size_t outleft = sizeof(term_dst)-1;
2482         size_t ret;
2483
2484         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2485                          &outbuf, &outleft);
2486         if (ret == (size_t)(-1))
2487             len = 0;
2488         else
2489             len = outbuf - term_dst;
2490         *dst = nmem_malloc(stream, len + 1);
2491         if (len > 0)
2492             memcpy (*dst, term_dst, len);
2493         (*dst)[len] = '\0';
2494     }
2495     else
2496         *dst = nmem_strdup(stream, term_src);
2497 }
2498
2499 static void count_set (RSET r, int *count)
2500 {
2501     zint psysno = 0;
2502     int kno = 0;
2503     struct it_key key;
2504     RSFD rfd;
2505
2506     yaz_log(YLOG_DEBUG, "count_set");
2507
2508     *count = 0;
2509     rfd = rset_open (r, RSETF_READ);
2510     while (rset_read (rfd, &key,0 /* never mind terms */))
2511     {
2512         if (key.mem[0] != psysno)
2513         {
2514             psysno = key.mem[0];
2515             (*count)++;
2516         }
2517         kno++;
2518     }
2519     rset_close (rfd);
2520     yaz_log(YLOG_DEBUG, "%d keys, %d records", kno, *count);
2521 }
2522
2523 ZEBRA_RES rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2524                    oid_value attributeset,
2525                    int num_bases, char **basenames,
2526                    int *position, int *num_entries, ZebraScanEntry **list,
2527                    int *is_partial, RSET limit_set, int return_zero)
2528 {
2529     int i;
2530     int pos = *position;
2531     int num = *num_entries;
2532     int before;
2533     int after;
2534     int base_no;
2535     char termz[IT_MAX_WORD+20];
2536     AttrType use;
2537     int use_value;
2538     const char *use_string = 0;
2539     struct scan_info *scan_info_array;
2540     ZebraScanEntry *glist;
2541     int ords[32], ord_no = 0;
2542     int ptr[32];
2543
2544     int bases_ok = 0;     /* no of databases with OK attribute */
2545     int errCode = 0;      /* err code (if any is not OK) */
2546     char *errString = 0;  /* addinfo */
2547
2548     unsigned reg_id;
2549     char *search_type = NULL;
2550     char rank_type[128];
2551     int complete_flag;
2552     int sort_flag;
2553     NMEM rset_nmem = NULL;
2554
2555     *list = 0;
2556     *is_partial = 0;
2557
2558     if (attributeset == VAL_NONE)
2559         attributeset = VAL_BIB1;
2560
2561     if (!limit_set)
2562     {
2563         AttrType termset;
2564         int termset_value_numeric;
2565         const char *termset_value_string;
2566         attr_init (&termset, zapt, 8);
2567         termset_value_numeric =
2568             attr_find_ex (&termset, NULL, &termset_value_string);
2569         if (termset_value_numeric != -1)
2570         {
2571             char resname[32];
2572             const char *termset_name = 0;
2573
2574             if (termset_value_numeric != -2)
2575             {
2576
2577                 sprintf (resname, "%d", termset_value_numeric);
2578                 termset_name = resname;
2579             }
2580             else
2581                 termset_name = termset_value_string;
2582
2583             limit_set = resultSetRef (zh, termset_name);
2584         }
2585     }
2586
2587     yaz_log (YLOG_DEBUG, "position = %d, num = %d set=%d",
2588              pos, num, attributeset);
2589
2590     attr_init (&use, zapt, 1);
2591     use_value = attr_find_ex (&use, &attributeset, &use_string);
2592
2593     if (zebra_maps_attr (zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2594                          rank_type, &complete_flag, &sort_flag))
2595     {
2596         *num_entries = 0;
2597         zh->errCode = 113;
2598         return ZEBRA_FAIL;
2599     }
2600     yaz_log (YLOG_DEBUG, "use_value = %d", use_value);
2601
2602     if (use_value == -1)
2603         use_value = 1016;
2604     for (base_no = 0; base_no < num_bases && ord_no < 32; base_no++)
2605     {
2606         data1_local_attribute *local_attr;
2607         attent attp;
2608         int ord;
2609
2610         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2611         {
2612             zh->errString = basenames[base_no];
2613             zh->errCode = 109; /* Database unavailable */
2614             *num_entries = 0;
2615             return ZEBRA_FAIL;
2616         }
2617
2618         if (use_string &&
2619             (ord = zebraExplain_lookup_attr_str(zh->reg->zei,
2620                                                 use_string)) >= 0)
2621         {
2622             /* we have a match for a raw string attribute */
2623             if (ord > 0)
2624                 ords[ord_no++] = ord;
2625             attp.local_attributes = 0;  /* no more attributes */
2626         }
2627         else
2628         {
2629             int r;
2630
2631             if ((r = att_getentbyatt (zh, &attp, attributeset, use_value,
2632                                       use_string)))
2633             {
2634                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
2635                         attributeset, use_value);
2636                 if (r == -1)
2637                 {
2638                     errCode = 114;
2639                     if (use_string)
2640                         errString = odr_strdup(stream, use_string);
2641                     else
2642                     {
2643                         char val_str[32];
2644                         sprintf (val_str, "%d", use_value);
2645                         errString = odr_strdup(stream, val_str);
2646                     }
2647                 }
2648                 else
2649                     errCode = 121;
2650                 continue;
2651             }
2652         }
2653         bases_ok++;
2654         for (local_attr = attp.local_attributes; local_attr && ord_no < 32;
2655              local_attr = local_attr->next)
2656         {
2657             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
2658                                               attp.attset_ordinal,
2659                                               local_attr->local);
2660             if (ord > 0)
2661                 ords[ord_no++] = ord;
2662         }
2663     }
2664     if (!bases_ok && errCode)
2665     {
2666         zh->errCode = errCode;
2667         zh->errString = errString;
2668         *num_entries = 0;
2669         return ZEBRA_FAIL;
2670     }
2671     if (ord_no == 0)
2672     {
2673         *num_entries = 0;
2674         return ZEBRA_OK;
2675     }
2676     /* prepare dictionary scanning */
2677     before = pos-1;
2678     after = 1+num-pos;
2679     scan_info_array = (struct scan_info *)
2680         odr_malloc(stream, ord_no * sizeof(*scan_info_array));
2681     for (i = 0; i < ord_no; i++)
2682     {
2683         int j, prefix_len = 0;
2684         int before_tmp = before, after_tmp = after;
2685         struct scan_info *scan_info = scan_info_array + i;
2686         struct rpn_char_map_info rcmi;
2687
2688         rpn_char_map_prepare (zh->reg, reg_id, &rcmi);
2689
2690         scan_info->before = before;
2691         scan_info->after = after;
2692         scan_info->odr = stream;
2693
2694         scan_info->list = (struct scan_info_entry *)
2695             odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
2696         for (j = 0; j<before+after; j++)
2697             scan_info->list[j].term = NULL;
2698
2699         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2700         termz[prefix_len++] = reg_id;
2701         termz[prefix_len] = 0;
2702         strcpy(scan_info->prefix, termz);
2703
2704         if (trans_scan_term(zh, zapt, termz+prefix_len, reg_id))
2705             return ZEBRA_FAIL;
2706
2707         dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
2708                   scan_info, scan_handle);
2709     }
2710     glist = (ZebraScanEntry *)
2711         odr_malloc(stream, (before+after)*sizeof(*glist));
2712
2713     rset_nmem = nmem_create();
2714
2715     /* consider terms after main term */
2716     for (i = 0; i < ord_no; i++)
2717         ptr[i] = before;
2718
2719     *is_partial = 0;
2720     for (i = 0; i<after; i++)
2721     {
2722         int j, j0 = -1;
2723         const char *mterm = NULL;
2724         const char *tst;
2725         RSET rset;
2726
2727         for (j = 0; j < ord_no; j++)
2728         {
2729             if (ptr[j] < before+after &&
2730                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2731                 (!mterm || strcmp (tst, mterm) < 0))
2732             {
2733                 j0 = j;
2734                 mterm = tst;
2735             }
2736         }
2737         if (j0 == -1)
2738             break;
2739         scan_term_untrans (zh, stream->mem, reg_id,
2740                            &glist[i+before].term, mterm);
2741         rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2742                           glist[i+before].term, strlen(glist[i+before].term),
2743                           NULL, 0, zapt->term->which, rset_nmem,
2744                           key_it_ctrl,key_it_ctrl->scope);
2745         ptr[j0]++;
2746         for (j = j0+1; j<ord_no; j++)
2747         {
2748             if (ptr[j] < before+after &&
2749                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2750                 !strcmp (tst, mterm))
2751             {
2752                 RSET rsets[2];
2753
2754                 rsets[0] = rset;
2755                 rsets[1] =
2756                     rset_trunc(zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
2757                                glist[i+before].term,
2758                                strlen(glist[i+before].term), NULL, 0,
2759                                zapt->term->which,rset_nmem,
2760                                key_it_ctrl, key_it_ctrl->scope);
2761                 rset = rsmulti_or_create(rset_nmem, key_it_ctrl,
2762                                          2, key_it_ctrl->scope, rsets);
2763                 ptr[j]++;
2764             }
2765         }
2766         if (limit_set)
2767         {
2768             RSET rsets[2];
2769             rsets[0] = rset;
2770             rsets[1] = rset_dup(limit_set);
2771
2772             rset = rsmulti_and_create(rset_nmem, key_it_ctrl,
2773                                       key_it_ctrl->scope, 2, rsets);
2774         }
2775         count_set(rset, &glist[i+before].occurrences);
2776         rset_delete(rset);
2777     }
2778     if (i < after)
2779     {
2780         *num_entries -= (after-i);
2781         *is_partial = 1;
2782     }
2783
2784     /* consider terms before main term */
2785     for (i = 0; i<ord_no; i++)
2786         ptr[i] = 0;
2787
2788     for (i = 0; i<before; i++)
2789     {
2790         int j, j0 = -1;
2791         const char *mterm = NULL;
2792         const char *tst;
2793         RSET rset;
2794
2795         for (j = 0; j <ord_no; j++)
2796         {
2797             if (ptr[j] < before &&
2798                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2799                 (!mterm || strcmp (tst, mterm) > 0))
2800             {
2801                 j0 = j;
2802                     mterm = tst;
2803             }
2804         }
2805         if (j0 == -1)
2806             break;
2807
2808         scan_term_untrans (zh, stream->mem, reg_id,
2809                            &glist[before-1-i].term, mterm);
2810
2811         rset = rset_trunc
2812             (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
2813              glist[before-1-i].term, strlen(glist[before-1-i].term),
2814              NULL, 0, zapt->term->which,rset_nmem,
2815              key_it_ctrl,key_it_ctrl->scope);
2816
2817         ptr[j0]++;
2818
2819         for (j = j0+1; j<ord_no; j++)
2820         {
2821             if (ptr[j] < before &&
2822                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2823                 !strcmp (tst, mterm))
2824             {
2825                 RSET rsets[2];
2826
2827                 rsets[0] = rset;
2828                 rsets[1] = rset_trunc(
2829                     zh,
2830                     &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
2831                     glist[before-1-i].term,
2832                     strlen(glist[before-1-i].term), NULL, 0,
2833                     zapt->term->which, rset_nmem,
2834                     key_it_ctrl, key_it_ctrl->scope);
2835                 rset = rsmulti_or_create(rset_nmem, key_it_ctrl,
2836                                          2, key_it_ctrl->scope, rsets);
2837
2838                 ptr[j]++;
2839             }
2840         }
2841         if (limit_set)
2842         {
2843             RSET rsets[2];
2844             rsets[0] = rset;
2845             rsets[1] = rset_dup(limit_set);
2846
2847             rset = rsmulti_and_create(rset_nmem, key_it_ctrl,
2848                                       key_it_ctrl->scope, 2, rsets);
2849         }
2850         count_set (rset, &glist[before-1-i].occurrences);
2851         rset_delete (rset);
2852     }
2853     i = before-i;
2854     if (i)
2855     {
2856         *is_partial = 1;
2857         *position -= i;
2858         *num_entries -= i;
2859     }
2860
2861     nmem_destroy(rset_nmem);
2862     *list = glist + i;               /* list is set to first 'real' entry */
2863
2864     yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
2865             *position, *num_entries);
2866     return ZEBRA_OK;
2867 }
2868