index/zrpn.c

   1 /* $Id: zrpn.c,v 1.141.2.15 2006-08-14 10:39:01 adam Exp $
   2    Copyright (C) 1995-2005
   3    Index Data Aps
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with this program; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 */
  22
  23
  24 #include <stdio.h>
  25 #include <assert.h>
  26 #ifdef WIN32
  27 #include <io.h>
  28 #else
  29 #include <unistd.h>
  30 #endif
  31 #include <ctype.h>
  32
  33 #include "index.h"
  34 #include <zebra_xpath.h>
  35
  36 #include <charmap.h>
  37 #include <rstemp.h>
  38 #include <rsnull.h>
  39 #include <rsbool.h>
  40 #include <rsbetween.h>
  41 #include <rsprox.h>
  42
  43 struct rpn_char_map_info {
  44     ZebraMaps zm;
  45     int reg_type;
  46 };
  47
  48 typedef struct {
  49     int type;
  50     int major;
  51     int minor;
  52     Z_AttributesPlusTerm *zapt;
  53 } AttrType;
  54
  55
  56 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  57 {
  58     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  59     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  60 #if 0
  61     if (out && *out)
  62     {
  63         const char *outp = *out;
  64         yaz_log(LOG_LOG, "---");
  65         while (*outp)
  66         {
  67             yaz_log(LOG_LOG, "%02X", *outp);
  68             outp++;
  69         }
  70     }
  71 #endif
  72     return out;
  73 }
  74
  75 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  76                                   struct rpn_char_map_info *map_info)
  77 {
  78     map_info->zm = reg->zebra_maps;
  79     map_info->reg_type = reg_type;
  80     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  81 }
  82
  83 static int attr_find_ex(AttrType *src, oid_value *attributeSetP,
  84                          const char **string_value)
  85 {
  86     int num_attributes;
  87
  88     num_attributes = src->zapt->attributes->num_attributes;
  89     while (src->major < num_attributes)
  90     {
  91         Z_AttributeElement *element;
  92
  93         element = src->zapt->attributes->attributes[src->major];
  94         if (src->type == *element->attributeType)
  95         {
  96             switch (element->which)
  97             {
  98             case Z_AttributeValue_numeric:
  99                 ++(src->major);
 100                 if (element->attributeSet && attributeSetP)
 101                 {
 102                     oident *attrset;
 103
 104                     attrset = oid_getentbyoid(element->attributeSet);
 105                     *attributeSetP = attrset->value;
 106                 }
 107                 return *element->value.numeric;
 108                 break;
 109             case Z_AttributeValue_complex:
 110                 if (src->minor >= element->value.complex->num_list)
 111                     break;
 112                 if (element->attributeSet && attributeSetP)
 113                 {
 114                     oident *attrset;
 115
 116                     attrset = oid_getentbyoid(element->attributeSet);
 117                     *attributeSetP = attrset->value;
 118                 }
 119                 if (element->value.complex->list[src->minor]->which ==
 120                     Z_StringOrNumeric_numeric)
 121                 {
 122                     ++(src->minor);
 123                     return
 124                         *element->value.complex->list[src->minor-1]->u.numeric;
 125                 }
 126                 else if (element->value.complex->list[src->minor]->which ==
 127                          Z_StringOrNumeric_string)
 128                 {
 129                     if (!string_value)
 130                         break;
 131                     ++(src->minor);
 132                     *string_value =
 133                         element->value.complex->list[src->minor-1]->u.string;
 134                     return -2;
 135                 }
 136                 else
 137                     break;
 138             default:
 139                 assert(0);
 140             }
 141         }
 142         ++(src->major);
 143     }
 144     return -1;
 145 }
 146
 147 static int attr_find(AttrType *src, oid_value *attributeSetP)
 148 {
 149     return attr_find_ex(src, attributeSetP, 0);
 150 }
 151
 152 static void attr_init(AttrType *src, Z_AttributesPlusTerm *zapt,
 153                        int type)
 154 {
 155     src->zapt = zapt;
 156     src->type = type;
 157     src->major = 0;
 158     src->minor = 0;
 159 }
 160
 161 #define TERM_COUNT
 162
 163 struct grep_info {
 164 #ifdef TERM_COUNT
 165     int *term_no;
 166 #endif
 167     ISAMS_P *isam_p_buf;
 168     int isam_p_size;
 169     int isam_p_indx;
 170     ZebraHandle zh;
 171     int reg_type;
 172     ZebraSet termset;
 173 };
 174
 175 static void term_untrans(ZebraHandle zh, int reg_type,
 176                            char *dst, const char *src)
 177 {
 178     int len = 0;
 179     while (*src)
 180     {
 181         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
 182                                             reg_type, &src);
 183         if (!cp && len < IT_MAX_WORD-1)
 184             dst[len++] = *src++;
 185         else
 186             while (*cp && len < IT_MAX_WORD-1)
 187                 dst[len++] = *cp++;
 188     }
 189     dst[len] = '\0';
 190 }
 191
 192 static void add_isam_p(const char *name, const char *info,
 193                         struct grep_info *p)
 194 {
 195     if (p->isam_p_indx == p->isam_p_size)
 196     {
 197         ISAMS_P *new_isam_p_buf;
 198 #ifdef TERM_COUNT
 199         int *new_term_no;
 200 #endif
 201         p->isam_p_size = 2*p->isam_p_size + 100;
 202         new_isam_p_buf = (ISAMS_P *) xmalloc(sizeof(*new_isam_p_buf) *
 203                                              p->isam_p_size);
 204         if (p->isam_p_buf)
 205         {
 206             memcpy(new_isam_p_buf, p->isam_p_buf,
 207                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 208             xfree(p->isam_p_buf);
 209         }
 210         p->isam_p_buf = new_isam_p_buf;
 211
 212 #ifdef TERM_COUNT
 213         new_term_no = (int *) xmalloc(sizeof(*new_term_no) *
 214                                        p->isam_p_size);
 215         if (p->term_no)
 216         {
 217             memcpy(new_term_no, p->isam_p_buf,
 218                     p->isam_p_indx * sizeof(*p->term_no));
 219             xfree(p->term_no);
 220         }
 221         p->term_no = new_term_no;
 222 #endif
 223     }
 224     assert(*info == sizeof(*p->isam_p_buf));
 225     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 226
 227 #if 1
 228     if (p->termset)
 229     {
 230         const char *db;
 231         int set, use;
 232         char term_tmp[IT_MAX_WORD];
 233         int su_code = 0;
 234         int len = key_SU_decode(&su_code, name);
 235
 236         term_untrans(p->zh, p->reg_type, term_tmp, name+len+1);
 237         yaz_log(LOG_LOG, "grep: %d %c %s", su_code, name[len], term_tmp);
 238         zebraExplain_lookup_ord(p->zh->reg->zei,
 239                                  su_code, &db, &set, &use);
 240         yaz_log(LOG_LOG, "grep:  set=%d use=%d db=%s", set, use, db);
 241
 242         resultSetAddTerm(p->zh, p->termset, name[len], db,
 243                           set, use, term_tmp);
 244     }
 245 #endif
 246     (p->isam_p_indx)++;
 247 }
 248
 249 static int grep_handle(char *name, const char *info, void *p)
 250 {
 251     add_isam_p(name, info, (struct grep_info *) p);
 252     return 0;
 253 }
 254
 255 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 256                      const char *ct1, const char *ct2, int first)
 257 {
 258     const char *s1, *s0 = *src;
 259     const char **map;
 260
 261     /* skip white space */
 262     while (*s0)
 263     {
 264         if (ct1 && strchr(ct1, *s0))
 265             break;
 266         if (ct2 && strchr(ct2, *s0))
 267             break;
 268         s1 = s0;
 269         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 270         if (**map != *CHR_SPACE)
 271             break;
 272         s0 = s1;
 273     }
 274     *src = s0;
 275     return *s0;
 276 }
 277
 278
 279 static void esc_str(char *out_buf, int out_size,
 280                     const char *in_buf, int in_size)
 281 {
 282     int k;
 283
 284     assert(out_buf);
 285     assert(in_buf);
 286     assert(out_size > 20);
 287     *out_buf = '\0';
 288     for (k = 0; k<in_size; k++)
 289     {
 290         int c = in_buf[k] & 0xff;
 291         int pc;
 292         if (c < 32 || c > 126)
 293             pc = '?';
 294         else
 295             pc = c;
 296         sprintf(out_buf +strlen(out_buf), "%02X:%c  ", c, pc);
 297         if (strlen(out_buf) > out_size-20)
 298         {
 299             strcat(out_buf, "..");
 300             break;
 301         }
 302     }
 303 }
 304
 305 #define REGEX_CHARS " []()|.*+?!"
 306
 307 /* term_100: handle term, where trunc = none(no operators at all) */
 308 static int term_100(ZebraMaps zebra_maps, int reg_type,
 309                      const char **src, char *dst, int space_split,
 310                      char *dst_term)
 311 {
 312     const char *s0;
 313     const char **map;
 314     int i = 0;
 315     int j = 0;
 316
 317     const char *space_start = 0;
 318     const char *space_end = 0;
 319
 320     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 321         return 0;
 322     s0 = *src;
 323     while (*s0)
 324     {
 325         const char *s1 = s0;
 326         int q_map_match = 0;
 327         map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 328                                 &q_map_match);
 329         if (space_split)
 330         {
 331             if (**map == *CHR_SPACE)
 332                 break;
 333         }
 334         else  /* complete subfield only. */
 335         {
 336             if (**map == *CHR_SPACE)
 337             {   /* save space mapping for later  .. */
 338                 space_start = s1;
 339                 space_end = s0;
 340                 continue;
 341             }
 342             else if (space_start)
 343             {   /* reload last space */
 344                 while (space_start < space_end)
 345                 {
 346                     if (strchr(REGEX_CHARS, *space_start))
 347                         dst[i++] = '\\';
 348                     dst_term[j++] = *space_start;
 349                     dst[i++] = *space_start++;
 350                 }
 351                 /* and reset */
 352                 space_start = space_end = 0;
 353             }
 354         }
 355         /* add non-space char */
 356         memcpy(dst_term+j, s1, s0 - s1);
 357         j += (s0 - s1);
 358         if (!q_map_match)
 359         {
 360             while (s1 < s0)
 361             {
 362                 if (strchr(REGEX_CHARS, *s1))
 363                     dst[i++] = '\\';
 364                 dst[i++] = *s1++;
 365             }
 366         }
 367         else
 368         {
 369             char tmpbuf[80];
 370             esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 371
 372             strcpy(dst + i, map[0]);
 373             i += strlen(map[0]);
 374         }
 375     }
 376     dst[i] = '\0';
 377     dst_term[j] = '\0';
 378     *src = s0;
 379     return i;
 380 }
 381
 382 /* term_101: handle term, where trunc = Process # */
 383 static int term_101(ZebraMaps zebra_maps, int reg_type,
 384                      const char **src, char *dst, int space_split,
 385                      char *dst_term)
 386 {
 387     const char *s0;
 388     const char **map;
 389     int i = 0;
 390     int j = 0;
 391
 392     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 393         return 0;
 394     s0 = *src;
 395     while (*s0)
 396     {
 397         if (*s0 == '#')
 398         {
 399             dst[i++] = '.';
 400             dst[i++] = '*';
 401             dst_term[j++] = *s0++;
 402         }
 403         else
 404         {
 405             const char *s1 = s0;
 406             int q_map_match = 0;
 407             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 408                                     &q_map_match);
 409             if (space_split && **map == *CHR_SPACE)
 410                 break;
 411
 412             /* add non-space char */
 413             memcpy(dst_term+j, s1, s0 - s1);
 414             j += (s0 - s1);
 415             if (!q_map_match)
 416             {
 417                 while (s1 < s0)
 418                 {
 419                     if (strchr(REGEX_CHARS, *s1))
 420                         dst[i++] = '\\';
 421                     dst[i++] = *s1++;
 422                 }
 423             }
 424             else
 425             {
 426                 char tmpbuf[80];
 427                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 428
 429                 strcpy(dst + i, map[0]);
 430                 i += strlen(map[0]);
 431             }
 432         }
 433     }
 434     dst[i] = '\0';
 435     dst_term[j++] = '\0';
 436     *src = s0;
 437     return i;
 438 }
 439
 440 /* term_103: handle term, where trunc = re-2 (regular expressions) */
 441 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 442                      char *dst, int *errors, int space_split,
 443                      char *dst_term)
 444 {
 445     int i = 0;
 446     int j = 0;
 447     const char *s0;
 448     const char **map;
 449
 450     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 451         return 0;
 452     s0 = *src;
 453     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 454         isdigit(((const unsigned char *)s0)[1]))
 455     {
 456         *errors = s0[1] - '0';
 457         s0 += 3;
 458         if (*errors > 3)
 459             *errors = 3;
 460     }
 461     while (*s0)
 462     {
 463         if (strchr("^\\()[].*+?|-", *s0))
 464         {
 465             dst_term[j++] = *s0;
 466             dst[i++] = *s0++;
 467         }
 468         else
 469         {
 470             const char *s1 = s0;
 471             int q_map_match = 0;
 472             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 473                                     &q_map_match);
 474             if (space_split && **map == *CHR_SPACE)
 475                 break;
 476
 477             /* add non-space char */
 478             memcpy(dst_term+j, s1, s0 - s1);
 479             j += (s0 - s1);
 480             if (!q_map_match)
 481             {
 482                 while (s1 < s0)
 483                 {
 484                     if (strchr(REGEX_CHARS, *s1))
 485                         dst[i++] = '\\';
 486                     dst[i++] = *s1++;
 487                 }
 488             }
 489             else
 490             {
 491                 char tmpbuf[80];
 492                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 493
 494                 strcpy(dst + i, map[0]);
 495                 i += strlen(map[0]);
 496             }
 497         }
 498     }
 499     dst[i] = '\0';
 500     dst_term[j] = '\0';
 501     *src = s0;
 502
 503     return i;
 504 }
 505
 506 /* term_103: handle term, where trunc = re-1 (regular expressions) */
 507 static int term_102 (ZebraMaps zebra_maps, int reg_type, const char **src,
 508                      char *dst, int space_split, char *dst_term)
 509 {
 510     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 511                      dst_term);
 512 }
 513
 514 /* term_104: handle term, where trunc=Process # and ! */
 515 static int term_104(ZebraMaps zebra_maps, int reg_type,
 516                      const char **src, char *dst, int space_split,
 517                      char *dst_term)
 518 {
 519     const char *s0;
 520     const char **map;
 521     int i = 0;
 522     int j = 0;
 523
 524     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 525         return 0;
 526     s0 = *src;
 527     while (*s0)
 528     {
 529         if (*s0 == '?')
 530         {
 531             dst_term[j++] = *s0++;
 532             if (*s0 >= '0' && *s0 <= '9')
 533             {
 534                 int limit = 0;
 535                 while (*s0 >= '0' && *s0 <= '9')
 536                 {
 537                     limit = limit * 10 + (*s0 - '0');
 538                     dst_term[j++] = *s0++;
 539                 }
 540                 if (limit > 20)
 541                     limit = 20;
 542                 while (--limit >= 0)
 543                 {
 544                     dst[i++] = '.';
 545                     dst[i++] = '?';
 546                 }
 547             }
 548             else
 549             {
 550                 dst[i++] = '.';
 551                 dst[i++] = '*';
 552             }
 553         }
 554         else if (*s0 == '*')
 555         {
 556             dst[i++] = '.';
 557             dst[i++] = '*';
 558             dst_term[j++] = *s0++;
 559         }
 560         else if (*s0 == '#')
 561         {
 562             dst[i++] = '.';
 563             dst_term[j++] = *s0++;
 564         }
 565         else
 566         {
 567             const char *s1 = s0;
 568             int q_map_match = 0;
 569             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 570                                     &q_map_match);
 571             if (space_split && **map == *CHR_SPACE)
 572                 break;
 573
 574             /* add non-space char */
 575             memcpy(dst_term+j, s1, s0 - s1);
 576             j += (s0 - s1);
 577             if (!q_map_match)
 578             {
 579                 while (s1 < s0)
 580                 {
 581                     if (strchr(REGEX_CHARS, *s1))
 582                         dst[i++] = '\\';
 583                     dst[i++] = *s1++;
 584                 }
 585             }
 586             else
 587             {
 588                 char tmpbuf[80];
 589                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 590
 591                 strcpy(dst + i, map[0]);
 592                 i += strlen(map[0]);
 593             }
 594         }
 595     }
 596     dst[i] = '\0';
 597     dst_term[j++] = '\0';
 598     *src = s0;
 599     return i;
 600 }
 601
 602 /* term_105/106: handle term, where trunc=Process * and ! and right trunc */
 603 static int term_105 (ZebraMaps zebra_maps, int reg_type,
 604                      const char **src, char *dst, int space_split,
 605                      char *dst_term, int right_truncate)
 606 {
 607     const char *s0;
 608     const char **map;
 609     int i = 0;
 610     int j = 0;
 611
 612     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 613         return 0;
 614     s0 = *src;
 615     while (*s0)
 616     {
 617         if (*s0 == '*')
 618         {
 619             dst[i++] = '.';
 620             dst[i++] = '*';
 621             dst_term[j++] = *s0++;
 622         }
 623         else if (*s0 == '!')
 624         {
 625             dst[i++] = '.';
 626             dst_term[j++] = *s0++;
 627         }
 628         else
 629         {
 630             const char *s1 = s0;
 631             int q_map_match = 0;
 632             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 633                                     &q_map_match);
 634             if (space_split && **map == *CHR_SPACE)
 635                 break;
 636
 637             /* add non-space char */
 638             memcpy(dst_term+j, s1, s0 - s1);
 639             j += (s0 - s1);
 640             if (!q_map_match)
 641             {
 642                 while (s1 < s0)
 643                 {
 644                     if (strchr(REGEX_CHARS, *s1))
 645                         dst[i++] = '\\';
 646                     dst[i++] = *s1++;
 647                 }
 648             }
 649             else
 650             {
 651                 char tmpbuf[80];
 652                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 653
 654                 strcpy(dst + i, map[0]);
 655                 i += strlen(map[0]);
 656             }
 657         }
 658     }
 659     if (right_truncate)
 660     {
 661         dst[i++] = '.';
 662         dst[i++] = '*';
 663     }
 664     dst[i] = '\0';
 665
 666     dst_term[j++] = '\0';
 667     *src = s0;
 668     return i;
 669 }
 670
 671
 672 /* gen_regular_rel - generate regular expression from relation
 673  *  val:     border value (inclusive)
 674  *  islt:    1 if <=; 0 if >=.
 675  */
 676 static void gen_regular_rel(char *dst, int val, int islt)
 677 {
 678     int dst_p;
 679     int w, d, i;
 680     int pos = 0;
 681     char numstr[20];
 682
 683     yaz_log(LOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 684     if (val >= 0)
 685     {
 686         if (islt)
 687             strcpy(dst, "(-[0-9]+|(");
 688         else
 689             strcpy(dst, "((");
 690     }
 691     else
 692     {
 693         if (!islt)
 694         {
 695             strcpy(dst, "([0-9]+|-(");
 696             dst_p = strlen(dst);
 697             islt = 1;
 698         }
 699         else
 700         {
 701             strcpy(dst, "(-(");
 702             islt = 0;
 703         }
 704         val = -val;
 705     }
 706     dst_p = strlen(dst);
 707     sprintf(numstr, "%d", val);
 708     for (w = strlen(numstr); --w >= 0; pos++)
 709     {
 710         d = numstr[w];
 711         if (pos > 0)
 712         {
 713             if (islt)
 714             {
 715                 if (d == '0')
 716                     continue;
 717                 d--;
 718             }
 719             else
 720             {
 721                 if (d == '9')
 722                     continue;
 723                 d++;
 724             }
 725         }
 726
 727         strcpy(dst + dst_p, numstr);
 728         dst_p = strlen(dst) - pos - 1;
 729
 730         if (islt)
 731         {
 732             if (d != '0')
 733             {
 734                 dst[dst_p++] = '[';
 735                 dst[dst_p++] = '0';
 736                 dst[dst_p++] = '-';
 737                 dst[dst_p++] = d;
 738                 dst[dst_p++] = ']';
 739             }
 740             else
 741                 dst[dst_p++] = d;
 742         }
 743         else
 744         {
 745             if (d != '9')
 746             {
 747                 dst[dst_p++] = '[';
 748                 dst[dst_p++] = d;
 749                 dst[dst_p++] = '-';
 750                 dst[dst_p++] = '9';
 751                 dst[dst_p++] = ']';
 752             }
 753             else
 754                 dst[dst_p++] = d;
 755         }
 756         for (i = 0; i<pos; i++)
 757         {
 758             dst[dst_p++] = '[';
 759             dst[dst_p++] = '0';
 760             dst[dst_p++] = '-';
 761             dst[dst_p++] = '9';
 762             dst[dst_p++] = ']';
 763         }
 764         dst[dst_p++] = '|';
 765     }
 766     dst[dst_p] = '\0';
 767     if (islt)
 768     {
 769         /* match everything less than 10^(pos-1) */
 770         strcat(dst, "0*");
 771         for (i = 1; i<pos; i++)
 772             strcat(dst, "[0-9]?");
 773     }
 774     else
 775     {
 776         /* match everything greater than 10^pos */
 777         for (i = 0; i <= pos; i++)
 778             strcat(dst, "[0-9]");
 779         strcat(dst, "[0-9]*");
 780     }
 781     strcat(dst, "))");
 782 }
 783
 784 void string_rel_add_char(char **term_p, const char *src, int *indx)
 785 {
 786     if (src[*indx] == '\\')
 787         *(*term_p)++ = src[(*indx)++];
 788     *(*term_p)++ = src[(*indx)++];
 789 }
 790
 791 /*
 792  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 793  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 794  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 795  *              ([^-a].*|a[^-b].*|ab[c-].*)
 796  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 797  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 798  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 799  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 800  */
 801 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 802                             const char **term_sub, char *term_dict,
 803                             oid_value attributeSet,
 804                             int reg_type, int space_split, char *term_dst)
 805 {
 806     AttrType relation;
 807     int relation_value;
 808     int i;
 809     char *term_tmp = term_dict + strlen(term_dict);
 810     char term_component[2*IT_MAX_WORD+20];
 811
 812     attr_init(&relation, zapt, 2);
 813     relation_value = attr_find(&relation, NULL);
 814
 815     yaz_log(LOG_DEBUG, "string relation value=%d", relation_value);
 816     switch (relation_value)
 817     {
 818     case 1:
 819         if (!term_100 (zh->reg->zebra_maps, reg_type,
 820                        term_sub, term_component,
 821                        space_split, term_dst))
 822             return 0;
 823         yaz_log(LOG_DEBUG, "Relation <");
 824
 825         *term_tmp++ = '(';
 826         for (i = 0; term_component[i]; )
 827         {
 828             int j = 0;
 829
 830             if (i)
 831                 *term_tmp++ = '|';
 832             while (j < i)
 833                 string_rel_add_char (&term_tmp, term_component, &j);
 834
 835             *term_tmp++ = '[';
 836
 837             *term_tmp++ = '^';
 838             string_rel_add_char (&term_tmp, term_component, &i);
 839             *term_tmp++ = '-';
 840
 841             *term_tmp++ = ']';
 842             *term_tmp++ = '.';
 843             *term_tmp++ = '*';
 844
 845             if ((term_tmp - term_dict) > IT_MAX_WORD)
 846                 break;
 847         }
 848         *term_tmp++ = ')';
 849         *term_tmp = '\0';
 850         break;
 851     case 2:
 852         if (!term_100 (zh->reg->zebra_maps, reg_type,
 853                        term_sub, term_component,
 854                        space_split, term_dst))
 855             return 0;
 856         yaz_log(LOG_DEBUG, "Relation <=");
 857
 858         *term_tmp++ = '(';
 859         for (i = 0; term_component[i]; )
 860         {
 861             int j = 0;
 862
 863             while (j < i)
 864                 string_rel_add_char (&term_tmp, term_component, &j);
 865             *term_tmp++ = '[';
 866
 867             *term_tmp++ = '^';
 868             string_rel_add_char (&term_tmp, term_component, &i);
 869             *term_tmp++ = '-';
 870
 871             *term_tmp++ = ']';
 872             *term_tmp++ = '.';
 873             *term_tmp++ = '*';
 874
 875             *term_tmp++ = '|';
 876
 877             if ((term_tmp - term_dict) > IT_MAX_WORD)
 878                 break;
 879         }
 880         for (i = 0; term_component[i]; )
 881             string_rel_add_char (&term_tmp, term_component, &i);
 882         *term_tmp++ = ')';
 883         *term_tmp = '\0';
 884         break;
 885     case 5:
 886         if (!term_100 (zh->reg->zebra_maps, reg_type,
 887                        term_sub, term_component, space_split, term_dst))
 888             return 0;
 889         yaz_log(LOG_DEBUG, "Relation >");
 890
 891         *term_tmp++ = '(';
 892         for (i = 0; term_component[i];)
 893         {
 894             int j = 0;
 895
 896             while (j < i)
 897                 string_rel_add_char (&term_tmp, term_component, &j);
 898             *term_tmp++ = '[';
 899
 900             *term_tmp++ = '^';
 901             *term_tmp++ = '-';
 902             string_rel_add_char (&term_tmp, term_component, &i);
 903
 904             *term_tmp++ = ']';
 905             *term_tmp++ = '.';
 906             *term_tmp++ = '*';
 907
 908             *term_tmp++ = '|';
 909
 910             if ((term_tmp - term_dict) > IT_MAX_WORD)
 911                 break;
 912         }
 913         for (i = 0; term_component[i];)
 914             string_rel_add_char (&term_tmp, term_component, &i);
 915         *term_tmp++ = '.';
 916         *term_tmp++ = '+';
 917         *term_tmp++ = ')';
 918         *term_tmp = '\0';
 919         break;
 920     case 4:
 921         if (!term_100 (zh->reg->zebra_maps, reg_type, term_sub,
 922                        term_component, space_split, term_dst))
 923             return 0;
 924         yaz_log(LOG_DEBUG, "Relation >=");
 925
 926         *term_tmp++ = '(';
 927         for (i = 0; term_component[i];)
 928         {
 929             int j = 0;
 930
 931             if (i)
 932                 *term_tmp++ = '|';
 933             while (j < i)
 934                 string_rel_add_char (&term_tmp, term_component, &j);
 935             *term_tmp++ = '[';
 936
 937             if (term_component[i+1])
 938             {
 939                 *term_tmp++ = '^';
 940                 *term_tmp++ = '-';
 941                 string_rel_add_char (&term_tmp, term_component, &i);
 942             }
 943             else
 944             {
 945                 string_rel_add_char (&term_tmp, term_component, &i);
 946                 *term_tmp++ = '-';
 947             }
 948             *term_tmp++ = ']';
 949             *term_tmp++ = '.';
 950             *term_tmp++ = '*';
 951
 952             if ((term_tmp - term_dict) > IT_MAX_WORD)
 953                 break;
 954         }
 955         *term_tmp++ = ')';
 956         *term_tmp = '\0';
 957         break;
 958     case 3:
 959     default:
 960         yaz_log(LOG_DEBUG, "Relation =");
 961         if (!term_100 (zh->reg->zebra_maps, reg_type, term_sub,
 962                        term_component, space_split, term_dst))
 963             return 0;
 964         strcat (term_tmp, "(");
 965         strcat (term_tmp, term_component);
 966         strcat (term_tmp, ")");
 967     }
 968     return 1;
 969 }
 970
 971 static int string_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 972                         const char **term_sub,
 973                         oid_value attributeSet, NMEM stream,
 974                         struct grep_info *grep_info,
 975                         int reg_type, int complete_flag,
 976                         int num_bases, char **basenames,
 977                         char *term_dst, int xpath_use);
 978
 979 static RSET term_trunc (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 980                         const char **term_sub,
 981                         oid_value attributeSet, NMEM stream,
 982                         struct grep_info *grep_info,
 983                         int reg_type, int complete_flag,
 984                         int num_bases, char **basenames,
 985                         char *term_dst,
 986                         const char *rank_type, int xpath_use)
 987 {
 988     int r;
 989     grep_info->isam_p_indx = 0;
 990     r = string_term (zh, zapt, term_sub, attributeSet, stream, grep_info,
 991                      reg_type, complete_flag, num_bases, basenames,
 992                      term_dst, xpath_use);
 993     if (r < 1)
 994         return 0;
 995     yaz_log(LOG_DEBUG, "term: %s", term_dst);
 996     return rset_trunc (zh, grep_info->isam_p_buf,
 997                        grep_info->isam_p_indx, term_dst,
 998                        strlen(term_dst), rank_type, 1 /* preserve pos */,
 999                        zapt->term->which);
1000 }
1001
1002 static char *nmem_strdup_i(NMEM nmem, int v)
1003 {
1004     char val_str[64];
1005     sprintf (val_str, "%d", v);
1006     return nmem_strdup (nmem, val_str);
1007 }
1008
1009 static int string_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1010                         const char **term_sub,
1011                         oid_value attributeSet, NMEM stream,
1012                         struct grep_info *grep_info,
1013                         int reg_type, int complete_flag,
1014                         int num_bases, char **basenames,
1015                         char *term_dst, int xpath_use)
1016 {
1017     char term_dict[2*IT_MAX_WORD+4000];
1018     int j, r, base_no;
1019     AttrType truncation;
1020     int truncation_value;
1021     AttrType use;
1022     int use_value;
1023     const char *use_string = 0;
1024     oid_value curAttributeSet = attributeSet;
1025     const char *termp;
1026     struct rpn_char_map_info rcmi;
1027     int space_split = complete_flag ? 0 : 1;
1028
1029     int bases_ok = 0;     /* no of databases with OK attribute */
1030     int errCode = 0;      /* err code (if any is not OK) */
1031     char *errString = 0;  /* addinfo */
1032
1033     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1034     attr_init (&use, zapt, 1);
1035     use_value = attr_find_ex (&use, &curAttributeSet, &use_string);
1036     yaz_log(LOG_DEBUG, "string_term, use value %d", use_value);
1037     attr_init (&truncation, zapt, 5);
1038     truncation_value = attr_find (&truncation, NULL);
1039     yaz_log(LOG_DEBUG, "truncation value %d", truncation_value);
1040
1041     if (use_value == -1)    /* no attribute - assumy "any" */
1042         use_value = 1016;
1043     for (base_no = 0; base_no < num_bases; base_no++)
1044     {
1045         int attr_ok = 0;
1046         int regex_range = 0;
1047         int init_pos = 0;
1048         attent attp;
1049         data1_local_attribute id_xpath_attr;
1050         data1_local_attribute *local_attr;
1051         int max_pos, prefix_len = 0;
1052
1053         termp = *term_sub;
1054
1055         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1056         {
1057             zh->errCode = 109; /* Database unavailable */
1058             zh->errString = basenames[base_no];
1059             return -1;
1060         }
1061         if (xpath_use > 0 && use_value == -2)
1062         {
1063             use_value = xpath_use;
1064             attp.local_attributes = &id_xpath_attr;
1065             attp.attset_ordinal = VAL_IDXPATH;
1066             id_xpath_attr.next = 0;
1067             id_xpath_attr.local = use_value;
1068         }
1069         else if (curAttributeSet == VAL_IDXPATH)
1070         {
1071             attp.local_attributes = &id_xpath_attr;
1072             attp.attset_ordinal = VAL_IDXPATH;
1073             id_xpath_attr.next = 0;
1074             id_xpath_attr.local = use_value;
1075         }
1076         else
1077         {
1078             if ((r=att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1079                                             use_string)))
1080             {
1081                 yaz_log(LOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1082                       curAttributeSet, use_value, r);
1083                 if (r == -1)
1084                 {
1085                     /* set was found, but value wasn't defined */
1086                     errCode = 114;
1087                     if (use_string)
1088                         errString = nmem_strdup(stream, use_string);
1089                     else
1090                         errString = nmem_strdup_i(stream, use_value);
1091                 }
1092                 else
1093                 {
1094                     int oid[OID_SIZE];
1095                     struct oident oident;
1096
1097                     oident.proto = PROTO_Z3950;
1098                     oident.oclass = CLASS_ATTSET;
1099                     oident.value = curAttributeSet;
1100                     oid_ent_to_oid (&oident, oid);
1101
1102                     errCode = 121;
1103                     errString = nmem_strdup (stream, oident.desc);
1104                 }
1105                 continue;
1106             }
1107         }
1108         for (local_attr = attp.local_attributes; local_attr;
1109              local_attr = local_attr->next)
1110         {
1111             int ord;
1112             char ord_buf[32];
1113             int i, ord_len;
1114
1115             ord = zebraExplain_lookupSU (zh->reg->zei, attp.attset_ordinal,
1116                                          local_attr->local);
1117             if (ord < 0)
1118                 continue;
1119             if (prefix_len)
1120                 term_dict[prefix_len++] = '|';
1121             else
1122                 term_dict[prefix_len++] = '(';
1123
1124             ord_len = key_SU_encode (ord, ord_buf);
1125             for (i = 0; i<ord_len; i++)
1126             {
1127                 term_dict[prefix_len++] = 1;
1128                 term_dict[prefix_len++] = ord_buf[i];
1129             }
1130         }
1131         if (!prefix_len)
1132         {
1133 #if 1
1134             bases_ok++;
1135 #else
1136             errCode = 114;
1137             errString = nmem_strdup_i(stream, use_value);
1138             continue;
1139 #endif
1140         }
1141         else
1142         {
1143             attr_ok = 1;
1144             bases_ok++; /* this has OK attributes */
1145         }
1146
1147         term_dict[prefix_len++] = ')';
1148         term_dict[prefix_len++] = 1;
1149         term_dict[prefix_len++] = reg_type;
1150         yaz_log(LOG_DEBUG, "reg_type = %d", term_dict[prefix_len-1]);
1151         term_dict[prefix_len] = '\0';
1152         j = prefix_len;
1153         switch (truncation_value)
1154         {
1155         case -1:         /* not specified */
1156         case 100:        /* do not truncate */
1157             if (!string_relation (zh, zapt, &termp, term_dict,
1158                                   attributeSet,
1159                                   reg_type, space_split, term_dst))
1160                 return 0;
1161             break;
1162         case 1:          /* right truncation */
1163             term_dict[j++] = '(';
1164             if (!term_100 (zh->reg->zebra_maps, reg_type,
1165                            &termp, term_dict + j, space_split, term_dst))
1166                 return 0;
1167             strcat (term_dict, ".*)");
1168             break;
1169         case 2:          /* keft truncation */
1170             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1171             if (!term_100 (zh->reg->zebra_maps, reg_type,
1172                            &termp, term_dict + j, space_split, term_dst))
1173                 return 0;
1174             strcat (term_dict, ")");
1175             break;
1176         case 3:          /* left&right truncation */
1177             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1178             if (!term_100 (zh->reg->zebra_maps, reg_type,
1179                            &termp, term_dict + j, space_split, term_dst))
1180                 return 0;
1181             strcat (term_dict, ".*)");
1182             break;
1183         case 101:        /* process # in term */
1184             term_dict[j++] = '(';
1185             if (!term_101 (zh->reg->zebra_maps, reg_type,
1186                            &termp, term_dict + j, space_split, term_dst))
1187                 return 0;
1188             strcat (term_dict, ")");
1189             break;
1190         case 102:        /* Regexp-1 */
1191             term_dict[j++] = '(';
1192             if (!term_102 (zh->reg->zebra_maps, reg_type,
1193                            &termp, term_dict + j, space_split, term_dst))
1194                 return 0;
1195             strcat (term_dict, ")");
1196             break;
1197         case 103:       /* Regexp-2 */
1198             regex_range = 1;
1199             term_dict[j++] = '(';
1200             init_pos = 2;
1201             if (!term_103 (zh->reg->zebra_maps, reg_type,
1202                            &termp, term_dict + j, &regex_range,
1203                            space_split, term_dst))
1204                 return 0;
1205             strcat (term_dict, ")");
1206             break;
1207         case 104:        /* process # and ! in term */
1208             term_dict[j++] = '(';
1209             if (!term_104 (zh->reg->zebra_maps, reg_type,
1210                            &termp, term_dict + j, space_split, term_dst))
1211                 return 0;
1212             strcat (term_dict, ")");
1213             break;
1214         case 105:        /* process * and ! in term */
1215             term_dict[j++] = '(';
1216             if (!term_105 (zh->reg->zebra_maps, reg_type,
1217                            &termp, term_dict + j, space_split, term_dst, 1))
1218                 return 0;
1219             strcat (term_dict, ")");
1220             break;
1221         case 106:        /* process * and ! in term */
1222             term_dict[j++] = '(';
1223             if (!term_105 (zh->reg->zebra_maps, reg_type,
1224                            &termp, term_dict + j, space_split, term_dst, 0))
1225                 return 0;
1226             strcat (term_dict, ")");
1227             break;
1228         default:
1229             zh->errCode = 120;
1230             zh->errString = nmem_strdup_i(stream, truncation_value);
1231             return -1;
1232         }
1233         if (attr_ok)
1234         {
1235             yaz_log(LOG_DEBUG, "dict_lookup_grep: %s", term_dict+prefix_len);
1236             r = dict_lookup_grep (zh->reg->dict, term_dict, regex_range,
1237                                   grep_info, &max_pos, init_pos,
1238                                   grep_handle);
1239             if (r)
1240                 yaz_log(LOG_WARN, "dict_lookup_grep fail %d", r);
1241         }
1242     }
1243     if (!bases_ok)
1244     {
1245         zh->errCode = errCode;
1246         zh->errString = errString;
1247         return -1;
1248     }
1249     *term_sub = termp;
1250     yaz_log(LOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1251     return 1;
1252 }
1253
1254
1255 /* convert APT search term to UTF8 */
1256 static int zapt_term_to_utf8 (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1257                               char *termz)
1258 {
1259     size_t sizez;
1260     Z_Term *term = zapt->term;
1261
1262     switch (term->which)
1263     {
1264     case Z_Term_general:
1265         if (zh->iconv_to_utf8 != 0)
1266         {
1267             char *inbuf = term->u.general->buf;
1268             size_t inleft = term->u.general->len;
1269             char *outbuf = termz;
1270             size_t outleft = IT_MAX_WORD-1;
1271             size_t ret;
1272
1273             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1274                         &outbuf, &outleft);
1275             if (ret == (size_t)(-1))
1276             {
1277                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1278                 zh->errCode = 125;
1279                 return -1;
1280             }
1281             *outbuf = 0;
1282         }
1283         else
1284         {
1285             sizez = term->u.general->len;
1286             if (sizez > IT_MAX_WORD-1)
1287                 sizez = IT_MAX_WORD-1;
1288             memcpy (termz, term->u.general->buf, sizez);
1289             termz[sizez] = '\0';
1290         }
1291         break;
1292     case Z_Term_characterString:
1293         sizez = strlen(term->u.characterString);
1294         if (sizez > IT_MAX_WORD-1)
1295             sizez = IT_MAX_WORD-1;
1296         memcpy (termz, term->u.characterString, sizez);
1297         termz[sizez] = '\0';
1298         break;
1299     default:
1300         zh->errCode = 124;
1301         return -1;
1302     }
1303     return 0;
1304 }
1305
1306 /* convert APT SCAN term to internal cmap */
1307 static int trans_scan_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1308                             char *termz, int reg_type)
1309 {
1310     char termz0[IT_MAX_WORD];
1311
1312     if (zapt_term_to_utf8(zh, zapt, termz0))
1313         return -1;    /* error */
1314     else
1315     {
1316         const char **map;
1317         const char *cp = (const char *) termz0;
1318         const char *cp_end = cp + strlen(cp);
1319         const char *src;
1320         int i = 0;
1321         const char *space_map = NULL;
1322         int len;
1323
1324         while ((len = (cp_end - cp)) > 0)
1325         {
1326             map = zebra_maps_input (zh->reg->zebra_maps, reg_type, &cp, len, 0);
1327             if (**map == *CHR_SPACE)
1328                 space_map = *map;
1329             else
1330             {
1331                 if (i && space_map)
1332                     for (src = space_map; *src; src++)
1333                         termz[i++] = *src;
1334                 space_map = NULL;
1335                 for (src = *map; *src; src++)
1336                     termz[i++] = *src;
1337             }
1338         }
1339         termz[i] = '\0';
1340     }
1341     return 0;
1342 }
1343
1344 char *normalize_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1345                      const char *termz, NMEM stream, unsigned reg_id)
1346 {
1347     WRBUF wrbuf = 0;
1348     AttrType truncation;
1349     int truncation_value;
1350     char *ex_list = 0;
1351
1352     attr_init (&truncation, zapt, 5);
1353     truncation_value = attr_find (&truncation, NULL);
1354
1355     switch (truncation_value)
1356     {
1357     default:
1358         ex_list = "";
1359         break;
1360     case 101:
1361         ex_list = "#";
1362         break;
1363     case 102:
1364     case 103:
1365         ex_list = 0;
1366         break;
1367     case 104:
1368         ex_list = "!#";
1369         break;
1370     case 105:
1371         ex_list = "!*";
1372         break;
1373     }
1374     if (ex_list)
1375         wrbuf = zebra_replace(zh->reg->zebra_maps, reg_id, ex_list,
1376                               termz, strlen(termz));
1377     if (!wrbuf)
1378         return nmem_strdup(stream, termz);
1379     else
1380     {
1381         char *buf = (char*) nmem_malloc (stream, wrbuf_len(wrbuf)+1);
1382         memcpy (buf, wrbuf_buf(wrbuf), wrbuf_len(wrbuf));
1383         buf[wrbuf_len(wrbuf)] = '\0';
1384         return buf;
1385     }
1386 }
1387
1388 static void grep_info_delete (struct grep_info *grep_info)
1389 {
1390 #ifdef TERM_COUNT
1391     xfree(grep_info->term_no);
1392 #endif
1393     xfree (grep_info->isam_p_buf);
1394 }
1395
1396 static int grep_info_prepare (ZebraHandle zh,
1397                               Z_AttributesPlusTerm *zapt,
1398                               struct grep_info *grep_info,
1399                               int reg_type,
1400                               NMEM stream)
1401 {
1402     AttrType termset;
1403     int termset_value_numeric;
1404     const char *termset_value_string;
1405
1406 #ifdef TERM_COUNT
1407     grep_info->term_no = 0;
1408 #endif
1409     grep_info->isam_p_size = 0;
1410     grep_info->isam_p_buf = NULL;
1411     grep_info->zh = zh;
1412     grep_info->reg_type = reg_type;
1413     grep_info->termset = 0;
1414
1415     if (!zapt)
1416         return 0;
1417     attr_init (&termset, zapt, 8);
1418     termset_value_numeric =
1419         attr_find_ex (&termset, NULL, &termset_value_string);
1420     if (termset_value_numeric != -1)
1421     {
1422         char resname[32];
1423         const char *termset_name = 0;
1424         if (termset_value_numeric != -2)
1425         {
1426             sprintf (resname, "%d", termset_value_numeric);
1427             termset_name = resname;
1428         }
1429         else
1430             termset_name = termset_value_string;
1431         yaz_log(LOG_LOG, "creating termset set %s", termset_name);
1432         grep_info->termset = resultSetAdd (zh, termset_name, 1);
1433         if (!grep_info->termset)
1434         {
1435             zh->errCode = 128;
1436             zh->errString = nmem_strdup (stream, termset_name);
1437             return -1;
1438         }
1439     }
1440     return 0;
1441 }
1442
1443
1444 static RSET rpn_search_APT_phrase (ZebraHandle zh,
1445                                    Z_AttributesPlusTerm *zapt,
1446                                    const char *termz_org,
1447                                    oid_value attributeSet,
1448                                    NMEM stream,
1449                                    int reg_type, int complete_flag,
1450                                    const char *rank_type, int xpath_use,
1451                                    int num_bases, char **basenames)
1452 {
1453     char term_dst[IT_MAX_WORD+1];
1454     RSET rset[60], result;
1455     int  rset_no = 0;
1456     struct grep_info grep_info;
1457     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1458     const char *termp = termz;
1459
1460     *term_dst = 0;
1461     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1462         return 0;
1463     while (1)
1464     {
1465         yaz_log(LOG_DEBUG, "APT_phrase termp=%s", termp);
1466         rset[rset_no] = term_trunc (zh, zapt, &termp, attributeSet,
1467                                     stream, &grep_info,
1468                                     reg_type, complete_flag,
1469                                     num_bases, basenames,
1470                                     term_dst, rank_type,
1471                                     xpath_use);
1472         if (!rset[rset_no])
1473             break;
1474         if (++rset_no >= (int) (sizeof(rset)/sizeof(*rset)))
1475             break;
1476     }
1477     grep_info_delete (&grep_info);
1478     if (rset_no == 0)
1479     {
1480         rset_null_parms parms;
1481
1482         parms.rset_term = rset_term_create (termz, -1, rank_type,
1483                                             zapt->term->which);
1484         return rset_create (rset_kind_null, &parms);
1485     }
1486     else if (rset_no == 1)
1487         return (rset[0]);
1488     else
1489     {
1490         /* new / old prox */
1491         rset_prox_parms parms;
1492
1493         parms.rset = rset;
1494         parms.rset_no = rset_no;
1495         parms.ordered = 1;
1496         parms.exclusion = 0;
1497         parms.relation = 3;
1498         parms.distance = 1;
1499         parms.key_size = sizeof(struct it_key);
1500         parms.cmp = key_compare_it;
1501         parms.getseq = key_get_seq;
1502         parms.log_item = key_logdump_txt;
1503         result = rset_create(rset_kind_prox, &parms);
1504     }
1505     return result;
1506 }
1507
1508 static RSET rpn_search_APT_or_list (ZebraHandle zh,
1509                                     Z_AttributesPlusTerm *zapt,
1510                                     const char *termz_org,
1511                                     oid_value attributeSet,
1512                                     NMEM stream,
1513                                     int reg_type, int complete_flag,
1514                                     const char *rank_type,
1515                                     int xpath_use,
1516                                     int num_bases, char **basenames)
1517 {
1518     char term_dst[IT_MAX_WORD+1];
1519     RSET rset[60], result;
1520     int i, rset_no = 0;
1521     struct grep_info grep_info;
1522     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1523     const char *termp = termz;
1524
1525     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1526         return 0;
1527     while (1)
1528     {
1529         yaz_log(LOG_DEBUG, "APT_or_list termp=%s", termp);
1530         rset[rset_no] = term_trunc (zh, zapt, &termp, attributeSet,
1531                                     stream, &grep_info,
1532                                     reg_type, complete_flag,
1533                                     num_bases, basenames,
1534                                     term_dst, rank_type,
1535                                     xpath_use);
1536         if (!rset[rset_no])
1537             break;
1538         if (++rset_no >= (int) (sizeof(rset)/sizeof(*rset)))
1539             break;
1540     }
1541     grep_info_delete (&grep_info);
1542     if (rset_no == 0)
1543     {
1544         rset_null_parms parms;
1545
1546         parms.rset_term = rset_term_create (termz, -1, rank_type,
1547                                             zapt->term->which);
1548         return rset_create (rset_kind_null, &parms);
1549     }
1550     result = rset[0];
1551     for (i = 1; i<rset_no; i++)
1552     {
1553         rset_bool_parms bool_parms;
1554
1555         bool_parms.rset_l = result;
1556         bool_parms.rset_r = rset[i];
1557         bool_parms.key_size = sizeof(struct it_key);
1558         bool_parms.cmp = key_compare_it;
1559         bool_parms.log_item = key_logdump_txt;
1560         result = rset_create (rset_kind_or, &bool_parms);
1561     }
1562     return result;
1563 }
1564
1565 static RSET rpn_search_APT_and_list (ZebraHandle zh,
1566                                      Z_AttributesPlusTerm *zapt,
1567                                      const char *termz_org,
1568                                      oid_value attributeSet,
1569                                      NMEM stream,
1570                                      int reg_type, int complete_flag,
1571                                      const char *rank_type,
1572                                      int xpath_use,
1573                                      int num_bases, char **basenames)
1574 {
1575     char term_dst[IT_MAX_WORD+1];
1576     RSET rset[60], result;
1577     int i, rset_no = 0;
1578     struct grep_info grep_info;
1579     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1580     const char *termp = termz;
1581
1582     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1583         return 0;
1584     while (1)
1585     {
1586         yaz_log(LOG_DEBUG, "APT_and_list termp=%s", termp);
1587         rset[rset_no] = term_trunc (zh, zapt, &termp, attributeSet,
1588                                     stream, &grep_info,
1589                                     reg_type, complete_flag,
1590                                     num_bases, basenames,
1591                                     term_dst, rank_type,
1592                                     xpath_use);
1593         if (!rset[rset_no])
1594             break;
1595         assert (rset[rset_no]);
1596         if (++rset_no >= (int) (sizeof(rset)/sizeof(*rset)))
1597             break;
1598     }
1599     grep_info_delete (&grep_info);
1600     if (rset_no == 0)
1601     {
1602         rset_null_parms parms;
1603
1604         parms.rset_term = rset_term_create (termz, -1, rank_type,
1605                                             zapt->term->which);
1606         return rset_create (rset_kind_null, &parms);
1607     }
1608     result = rset[0];
1609     for (i = 1; i<rset_no; i++)
1610     {
1611         rset_bool_parms bool_parms;
1612
1613         bool_parms.rset_l = result;
1614         bool_parms.rset_r = rset[i];
1615         bool_parms.key_size = sizeof(struct it_key);
1616         bool_parms.cmp = key_compare_it;
1617         bool_parms.log_item = key_logdump_txt;
1618         result = rset_create (rset_kind_and, &bool_parms);
1619     }
1620     return result;
1621 }
1622
1623 static int numeric_relation (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1624                              const char **term_sub,
1625                              char *term_dict,
1626                              oid_value attributeSet,
1627                              struct grep_info *grep_info,
1628                              int *max_pos,
1629                              int reg_type,
1630                              char *term_dst)
1631 {
1632     AttrType relation;
1633     int relation_value;
1634     int term_value;
1635     int r;
1636     char *term_tmp = term_dict + strlen(term_dict);
1637
1638     attr_init (&relation, zapt, 2);
1639     relation_value = attr_find (&relation, NULL);
1640
1641     yaz_log(LOG_DEBUG, "numeric relation value=%d", relation_value);
1642
1643     if (!term_100 (zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1644                    term_dst))
1645         return 0;
1646     term_value = atoi (term_tmp);
1647     switch (relation_value)
1648     {
1649     case 1:
1650         yaz_log(LOG_DEBUG, "Relation <");
1651         gen_regular_rel (term_tmp, term_value-1, 1);
1652         break;
1653     case 2:
1654         yaz_log(LOG_DEBUG, "Relation <=");
1655         gen_regular_rel (term_tmp, term_value, 1);
1656         break;
1657     case 4:
1658         yaz_log(LOG_DEBUG, "Relation >=");
1659         gen_regular_rel (term_tmp, term_value, 0);
1660         break;
1661     case 5:
1662         yaz_log(LOG_DEBUG, "Relation >");
1663         gen_regular_rel (term_tmp, term_value+1, 0);
1664         break;
1665     case 3:
1666     default:
1667         yaz_log(LOG_DEBUG, "Relation =");
1668         sprintf (term_tmp, "(0*%d)", term_value);
1669     }
1670     yaz_log(LOG_DEBUG, "dict_lookup_grep: %s", term_tmp);
1671     r = dict_lookup_grep (zh->reg->dict, term_dict, 0, grep_info, max_pos,
1672                           0, grep_handle);
1673     if (r)
1674         yaz_log(LOG_WARN, "dict_lookup_grep fail, rel=gt: %d", r);
1675     yaz_log(LOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1676     return 1;
1677 }
1678
1679 static int numeric_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1680                          const char **term_sub,
1681                          oid_value attributeSet, struct grep_info *grep_info,
1682                          int reg_type, int complete_flag,
1683                          int num_bases, char **basenames,
1684                          char *term_dst, int xpath_use, NMEM stream)
1685 {
1686     char term_dict[2*IT_MAX_WORD+2];
1687     int r, base_no;
1688     AttrType use;
1689     int use_value;
1690     const char *use_string = 0;
1691     oid_value curAttributeSet = attributeSet;
1692     const char *termp;
1693     struct rpn_char_map_info rcmi;
1694
1695     int bases_ok = 0;     /* no of databases with OK attribute */
1696     int errCode = 0;      /* err code (if any is not OK) */
1697     char *errString = 0;  /* addinfo */
1698
1699     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1700     attr_init (&use, zapt, 1);
1701     use_value = attr_find_ex (&use, &curAttributeSet, &use_string);
1702
1703     if (use_value == -1)
1704         use_value = 1016;
1705
1706     for (base_no = 0; base_no < num_bases; base_no++)
1707     {
1708         attent attp;
1709         data1_local_attribute id_xpath_attr;
1710         data1_local_attribute *local_attr;
1711         int max_pos, prefix_len = 0;
1712
1713         termp = *term_sub;
1714         if (use_value == -2)  /* string attribute (assume IDXPATH/any) */
1715         {
1716             use_value = xpath_use;
1717             attp.local_attributes = &id_xpath_attr;
1718             attp.attset_ordinal = VAL_IDXPATH;
1719             id_xpath_attr.next = 0;
1720             id_xpath_attr.local = use_value;
1721         }
1722         else if (curAttributeSet == VAL_IDXPATH)
1723         {
1724             attp.local_attributes = &id_xpath_attr;
1725             attp.attset_ordinal = VAL_IDXPATH;
1726             id_xpath_attr.next = 0;
1727             id_xpath_attr.local = use_value;
1728         }
1729         else
1730         {
1731             if ((r=att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1732                                             use_string)))
1733             {
1734                 yaz_log(LOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1735                       curAttributeSet, use_value, r);
1736                 if (r == -1)
1737                 {
1738                     errString = nmem_strdup_i (stream, use_value);
1739                     errCode = 114;
1740                 }
1741                 else
1742                     errCode = 121;
1743                 continue;
1744             }
1745         }
1746         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1747         {
1748             zh->errCode = 109; /* Database unavailable */
1749             zh->errString = basenames[base_no];
1750             return -1;
1751         }
1752         for (local_attr = attp.local_attributes; local_attr;
1753              local_attr = local_attr->next)
1754         {
1755             int ord;
1756             char ord_buf[32];
1757             int i, ord_len;
1758
1759             ord = zebraExplain_lookupSU (zh->reg->zei, attp.attset_ordinal,
1760                                           local_attr->local);
1761             if (ord < 0)
1762                 continue;
1763             if (prefix_len)
1764                 term_dict[prefix_len++] = '|';
1765             else
1766                 term_dict[prefix_len++] = '(';
1767
1768             ord_len = key_SU_encode (ord, ord_buf);
1769             for (i = 0; i<ord_len; i++)
1770             {
1771                 term_dict[prefix_len++] = 1;
1772                 term_dict[prefix_len++] = ord_buf[i];
1773             }
1774         }
1775         if (!prefix_len)
1776         {
1777             char val_str[32];
1778             sprintf (val_str, "%d", use_value);
1779             errCode = 114;
1780             errString = nmem_strdup (stream, val_str);
1781             continue;
1782         }
1783         bases_ok++;
1784         term_dict[prefix_len++] = ')';
1785         term_dict[prefix_len++] = 1;
1786         term_dict[prefix_len++] = reg_type;
1787         yaz_log(LOG_DEBUG, "reg_type = %d", term_dict[prefix_len-1]);
1788         term_dict[prefix_len] = '\0';
1789         if (!numeric_relation (zh, zapt, &termp, term_dict,
1790                                attributeSet, grep_info, &max_pos, reg_type,
1791                                term_dst))
1792             return 0;
1793     }
1794     if (!bases_ok)
1795     {
1796         zh->errCode = errCode;
1797         zh->errString = errString;
1798         return -1;
1799     }
1800     *term_sub = termp;
1801     yaz_log(LOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1802     return 1;
1803 }
1804
1805 static RSET rpn_search_APT_numeric (ZebraHandle zh,
1806                                     Z_AttributesPlusTerm *zapt,
1807                                     const char *termz,
1808                                     oid_value attributeSet,
1809                                     NMEM stream,
1810                                     int reg_type, int complete_flag,
1811                                     const char *rank_type, int xpath_use,
1812                                     int num_bases, char **basenames)
1813 {
1814     char term_dst[IT_MAX_WORD+1];
1815     const char *termp = termz;
1816     RSET rset[60], result;
1817     int i, r, rset_no = 0;
1818     struct grep_info grep_info;
1819
1820     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1821         return 0;
1822     while (1)
1823     {
1824         yaz_log(LOG_DEBUG, "APT_numeric termp=%s", termp);
1825         grep_info.isam_p_indx = 0;
1826         r = numeric_term (zh, zapt, &termp, attributeSet, &grep_info,
1827                           reg_type, complete_flag, num_bases, basenames,
1828                           term_dst, xpath_use,
1829                           stream);
1830         if (r < 1)
1831             break;
1832         yaz_log(LOG_DEBUG, "term: %s", term_dst);
1833         rset[rset_no] = rset_trunc (zh, grep_info.isam_p_buf,
1834                                     grep_info.isam_p_indx, term_dst,
1835                                     strlen(term_dst), rank_type,
1836                                     0 /* preserve position */,
1837                                     zapt->term->which);
1838         assert (rset[rset_no]);
1839         if (++rset_no >= (int) (sizeof(rset)/sizeof(*rset)))
1840             break;
1841     }
1842     grep_info_delete (&grep_info);
1843     if (rset_no == 0)
1844     {
1845         rset_null_parms parms;
1846
1847         parms.rset_term = rset_term_create (term_dst, -1, rank_type,
1848                                             zapt->term->which);
1849         return rset_create (rset_kind_null, &parms);
1850     }
1851     result = rset[0];
1852     for (i = 1; i<rset_no; i++)
1853     {
1854         rset_bool_parms bool_parms;
1855
1856         bool_parms.rset_l = result;
1857         bool_parms.rset_r = rset[i];
1858         bool_parms.key_size = sizeof(struct it_key);
1859         bool_parms.cmp = key_compare_it;
1860         bool_parms.log_item = key_logdump_txt;
1861         result = rset_create (rset_kind_and, &bool_parms);
1862     }
1863     return result;
1864 }
1865
1866 static RSET rpn_search_APT_local (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1867                                   const char *termz,
1868                                   oid_value attributeSet,
1869                                   NMEM stream,
1870                                   const char *rank_type)
1871 {
1872     RSET result;
1873     RSFD rsfd;
1874     struct it_key key;
1875     rset_temp_parms parms;
1876
1877     parms.rset_term = rset_term_create (termz, -1, rank_type,
1878                                         zapt->term->which);
1879     parms.cmp = key_compare_it;
1880     parms.key_size = sizeof (struct it_key);
1881     parms.temp_path = res_get (zh->res, "setTmpDir");
1882     result = rset_create (rset_kind_temp, &parms);
1883     rsfd = rset_open (result, RSETF_WRITE);
1884
1885     key.sysno = atoi (termz);
1886     key.seqno = 1;
1887     if (key.sysno <= 0)
1888         key.sysno = 1;
1889     rset_write (result, rsfd, &key);
1890     rset_close (result, rsfd);
1891     return result;
1892 }
1893
1894 static RSET rpn_sort_spec (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1895                            oid_value attributeSet, NMEM stream,
1896                            Z_SortKeySpecList *sort_sequence,
1897                            const char *rank_type)
1898 {
1899     rset_null_parms parms;
1900     int i;
1901     int sort_relation_value;
1902     AttrType sort_relation_type;
1903     Z_SortKeySpec *sks;
1904     Z_SortKey *sk;
1905     int oid[OID_SIZE];
1906     oident oe;
1907     char termz[20];
1908
1909     attr_init (&sort_relation_type, zapt, 7);
1910     sort_relation_value = attr_find (&sort_relation_type, &attributeSet);
1911
1912     if (!sort_sequence->specs)
1913     {
1914         sort_sequence->num_specs = 10;
1915         sort_sequence->specs = (Z_SortKeySpec **)
1916             nmem_malloc (stream, sort_sequence->num_specs *
1917                          sizeof(*sort_sequence->specs));
1918         for (i = 0; i<sort_sequence->num_specs; i++)
1919             sort_sequence->specs[i] = 0;
1920     }
1921     if (zapt->term->which != Z_Term_general)
1922         i = 0;
1923     else
1924         i = atoi_n ((char *) zapt->term->u.general->buf,
1925                     zapt->term->u.general->len);
1926     if (i >= sort_sequence->num_specs)
1927         i = 0;
1928     sprintf (termz, "%d", i);
1929
1930     oe.proto = PROTO_Z3950;
1931     oe.oclass = CLASS_ATTSET;
1932     oe.value = attributeSet;
1933     if (!oid_ent_to_oid (&oe, oid))
1934         return 0;
1935
1936     sks = (Z_SortKeySpec *) nmem_malloc (stream, sizeof(*sks));
1937     sks->sortElement = (Z_SortElement *)
1938         nmem_malloc (stream, sizeof(*sks->sortElement));
1939     sks->sortElement->which = Z_SortElement_generic;
1940     sk = sks->sortElement->u.generic = (Z_SortKey *)
1941         nmem_malloc (stream, sizeof(*sk));
1942     sk->which = Z_SortKey_sortAttributes;
1943     sk->u.sortAttributes = (Z_SortAttributes *)
1944         nmem_malloc (stream, sizeof(*sk->u.sortAttributes));
1945
1946     sk->u.sortAttributes->id = oid;
1947     sk->u.sortAttributes->list = zapt->attributes;
1948
1949     sks->sortRelation = (int *)
1950         nmem_malloc (stream, sizeof(*sks->sortRelation));
1951     if (sort_relation_value == 1)
1952         *sks->sortRelation = Z_SortKeySpec_ascending;
1953     else if (sort_relation_value == 2)
1954         *sks->sortRelation = Z_SortKeySpec_descending;
1955     else
1956         *sks->sortRelation = Z_SortKeySpec_ascending;
1957
1958     sks->caseSensitivity = (int *)
1959         nmem_malloc (stream, sizeof(*sks->caseSensitivity));
1960     *sks->caseSensitivity = 0;
1961
1962     sks->which = Z_SortKeySpec_null;
1963     sks->u.null = odr_nullval ();
1964     sort_sequence->specs[i] = sks;
1965
1966     parms.rset_term = rset_term_create (termz, -1, rank_type,
1967                                         zapt->term->which);
1968     return rset_create (rset_kind_null, &parms);
1969 }
1970
1971
1972 static int parse_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1973                        oid_value attributeSet,
1974                        struct xpath_location_step *xpath, int max, NMEM mem)
1975 {
1976     oid_value curAttributeSet = attributeSet;
1977     AttrType use;
1978     const char *use_string = 0;
1979
1980     attr_init (&use, zapt, 1);
1981     attr_find_ex (&use, &curAttributeSet, &use_string);
1982
1983     if (!use_string || *use_string != '/')
1984         return -1;
1985
1986     return zebra_parse_xpath_str(use_string, xpath, max, mem);
1987 }
1988
1989
1990
1991 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
1992                         int reg_type, const char *term, int use,
1993                         oid_value curAttributeSet)
1994 {
1995     RSET rset;
1996     struct grep_info grep_info;
1997     char term_dict[2048];
1998     char ord_buf[32];
1999     int prefix_len = 0;
2000     int ord = zebraExplain_lookupSU (zh->reg->zei, curAttributeSet, use);
2001     int ord_len, i, r, max_pos;
2002     int term_type = Z_Term_characterString;
2003     const char *flags = "void";
2004
2005     if (grep_info_prepare (zh, 0 /* zapt */, &grep_info, '0', stream))
2006     {
2007         rset_null_parms parms;
2008
2009         parms.rset_term = rset_term_create (term, strlen(term),
2010                                             flags, term_type);
2011         parms.rset_term->nn = 0;
2012         return rset_create (rset_kind_null, &parms);
2013     }
2014
2015     if (ord < 0)
2016     {
2017         rset_null_parms parms;
2018
2019         parms.rset_term = rset_term_create (term, strlen(term),
2020                                             flags, term_type);
2021         parms.rset_term->nn = 0;
2022         return rset_create (rset_kind_null, &parms);
2023     }
2024     if (prefix_len)
2025         term_dict[prefix_len++] = '|';
2026     else
2027         term_dict[prefix_len++] = '(';
2028
2029     ord_len = key_SU_encode (ord, ord_buf);
2030     for (i = 0; i<ord_len; i++)
2031     {
2032         term_dict[prefix_len++] = 1;
2033         term_dict[prefix_len++] = ord_buf[i];
2034     }
2035     term_dict[prefix_len++] = ')';
2036     term_dict[prefix_len++] = 1;
2037     term_dict[prefix_len++] = reg_type;
2038
2039     strcpy (term_dict+prefix_len, term);
2040
2041     grep_info.isam_p_indx = 0;
2042     r = dict_lookup_grep (zh->reg->dict, term_dict, 0,
2043                           &grep_info, &max_pos, 0, grep_handle);
2044     yaz_log (LOG_LOG, "%s %d positions", term, grep_info.isam_p_indx);
2045     rset = rset_trunc (zh, grep_info.isam_p_buf,
2046                        grep_info.isam_p_indx, term, strlen(term),
2047                        flags, 1, term_type);
2048     grep_info_delete (&grep_info);
2049     return rset;
2050 }
2051
2052 static RSET rpn_search_xpath (ZebraHandle zh,
2053                               oid_value attributeSet,
2054                               int num_bases, char **basenames,
2055                               NMEM stream, const char *rank_type, RSET rset,
2056                               int xpath_len, struct xpath_location_step *xpath)
2057 {
2058     oid_value curAttributeSet = attributeSet;
2059     int base_no;
2060     int i;
2061
2062     if (xpath_len < 0)
2063         return rset;
2064
2065     yaz_log (LOG_DEBUG, "len=%d", xpath_len);
2066     for (i = 0; i<xpath_len; i++)
2067     {
2068         yaz_log (LOG_DEBUG, "XPATH %d %s", i, xpath[i].part);
2069     }
2070
2071     curAttributeSet = VAL_IDXPATH;
2072
2073     /*
2074       //a    ->    a/.*
2075       //a/b  ->    b/a/.*
2076       /a     ->    a/
2077       /a/b   ->    b/a/
2078
2079       /      ->    none
2080
2081    a[@attr=value]/b[@other=othervalue]
2082
2083  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
2084  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
2085  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
2086  /a/b[@c=y] val range(b/a/,freetext(w,1016,val),b/a/,@c=y)
2087  /a[@c=y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c=y)
2088  /a[@c=x]/b[@c=y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c=y),a/,@c=x)
2089
2090     */
2091
2092     dict_grep_cmap (zh->reg->dict, 0, 0);
2093
2094     for (base_no = 0; base_no < num_bases; base_no++)
2095     {
2096         int level = xpath_len;
2097         int first_path = 1;
2098
2099         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2100         {
2101             zh->errCode = 109; /* Database unavailable */
2102             zh->errString = basenames[base_no];
2103             return rset;
2104         }
2105         while (--level >= 0)
2106         {
2107             char xpath_rev[128];
2108             int i, len;
2109             rset_between_parms parms;
2110             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
2111
2112             *xpath_rev = 0;
2113             len = 0;
2114             for (i = level; i >= 1; --i)
2115             {
2116                 const char *cp = xpath[i].part;
2117                 if (*cp)
2118                 {
2119                     for (;*cp; cp++)
2120                         if (*cp == '*')
2121                         {
2122                             memcpy (xpath_rev + len, "[^/]*", 5);
2123                             len += 5;
2124                         }
2125                         else if (*cp == ' ')
2126                         {
2127
2128                             xpath_rev[len++] = 1;
2129                             xpath_rev[len++] = ' ';
2130                         }
2131
2132                         else
2133                             xpath_rev[len++] = *cp;
2134                     xpath_rev[len++] = '/';
2135                 }
2136                 else if (i == 1)  /* // case */
2137                 {
2138                     xpath_rev[len++] = '.';
2139                     xpath_rev[len++] = '*';
2140                 }
2141             }
2142             xpath_rev[len] = 0;
2143
2144             if (xpath[level].predicate &&
2145                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
2146                 xpath[level].predicate->u.relation.name[0])
2147             {
2148                 WRBUF wbuf = wrbuf_alloc();
2149                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
2150                 if (xpath[level].predicate->u.relation.value)
2151                 {
2152                     const char *cp = xpath[level].predicate->u.relation.value;
2153                     wrbuf_putc(wbuf, '=');
2154
2155                     while (*cp)
2156                     {
2157                         if (strchr(REGEX_CHARS, *cp))
2158                             wrbuf_putc(wbuf, '\\');
2159                         wrbuf_putc(wbuf, *cp);
2160                         cp++;
2161                     }
2162                 }
2163                 wrbuf_puts(wbuf, "");
2164                 rset_attr = xpath_trunc (
2165                     zh, stream, '0', wrbuf_buf(wbuf), 3, curAttributeSet);
2166                 wrbuf_free(wbuf, 1);
2167             }
2168             else
2169             {
2170                 if (!first_path)
2171                     continue;
2172             }
2173             yaz_log (LOG_DEBUG, "xpath_rev (%d) = %s", level, xpath_rev);
2174             if (strlen(xpath_rev))
2175             {
2176                 rset_start_tag = xpath_trunc(zh, stream,
2177                                          '0', xpath_rev, 1, curAttributeSet);
2178
2179                 rset_end_tag = xpath_trunc(zh, stream,
2180                                        '0', xpath_rev, 2, curAttributeSet);
2181
2182                 parms.key_size = sizeof(struct it_key);
2183                 parms.cmp = key_compare_it;
2184                 parms.rset_l = rset_start_tag;
2185                 parms.rset_m = rset;
2186                 parms.rset_r = rset_end_tag;
2187                 parms.rset_attr = rset_attr;
2188                 parms.printer = key_print_it;
2189                 rset = rset_create (rset_kind_between, &parms);
2190             }
2191             first_path = 0;
2192         }
2193     }
2194
2195     return rset;
2196 }
2197
2198
2199
2200 static RSET rpn_search_APT (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2201                             oid_value attributeSet, NMEM stream,
2202                             Z_SortKeySpecList *sort_sequence,
2203                             int num_bases, char **basenames)
2204 {
2205     unsigned reg_id;
2206     char *search_type = NULL;
2207     char rank_type[128];
2208     int complete_flag;
2209     int sort_flag;
2210     char termz[IT_MAX_WORD+1];
2211     RSET rset = 0;
2212     int xpath_len;
2213     int xpath_use = 0;
2214     struct xpath_location_step xpath[10];
2215
2216     zebra_maps_attr (zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2217                      rank_type, &complete_flag, &sort_flag);
2218
2219     yaz_log(LOG_DEBUG, "reg_id=%c", reg_id);
2220     yaz_log(LOG_DEBUG, "complete_flag=%d", complete_flag);
2221     yaz_log(LOG_DEBUG, "search_type=%s", search_type);
2222     yaz_log(LOG_DEBUG, "rank_type=%s", rank_type);
2223
2224     if (zapt_term_to_utf8(zh, zapt, termz))
2225         return 0;
2226
2227     if (sort_flag)
2228         return rpn_sort_spec (zh, zapt, attributeSet, stream, sort_sequence,
2229                               rank_type);
2230     xpath_len = parse_xpath(zh, zapt, attributeSet, xpath, 10, stream);
2231     if (xpath_len >= 0)
2232     {
2233         xpath_use = 1016;
2234         if (xpath[xpath_len-1].part[0] == '@')
2235             xpath_use = 1015;
2236     }
2237
2238     if (!strcmp (search_type, "phrase"))
2239     {
2240         rset = rpn_search_APT_phrase (zh, zapt, termz, attributeSet, stream,
2241                                       reg_id, complete_flag, rank_type,
2242                                       xpath_use,
2243                                       num_bases, basenames);
2244     }
2245     else if (!strcmp (search_type, "and-list"))
2246     {
2247         rset = rpn_search_APT_and_list (zh, zapt, termz, attributeSet, stream,
2248                                         reg_id, complete_flag, rank_type,
2249                                         xpath_use,
2250                                         num_bases, basenames);
2251     }
2252     else if (!strcmp (search_type, "or-list"))
2253     {
2254         rset = rpn_search_APT_or_list (zh, zapt, termz, attributeSet, stream,
2255                                        reg_id, complete_flag, rank_type,
2256                                        xpath_use,
2257                                        num_bases, basenames);
2258     }
2259     else if (!strcmp (search_type, "local"))
2260     {
2261         rset = rpn_search_APT_local (zh, zapt, termz, attributeSet, stream,
2262                                      rank_type);
2263     }
2264     else if (!strcmp (search_type, "numeric"))
2265     {
2266         rset = rpn_search_APT_numeric (zh, zapt, termz, attributeSet, stream,
2267                                        reg_id, complete_flag, rank_type,
2268                                        xpath_use,
2269                                        num_bases, basenames);
2270     }
2271     else if (!strcmp (search_type, "always"))
2272     {
2273         rset = 0;
2274     }
2275     else
2276         zh->errCode = 118;
2277     return rpn_search_xpath (zh, attributeSet, num_bases, basenames,
2278                              stream, rank_type, rset, xpath_len, xpath);
2279 }
2280
2281 static RSET rpn_search_structure (ZebraHandle zh, Z_RPNStructure *zs,
2282                                   oid_value attributeSet, NMEM stream,
2283                                   Z_SortKeySpecList *sort_sequence,
2284                                   int num_bases, char **basenames)
2285 {
2286     RSET r = NULL;
2287     if (zs->which == Z_RPNStructure_complex)
2288     {
2289         Z_Operator *zop = zs->u.complex->roperator;
2290         rset_bool_parms bool_parms;
2291
2292         bool_parms.rset_l = rpn_search_structure (zh, zs->u.complex->s1,
2293                                                   attributeSet, stream,
2294                                                   sort_sequence,
2295                                                   num_bases, basenames);
2296         if (bool_parms.rset_l == NULL)
2297             return NULL;
2298         bool_parms.rset_r = rpn_search_structure (zh, zs->u.complex->s2,
2299                                                   attributeSet, stream,
2300                                                   sort_sequence,
2301                                                   num_bases, basenames);
2302         if (bool_parms.rset_r == NULL)
2303         {
2304             rset_delete (bool_parms.rset_l);
2305             return NULL;
2306         }
2307         bool_parms.key_size = sizeof(struct it_key);
2308         bool_parms.cmp = key_compare_it;
2309         bool_parms.log_item = key_logdump_txt;
2310
2311         switch (zop->which)
2312         {
2313         case Z_Operator_and:
2314             if (res_get(zh->res, "rsetforward"))
2315                 r = rset_create (rset_kind_and_forward, &bool_parms);
2316             else
2317                 r = rset_create (rset_kind_and, &bool_parms);
2318             break;
2319         case Z_Operator_or:
2320             r = rset_create (rset_kind_or, &bool_parms);
2321             break;
2322         case Z_Operator_and_not:
2323             r = rset_create (rset_kind_not, &bool_parms);
2324             break;
2325         case Z_Operator_prox:
2326             if (zop->u.prox->which != Z_ProximityOperator_known)
2327             {
2328                 zh->errCode = 132;
2329                 return NULL;
2330             }
2331             if (*zop->u.prox->u.known != Z_ProxUnit_word)
2332             {
2333                 char *val = (char *) nmem_malloc (stream, 16);
2334                 zh->errCode = 132;
2335                 zh->errString = val;
2336                 sprintf (val, "%d", *zop->u.prox->u.known);
2337                 return NULL;
2338             }
2339             else
2340             {
2341                 /* new / old prox */
2342                 rset_prox_parms parms;
2343                 RSET twosets[2];
2344
2345                 twosets[0] = bool_parms.rset_l;
2346                 twosets[1] = bool_parms.rset_r;
2347                 parms.rset = twosets;
2348                 parms.rset_no = 2;
2349                 parms.ordered = *zop->u.prox->ordered;
2350                 parms.exclusion = (!zop->u.prox->exclusion ? 0 :
2351                                    *zop->u.prox->exclusion);
2352                 parms.relation = *zop->u.prox->relationType;
2353                 parms.distance = *zop->u.prox->distance;
2354                 parms.key_size = sizeof(struct it_key);
2355                 parms.cmp = key_compare_it;
2356                 parms.getseq = key_get_seq;
2357                 parms.log_item = key_logdump_txt;
2358                 r = rset_create(rset_kind_prox, &parms);
2359             }
2360             break;
2361         default:
2362             zh->errCode = 110;
2363             return NULL;
2364         }
2365     }
2366     else if (zs->which == Z_RPNStructure_simple)
2367     {
2368         if (zs->u.simple->which == Z_Operand_APT)
2369         {
2370             yaz_log(LOG_DEBUG, "rpn_search_APT");
2371             r = rpn_search_APT (zh, zs->u.simple->u.attributesPlusTerm,
2372                                 attributeSet, stream, sort_sequence,
2373                                 num_bases, basenames);
2374         }
2375         else if (zs->u.simple->which == Z_Operand_resultSetId)
2376         {
2377             yaz_log(LOG_DEBUG, "rpn_search_ref");
2378             r = resultSetRef (zh, zs->u.simple->u.resultSetId);
2379             if (!r)
2380             {
2381                 r = rset_create (rset_kind_null, NULL);
2382                 zh->errCode = 30;
2383                 zh->errString =
2384                     nmem_strdup (stream, zs->u.simple->u.resultSetId);
2385                 return 0;
2386             }
2387             else
2388                 rset_dup(r);
2389         }
2390         else
2391         {
2392             zh->errCode = 3;
2393             return 0;
2394         }
2395     }
2396     else
2397     {
2398         zh->errCode = 3;
2399         return 0;
2400     }
2401     return r;
2402 }
2403
2404
2405 RSET rpn_search (ZebraHandle zh, NMEM nmem,
2406                  Z_RPNQuery *rpn, int num_bases, char **basenames,
2407                  const char *setname,
2408                  ZebraSet sset)
2409 {
2410     RSET rset;
2411     oident *attrset;
2412     oid_value attributeSet;
2413     Z_SortKeySpecList *sort_sequence;
2414     int sort_status, i;
2415
2416     zh->errCode = 0;
2417     zh->errString = NULL;
2418     zh->hits = 0;
2419
2420     sort_sequence = (Z_SortKeySpecList *)
2421         nmem_malloc (nmem, sizeof(*sort_sequence));
2422     sort_sequence->num_specs = 10;
2423     sort_sequence->specs = (Z_SortKeySpec **)
2424         nmem_malloc (nmem, sort_sequence->num_specs *
2425                      sizeof(*sort_sequence->specs));
2426     for (i = 0; i<sort_sequence->num_specs; i++)
2427         sort_sequence->specs[i] = 0;
2428
2429     attrset = oid_getentbyoid (rpn->attributeSetId);
2430     attributeSet = attrset->value;
2431     rset = rpn_search_structure (zh, rpn->RPNStructure, attributeSet,
2432                                  nmem, sort_sequence, num_bases, basenames);
2433     if (!rset)
2434         return 0;
2435
2436     if (zh->errCode)
2437         yaz_log(LOG_DEBUG, "search error: %d", zh->errCode);
2438
2439     for (i = 0; sort_sequence->specs[i]; i++)
2440         ;
2441     sort_sequence->num_specs = i;
2442     if (!i)
2443         resultSetRank (zh, sset, rset);
2444     else
2445     {
2446         yaz_log(LOG_DEBUG, "resultSetSortSingle in rpn_search");
2447         resultSetSortSingle (zh, nmem, sset, rset,
2448                              sort_sequence, &sort_status);
2449         if (zh->errCode)
2450         {
2451             yaz_log(LOG_DEBUG, "resultSetSortSingle status = %d", zh->errCode);
2452         }
2453     }
2454     return rset;
2455 }
2456
2457 struct scan_info_entry {
2458     char *term;
2459     ISAMS_P isam_p;
2460 };
2461
2462 struct scan_info {
2463     struct scan_info_entry *list;
2464     ODR odr;
2465     int before, after;
2466     char prefix[20];
2467 };
2468
2469 static int scan_handle (char *name, const char *info, int pos, void *client)
2470 {
2471     int len_prefix, idx;
2472     struct scan_info *scan_info = (struct scan_info *) client;
2473
2474     len_prefix = strlen(scan_info->prefix);
2475     if (memcmp (name, scan_info->prefix, len_prefix))
2476         return 1;
2477     if (pos > 0)        idx = scan_info->after - pos + scan_info->before;
2478     else
2479         idx = - pos - 1;
2480     scan_info->list[idx].term = (char *)
2481         odr_malloc (scan_info->odr, strlen(name + len_prefix)+1);
2482     strcpy (scan_info->list[idx].term, name + len_prefix);
2483     assert (*info == sizeof(ISAMS_P));
2484     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAMS_P));
2485     return 0;
2486 }
2487
2488 static void scan_term_untrans (ZebraHandle zh, NMEM stream, int reg_type,
2489                                char **dst, const char *src)
2490 {
2491     char term_src[IT_MAX_WORD];
2492     char term_dst[IT_MAX_WORD];
2493
2494     term_untrans (zh, reg_type, term_src, src);
2495
2496     if (zh->iconv_from_utf8 != 0)
2497     {
2498         int len;
2499         char *inbuf = term_src;
2500         size_t inleft = strlen(term_src);
2501         char *outbuf = term_dst;
2502         size_t outleft = sizeof(term_dst)-1;
2503         size_t ret;
2504
2505         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2506                          &outbuf, &outleft);
2507         if (ret == (size_t)(-1))
2508             len = 0;
2509         else
2510             len = outbuf - term_dst;
2511         *dst = nmem_malloc (stream, len + 1);
2512         if (len > 0)
2513             memcpy (*dst, term_dst, len);
2514         (*dst)[len] = '\0';
2515     }
2516     else
2517         *dst = nmem_strdup (stream, term_src);
2518 }
2519
2520 static void count_set (RSET r, int *count)
2521 {
2522     int psysno = 0;
2523     int kno = 0;
2524     struct it_key key;
2525     RSFD rfd;
2526     int term_index;
2527
2528     yaz_log(LOG_DEBUG, "count_set");
2529
2530     *count = 0;
2531     rfd = rset_open (r, RSETF_READ);
2532     while (rset_read (r, rfd, &key, &term_index))
2533     {
2534         if (key.sysno != psysno)
2535         {
2536             psysno = key.sysno;
2537             (*count)++;
2538         }
2539         kno++;
2540     }
2541     rset_close (r, rfd);
2542     yaz_log(LOG_DEBUG, "%d keys, %d records", kno, *count);
2543 }
2544
2545 void rpn_scan (ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2546                oid_value attributeset,
2547                int num_bases, char **basenames,
2548                int *position, int *num_entries, ZebraScanEntry **list,
2549                int *is_partial, RSET limit_set, int return_zero)
2550 {
2551     int i;
2552     int pos = *position;
2553     int num = *num_entries;
2554     int before;
2555     int after;
2556     int base_no;
2557     char termz[IT_MAX_WORD+20];
2558     AttrType use;
2559     int use_value;
2560     const char *use_string = 0;
2561     struct scan_info *scan_info_array;
2562     ZebraScanEntry *glist;
2563     int ords[32], ord_no = 0;
2564     int ptr[32];
2565
2566     int bases_ok = 0;     /* no of databases with OK attribute */
2567     int errCode = 0;      /* err code (if any is not OK) */
2568     char *errString = 0;  /* addinfo */
2569
2570     unsigned reg_id;
2571     char *search_type = NULL;
2572     char rank_type[128];
2573     int complete_flag;
2574     int sort_flag;
2575
2576     *list = 0;
2577
2578     if (attributeset == VAL_NONE)
2579         attributeset = VAL_BIB1;
2580
2581     if (!limit_set)
2582     {
2583         AttrType termset;
2584         int termset_value_numeric;
2585         const char *termset_value_string;
2586         attr_init (&termset, zapt, 8);
2587         termset_value_numeric =
2588             attr_find_ex (&termset, NULL, &termset_value_string);
2589         if (termset_value_numeric != -1)
2590         {
2591             char resname[32];
2592             const char *termset_name = 0;
2593
2594             if (termset_value_numeric != -2)
2595             {
2596
2597                 sprintf (resname, "%d", termset_value_numeric);
2598                 termset_name = resname;
2599             }
2600             else
2601                 termset_name = termset_value_string;
2602
2603             limit_set = resultSetRef (zh, termset_name);
2604         }
2605     }
2606
2607     yaz_log (LOG_DEBUG, "position = %d, num = %d set=%d",
2608              pos, num, attributeset);
2609
2610     attr_init (&use, zapt, 1);
2611     use_value = attr_find_ex (&use, &attributeset, &use_string);
2612
2613     if (zebra_maps_attr (zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2614                          rank_type, &complete_flag, &sort_flag))
2615     {
2616         *num_entries = 0;
2617         zh->errCode = 113;
2618         return ;
2619     }
2620     yaz_log (LOG_DEBUG, "use_value = %d", use_value);
2621
2622     if (use_value == -1)
2623         use_value = 1016;
2624     for (base_no = 0; base_no < num_bases && ord_no < 32; base_no++)
2625     {
2626         int r;
2627         attent attp;
2628         data1_local_attribute *local_attr;
2629
2630         if ((r=att_getentbyatt (zh, &attp, attributeset, use_value,
2631                                 use_string)))
2632         {
2633             yaz_log(LOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
2634                   attributeset, use_value);
2635             if (r == -1)
2636             {
2637                 char val_str[32];
2638                 sprintf (val_str, "%d", use_value);
2639                 errCode = 114;
2640                 errString = odr_strdup (stream, val_str);
2641             }
2642             else
2643                 errCode = 121;
2644             continue;
2645         }
2646         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2647         {
2648             zh->errString = basenames[base_no];
2649             zh->errCode = 109; /* Database unavailable */
2650             *num_entries = 0;
2651             return;
2652         }
2653         bases_ok++;
2654         for (local_attr = attp.local_attributes; local_attr && ord_no < 32;
2655              local_attr = local_attr->next)
2656         {
2657             int ord;
2658
2659             ord = zebraExplain_lookupSU (zh->reg->zei, attp.attset_ordinal,
2660                                          local_attr->local);
2661             if (ord > 0)
2662                 ords[ord_no++] = ord;
2663         }
2664     }
2665     if (!bases_ok && errCode)
2666     {
2667         zh->errCode = errCode;
2668         zh->errString = errString;
2669         *num_entries = 0;
2670     }
2671     if (ord_no == 0)
2672     {
2673         *num_entries = 0;
2674         return;
2675     }
2676     /* prepare dictionary scanning */
2677     if (num < 1)
2678     {
2679         *num_entries = 0;
2680         return;
2681     }
2682     before = pos-1;
2683     if (before < 0)
2684         before = 0;
2685     after = 1+num-pos;
2686     if (after < 0)
2687         after = 0;
2688     scan_info_array = (struct scan_info *)
2689         odr_malloc (stream, ord_no * sizeof(*scan_info_array));
2690     for (i = 0; i < ord_no; i++)
2691     {
2692         int j, prefix_len = 0;
2693         int before_tmp = before, after_tmp = after;
2694         struct scan_info *scan_info = scan_info_array + i;
2695         struct rpn_char_map_info rcmi;
2696
2697         rpn_char_map_prepare (zh->reg, reg_id, &rcmi);
2698
2699         scan_info->before = before;
2700         scan_info->after = after;
2701         scan_info->odr = stream;
2702
2703         scan_info->list = (struct scan_info_entry *)
2704             odr_malloc (stream, (before+after) * sizeof(*scan_info->list));
2705         for (j = 0; j<before+after; j++)
2706             scan_info->list[j].term = NULL;
2707
2708         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2709         termz[prefix_len++] = reg_id;
2710         termz[prefix_len] = 0;
2711         strcpy (scan_info->prefix, termz);
2712
2713         if (trans_scan_term (zh, zapt, termz+prefix_len, reg_id))
2714             return ;
2715
2716         dict_scan (zh->reg->dict, termz, &before_tmp, &after_tmp,
2717                    scan_info, scan_handle);
2718     }
2719     glist = (ZebraScanEntry *)
2720         odr_malloc (stream, (before+after)*sizeof(*glist));
2721
2722     /* consider terms after main term */
2723     for (i = 0; i < ord_no; i++)
2724         ptr[i] = before;
2725
2726     *is_partial = 0;
2727     for (i = 0; i<after; i++)
2728     {
2729         int j, j0 = -1;
2730         const char *mterm = NULL;
2731         const char *tst;
2732         RSET rset;
2733         int lo = i + pos-1; /* offset in result list */
2734
2735         for (j = 0; j < ord_no; j++)
2736         {
2737             if (ptr[j] < before+after &&
2738                 (tst=scan_info_array[j].list[ptr[j]].term) &&
2739                 (!mterm || strcmp (tst, mterm) < 0))
2740             {
2741                 j0 = j;
2742                 mterm = tst;
2743             }
2744         }
2745         if (j0 == -1)
2746             break;
2747         if (lo >= 0)
2748         {
2749             scan_term_untrans (zh, stream->mem, reg_id,
2750                                &glist[lo].term, mterm);
2751             rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2752                               glist[lo].term, strlen(glist[lo].term),
2753                               NULL, 0, zapt->term->which);
2754         }
2755         ptr[j0]++;
2756         for (j = j0+1; j<ord_no; j++)
2757         {
2758             if (ptr[j] < before+after &&
2759                 (tst=scan_info_array[j].list[ptr[j]].term) &&
2760                 !strcmp (tst, mterm))
2761             {
2762                 if (lo >= 0)
2763                 {
2764                     rset_bool_parms bool_parms;
2765                     RSET rset2;
2766
2767                     rset2 =
2768                         rset_trunc(zh,
2769                                    &scan_info_array[j].list[ptr[j]].isam_p, 1,
2770                                    glist[lo].term,
2771                                    strlen(glist[lo].term), NULL, 0,
2772                                    zapt->term->which);
2773
2774                     bool_parms.key_size = sizeof(struct it_key);
2775                     bool_parms.cmp = key_compare_it;
2776                     bool_parms.log_item = key_logdump_txt;
2777                     bool_parms.rset_l = rset;
2778                     bool_parms.rset_r = rset2;
2779
2780                     rset = rset_create (rset_kind_or, &bool_parms);
2781                 }
2782                 ptr[j]++;
2783             }
2784         }
2785         if (lo >= 0)
2786         {
2787             if (limit_set)
2788             {
2789                 rset_bool_parms bool_parms;
2790
2791                 bool_parms.key_size = sizeof(struct it_key);
2792                 bool_parms.cmp = key_compare_it;
2793                 bool_parms.log_item = key_logdump_txt;
2794                 bool_parms.rset_l = rset;
2795                 bool_parms.rset_r = rset_dup(limit_set);
2796
2797                 rset = rset_create (rset_kind_and, &bool_parms);
2798             }
2799             count_set (rset, &glist[lo].occurrences);
2800             rset_delete (rset);
2801         }
2802     }
2803     if (i < after)
2804     {
2805         *num_entries -= (after-i);
2806         *is_partial = 1;
2807         if (*num_entries <= 0)
2808         {
2809             *num_entries = 0;
2810             return;
2811         }
2812     }
2813
2814     /* consider terms before main term */
2815     for (i = 0; i<ord_no; i++)
2816         ptr[i] = 0;
2817
2818     for (i = 0; i<before; i++)
2819     {
2820         int j, j0 = -1;
2821         const char *mterm = NULL;
2822         const char *tst;
2823         RSET rset;
2824
2825         for (j = 0; j <ord_no; j++)
2826         {
2827             if (ptr[j] < before &&
2828                 (tst=scan_info_array[j].list[before-1-ptr[j]].term) &&
2829                 (!mterm || strcmp (tst, mterm) > 0))
2830             {
2831                 j0 = j;
2832                 mterm = tst;
2833             }
2834         }
2835         if (j0 == -1)
2836             break;
2837
2838         scan_term_untrans (zh, stream->mem, reg_id,
2839                            &glist[before-1-i].term, mterm);
2840
2841         rset = rset_trunc
2842                (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
2843                 glist[before-1-i].term, strlen(glist[before-1-i].term),
2844                 NULL, 0, zapt->term->which);
2845
2846         ptr[j0]++;
2847
2848         for (j = j0+1; j<ord_no; j++)
2849         {
2850             if (ptr[j] < before &&
2851                 (tst=scan_info_array[j].list[before-1-ptr[j]].term) &&
2852                 !strcmp (tst, mterm))
2853             {
2854                 rset_bool_parms bool_parms;
2855                 RSET rset2;
2856
2857                 rset2 = rset_trunc (zh,
2858                          &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
2859                                     glist[before-1-i].term,
2860                                     strlen(glist[before-1-i].term), NULL, 0,
2861                                     zapt->term->which);
2862
2863                 bool_parms.key_size = sizeof(struct it_key);
2864                 bool_parms.cmp = key_compare_it;
2865                 bool_parms.log_item = key_logdump_txt;
2866                 bool_parms.rset_l = rset;
2867                 bool_parms.rset_r = rset2;
2868
2869                 rset = rset_create (rset_kind_or, &bool_parms);
2870
2871                 ptr[j]++;
2872             }
2873         }
2874         if (limit_set)
2875         {
2876             rset_bool_parms bool_parms;
2877
2878             bool_parms.key_size = sizeof(struct it_key);
2879             bool_parms.cmp = key_compare_it;
2880             bool_parms.log_item = key_logdump_txt;
2881             bool_parms.rset_l = rset;
2882             bool_parms.rset_r = rset_dup(limit_set);
2883
2884             rset = rset_create (rset_kind_and, &bool_parms);
2885         }
2886         count_set (rset, &glist[before-1-i].occurrences);
2887         rset_delete (rset);
2888     }
2889     i = before-i;
2890     if (i)
2891     {
2892         *is_partial = 1;
2893         *position -= i;
2894         *num_entries -= i;
2895         if (*num_entries <= 0)
2896         {
2897             *num_entries = 0;
2898             return;
2899         }
2900     }
2901     *list = glist + i;               /* list is set to first 'real' entry */
2902
2903     yaz_log(LOG_DEBUG, "position = %d, num_entries = %d",
2904           *position, *num_entries);
2905     if (zh->errCode)
2906         yaz_log(LOG_DEBUG, "scan error: %d", zh->errCode);
2907 }
2908