index/zrpn.c

   1 /* $Id: zrpn.c,v 1.223 2006-07-03 10:52:48 adam Exp $
   2    Copyright (C) 1995-2006
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with Zebra; see the file LICENSE.zebra.  If not, write to the
  19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  20 02111-1307, USA.
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #ifdef WIN32
  26 #include <io.h>
  27 #endif
  28 #if HAVE_UNISTD_H
  29 #include <unistd.h>
  30 #endif
  31 #include <ctype.h>
  32
  33 #include <yaz/diagbib1.h>
  34 #include "index.h"
  35 #include <zebra_xpath.h>
  36 #include <attrfind.h>
  37 #include <charmap.h>
  38 #include <rset.h>
  39
  40 struct rpn_char_map_info
  41 {
  42     ZebraMaps zm;
  43     int reg_type;
  44 };
  45
  46 static int log_level_set = 0;
  47 static int log_level_rpn = 0;
  48
  49 #define TERMSET_DISABLE 1
  50
  51 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  52 {
  53     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  54     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  55 #if 0
  56     if (out && *out)
  57     {
  58         const char *outp = *out;
  59         yaz_log(YLOG_LOG, "---");
  60         while (*outp)
  61         {
  62             yaz_log(YLOG_LOG, "%02X", *outp);
  63             outp++;
  64         }
  65     }
  66 #endif
  67     return out;
  68 }
  69
  70 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  71                                  struct rpn_char_map_info *map_info)
  72 {
  73     map_info->zm = reg->zebra_maps;
  74     map_info->reg_type = reg_type;
  75     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  76 }
  77
  78 #define TERM_COUNT
  79
  80 struct grep_info {
  81 #ifdef TERM_COUNT
  82     int *term_no;
  83 #endif
  84     ISAM_P *isam_p_buf;
  85     int isam_p_size;
  86     int isam_p_indx;
  87     ZebraHandle zh;
  88     int reg_type;
  89     ZebraSet termset;
  90 };
  91
  92 void zebra_term_untrans(ZebraHandle zh, int reg_type,
  93                         char *dst, const char *src)
  94 {
  95     int len = 0;
  96     while (*src)
  97     {
  98         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
  99                                            reg_type, &src);
 100         if (!cp)
 101         {
 102             if (len < IT_MAX_WORD-1)
 103                 dst[len++] = *src;
 104             src++;
 105         }
 106         else
 107             while (*cp && len < IT_MAX_WORD-1)
 108                 dst[len++] = *cp++;
 109     }
 110     dst[len] = '\0';
 111 }
 112
 113 static void add_isam_p(const char *name, const char *info,
 114                        struct grep_info *p)
 115 {
 116     if (!log_level_set)
 117     {
 118         log_level_rpn = yaz_log_module_level("rpn");
 119         log_level_set = 1;
 120     }
 121     if (p->isam_p_indx == p->isam_p_size)
 122     {
 123         ISAM_P *new_isam_p_buf;
 124 #ifdef TERM_COUNT
 125         int *new_term_no;
 126 #endif
 127         p->isam_p_size = 2*p->isam_p_size + 100;
 128         new_isam_p_buf = (ISAM_P *) xmalloc(sizeof(*new_isam_p_buf) *
 129                                             p->isam_p_size);
 130         if (p->isam_p_buf)
 131         {
 132             memcpy(new_isam_p_buf, p->isam_p_buf,
 133                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 134             xfree(p->isam_p_buf);
 135         }
 136         p->isam_p_buf = new_isam_p_buf;
 137
 138 #ifdef TERM_COUNT
 139         new_term_no = (int *) xmalloc(sizeof(*new_term_no) * p->isam_p_size);
 140         if (p->term_no)
 141         {
 142             memcpy(new_term_no, p->isam_p_buf,
 143                     p->isam_p_indx * sizeof(*p->term_no));
 144             xfree(p->term_no);
 145         }
 146         p->term_no = new_term_no;
 147 #endif
 148     }
 149     assert(*info == sizeof(*p->isam_p_buf));
 150     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 151
 152     if (p->termset)
 153     {
 154         const char *db;
 155         char term_tmp[IT_MAX_WORD];
 156         int ord = 0;
 157         const char *index_name;
 158         int len = key_SU_decode (&ord, (const unsigned char *) name);
 159
 160         zebra_term_untrans  (p->zh, p->reg_type, term_tmp, name+len);
 161         yaz_log(log_level_rpn, "grep: %d %c %s", ord, name[len], term_tmp);
 162         zebraExplain_lookup_ord(p->zh->reg->zei,
 163                                 ord, 0 /* index_type */, &db, &index_name);
 164         yaz_log(log_level_rpn, "grep:  db=%s index=%s", db, index_name);
 165
 166         resultSetAddTerm(p->zh, p->termset, name[len], db,
 167                          index_name, term_tmp);
 168     }
 169     (p->isam_p_indx)++;
 170 }
 171
 172 static int grep_handle(char *name, const char *info, void *p)
 173 {
 174     add_isam_p(name, info, (struct grep_info *) p);
 175     return 0;
 176 }
 177
 178 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 179                     const char *ct1, const char *ct2, int first)
 180 {
 181     const char *s1, *s0 = *src;
 182     const char **map;
 183
 184     /* skip white space */
 185     while (*s0)
 186     {
 187         if (ct1 && strchr(ct1, *s0))
 188             break;
 189         if (ct2 && strchr(ct2, *s0))
 190             break;
 191         s1 = s0;
 192         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 193         if (**map != *CHR_SPACE)
 194             break;
 195         s0 = s1;
 196     }
 197     *src = s0;
 198     return *s0;
 199 }
 200
 201
 202 static void esc_str(char *out_buf, size_t out_size,
 203                     const char *in_buf, int in_size)
 204 {
 205     int k;
 206
 207     assert(out_buf);
 208     assert(in_buf);
 209     assert(out_size > 20);
 210     *out_buf = '\0';
 211     for (k = 0; k<in_size; k++)
 212     {
 213         int c = in_buf[k] & 0xff;
 214         int pc;
 215         if (c < 32 || c > 126)
 216             pc = '?';
 217         else
 218             pc = c;
 219         sprintf(out_buf +strlen(out_buf), "%02X:%c  ", c, pc);
 220         if (strlen(out_buf) > out_size-20)
 221         {
 222             strcat(out_buf, "..");
 223             break;
 224         }
 225     }
 226 }
 227
 228 #define REGEX_CHARS " []()|.*+?!"
 229
 230 /* term_100: handle term, where trunc = none(no operators at all) */
 231 static int term_100(ZebraMaps zebra_maps, int reg_type,
 232                     const char **src, char *dst, int space_split,
 233                     char *dst_term)
 234 {
 235     const char *s0;
 236     const char **map;
 237     int i = 0;
 238     int j = 0;
 239
 240     const char *space_start = 0;
 241     const char *space_end = 0;
 242
 243     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 244         return 0;
 245     s0 = *src;
 246     while (*s0)
 247     {
 248         const char *s1 = s0;
 249         int q_map_match = 0;
 250         map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 251                                 &q_map_match);
 252         if (space_split)
 253         {
 254             if (**map == *CHR_SPACE)
 255                 break;
 256         }
 257         else  /* complete subfield only. */
 258         {
 259             if (**map == *CHR_SPACE)
 260             {   /* save space mapping for later  .. */
 261                 space_start = s1;
 262                 space_end = s0;
 263                 continue;
 264             }
 265             else if (space_start)
 266             {   /* reload last space */
 267                 while (space_start < space_end)
 268                 {
 269                     if (strchr(REGEX_CHARS, *space_start))
 270                         dst[i++] = '\\';
 271                     dst_term[j++] = *space_start;
 272                     dst[i++] = *space_start++;
 273                 }
 274                 /* and reset */
 275                 space_start = space_end = 0;
 276             }
 277         }
 278         /* add non-space char */
 279         memcpy(dst_term+j, s1, s0 - s1);
 280         j += (s0 - s1);
 281         if (!q_map_match)
 282         {
 283             while (s1 < s0)
 284             {
 285                 if (strchr(REGEX_CHARS, *s1))
 286                     dst[i++] = '\\';
 287                 dst[i++] = *s1++;
 288             }
 289         }
 290         else
 291         {
 292             char tmpbuf[80];
 293             esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 294
 295             strcpy(dst + i, map[0]);
 296             i += strlen(map[0]);
 297         }
 298     }
 299     dst[i] = '\0';
 300     dst_term[j] = '\0';
 301     *src = s0;
 302     return i;
 303 }
 304
 305 /* term_101: handle term, where trunc = Process # */
 306 static int term_101(ZebraMaps zebra_maps, int reg_type,
 307                     const char **src, char *dst, int space_split,
 308                     char *dst_term)
 309 {
 310     const char *s0;
 311     const char **map;
 312     int i = 0;
 313     int j = 0;
 314
 315     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 316         return 0;
 317     s0 = *src;
 318     while (*s0)
 319     {
 320         if (*s0 == '#')
 321         {
 322             dst[i++] = '.';
 323             dst[i++] = '*';
 324             dst_term[j++] = *s0++;
 325         }
 326         else
 327         {
 328             const char *s1 = s0;
 329             int q_map_match = 0;
 330             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 331                                     &q_map_match);
 332             if (space_split && **map == *CHR_SPACE)
 333                 break;
 334
 335             /* add non-space char */
 336             memcpy(dst_term+j, s1, s0 - s1);
 337             j += (s0 - s1);
 338             if (!q_map_match)
 339             {
 340                 while (s1 < s0)
 341                 {
 342                     if (strchr(REGEX_CHARS, *s1))
 343                         dst[i++] = '\\';
 344                     dst[i++] = *s1++;
 345                 }
 346             }
 347             else
 348             {
 349                 char tmpbuf[80];
 350                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 351
 352                 strcpy(dst + i, map[0]);
 353                 i += strlen(map[0]);
 354             }
 355         }
 356     }
 357     dst[i] = '\0';
 358     dst_term[j++] = '\0';
 359     *src = s0;
 360     return i;
 361 }
 362
 363 /* term_103: handle term, where trunc = re-2 (regular expressions) */
 364 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 365                     char *dst, int *errors, int space_split,
 366                     char *dst_term)
 367 {
 368     int i = 0;
 369     int j = 0;
 370     const char *s0;
 371     const char **map;
 372
 373     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 374         return 0;
 375     s0 = *src;
 376     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 377         isdigit(((const unsigned char *)s0)[1]))
 378     {
 379         *errors = s0[1] - '0';
 380         s0 += 3;
 381         if (*errors > 3)
 382             *errors = 3;
 383     }
 384     while (*s0)
 385     {
 386         if (strchr("^\\()[].*+?|-", *s0))
 387         {
 388             dst_term[j++] = *s0;
 389             dst[i++] = *s0++;
 390         }
 391         else
 392         {
 393             const char *s1 = s0;
 394             int q_map_match = 0;
 395             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 396                                     &q_map_match);
 397             if (space_split && **map == *CHR_SPACE)
 398                 break;
 399
 400             /* add non-space char */
 401             memcpy(dst_term+j, s1, s0 - s1);
 402             j += (s0 - s1);
 403             if (!q_map_match)
 404             {
 405                 while (s1 < s0)
 406                 {
 407                     if (strchr(REGEX_CHARS, *s1))
 408                         dst[i++] = '\\';
 409                     dst[i++] = *s1++;
 410                 }
 411             }
 412             else
 413             {
 414                 char tmpbuf[80];
 415                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 416
 417                 strcpy(dst + i, map[0]);
 418                 i += strlen(map[0]);
 419             }
 420         }
 421     }
 422     dst[i] = '\0';
 423     dst_term[j] = '\0';
 424     *src = s0;
 425
 426     return i;
 427 }
 428
 429 /* term_103: handle term, where trunc = re-1 (regular expressions) */
 430 static int term_102(ZebraMaps zebra_maps, int reg_type, const char **src,
 431                     char *dst, int space_split, char *dst_term)
 432 {
 433     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 434                     dst_term);
 435 }
 436
 437
 438 /* term_104: handle term, where trunc = Process # and ! */
 439 static int term_104(ZebraMaps zebra_maps, int reg_type,
 440                     const char **src, char *dst, int space_split,
 441                     char *dst_term)
 442 {
 443     const char *s0;
 444     const char **map;
 445     int i = 0;
 446     int j = 0;
 447
 448     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 449         return 0;
 450     s0 = *src;
 451     while (*s0)
 452     {
 453         if (*s0 == '?')
 454         {
 455             dst_term[j++] = *s0++;
 456             if (*s0 >= '0' && *s0 <= '9')
 457             {
 458                 int limit = 0;
 459                 while (*s0 >= '0' && *s0 <= '9')
 460                 {
 461                     limit = limit * 10 + (*s0 - '0');
 462                     dst_term[j++] = *s0++;
 463                 }
 464                 if (limit > 20)
 465                     limit = 20;
 466                 while (--limit >= 0)
 467                 {
 468                     dst[i++] = '.';
 469                     dst[i++] = '?';
 470                 }
 471             }
 472             else
 473             {
 474                 dst[i++] = '.';
 475                 dst[i++] = '*';
 476             }
 477         }
 478         else if (*s0 == '*')
 479         {
 480             dst[i++] = '.';
 481             dst[i++] = '*';
 482             dst_term[j++] = *s0++;
 483         }
 484         else if (*s0 == '#')
 485         {
 486             dst[i++] = '.';
 487             dst_term[j++] = *s0++;
 488         }
 489         else
 490         {
 491             const char *s1 = s0;
 492             int q_map_match = 0;
 493             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 494                                     &q_map_match);
 495             if (space_split && **map == *CHR_SPACE)
 496                 break;
 497
 498             /* add non-space char */
 499             memcpy(dst_term+j, s1, s0 - s1);
 500             j += (s0 - s1);
 501             if (!q_map_match)
 502             {
 503                 while (s1 < s0)
 504                 {
 505                     if (strchr(REGEX_CHARS, *s1))
 506                         dst[i++] = '\\';
 507                     dst[i++] = *s1++;
 508                 }
 509             }
 510             else
 511             {
 512                 char tmpbuf[80];
 513                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 514
 515                 strcpy(dst + i, map[0]);
 516                 i += strlen(map[0]);
 517             }
 518         }
 519     }
 520     dst[i] = '\0';
 521     dst_term[j++] = '\0';
 522     *src = s0;
 523     return i;
 524 }
 525
 526 /* term_105/106: handle term, where trunc = Process * and ! and right trunc */
 527 static int term_105(ZebraMaps zebra_maps, int reg_type,
 528                     const char **src, char *dst, int space_split,
 529                     char *dst_term, int right_truncate)
 530 {
 531     const char *s0;
 532     const char **map;
 533     int i = 0;
 534     int j = 0;
 535
 536     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 537         return 0;
 538     s0 = *src;
 539     while (*s0)
 540     {
 541         if (*s0 == '*')
 542         {
 543             dst[i++] = '.';
 544             dst[i++] = '*';
 545             dst_term[j++] = *s0++;
 546         }
 547         else if (*s0 == '!')
 548         {
 549             dst[i++] = '.';
 550             dst_term[j++] = *s0++;
 551         }
 552         else
 553         {
 554             const char *s1 = s0;
 555             int q_map_match = 0;
 556             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 557                                     &q_map_match);
 558             if (space_split && **map == *CHR_SPACE)
 559                 break;
 560
 561             /* add non-space char */
 562             memcpy(dst_term+j, s1, s0 - s1);
 563             j += (s0 - s1);
 564             if (!q_map_match)
 565             {
 566                 while (s1 < s0)
 567                 {
 568                     if (strchr(REGEX_CHARS, *s1))
 569                         dst[i++] = '\\';
 570                     dst[i++] = *s1++;
 571                 }
 572             }
 573             else
 574             {
 575                 char tmpbuf[80];
 576                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 577
 578                 strcpy(dst + i, map[0]);
 579                 i += strlen(map[0]);
 580             }
 581         }
 582     }
 583     if (right_truncate)
 584     {
 585         dst[i++] = '.';
 586         dst[i++] = '*';
 587     }
 588     dst[i] = '\0';
 589
 590     dst_term[j++] = '\0';
 591     *src = s0;
 592     return i;
 593 }
 594
 595
 596 /* gen_regular_rel - generate regular expression from relation
 597  *  val:     border value (inclusive)
 598  *  islt:    1 if <=; 0 if >=.
 599  */
 600 static void gen_regular_rel(char *dst, int val, int islt)
 601 {
 602     int dst_p;
 603     int w, d, i;
 604     int pos = 0;
 605     char numstr[20];
 606
 607     yaz_log(YLOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 608     if (val >= 0)
 609     {
 610         if (islt)
 611             strcpy(dst, "(-[0-9]+|(");
 612         else
 613             strcpy(dst, "((");
 614     }
 615     else
 616     {
 617         if (!islt)
 618         {
 619             strcpy(dst, "([0-9]+|-(");
 620             dst_p = strlen(dst);
 621             islt = 1;
 622         }
 623         else
 624         {
 625             strcpy(dst, "(-(");
 626             islt = 0;
 627         }
 628         val = -val;
 629     }
 630     dst_p = strlen(dst);
 631     sprintf(numstr, "%d", val);
 632     for (w = strlen(numstr); --w >= 0; pos++)
 633     {
 634         d = numstr[w];
 635         if (pos > 0)
 636         {
 637             if (islt)
 638             {
 639                 if (d == '0')
 640                     continue;
 641                 d--;
 642             }
 643             else
 644             {
 645                 if (d == '9')
 646                     continue;
 647                 d++;
 648             }
 649         }
 650
 651         strcpy(dst + dst_p, numstr);
 652         dst_p = strlen(dst) - pos - 1;
 653
 654         if (islt)
 655         {
 656             if (d != '0')
 657             {
 658                 dst[dst_p++] = '[';
 659                 dst[dst_p++] = '0';
 660                 dst[dst_p++] = '-';
 661                 dst[dst_p++] = d;
 662                 dst[dst_p++] = ']';
 663             }
 664             else
 665                 dst[dst_p++] = d;
 666         }
 667         else
 668         {
 669             if (d != '9')
 670             {
 671                 dst[dst_p++] = '[';
 672                 dst[dst_p++] = d;
 673                 dst[dst_p++] = '-';
 674                 dst[dst_p++] = '9';
 675                 dst[dst_p++] = ']';
 676             }
 677             else
 678                 dst[dst_p++] = d;
 679         }
 680         for (i = 0; i<pos; i++)
 681         {
 682             dst[dst_p++] = '[';
 683             dst[dst_p++] = '0';
 684             dst[dst_p++] = '-';
 685             dst[dst_p++] = '9';
 686             dst[dst_p++] = ']';
 687         }
 688         dst[dst_p++] = '|';
 689     }
 690     dst[dst_p] = '\0';
 691     if (islt)
 692     {
 693         /* match everything less than 10^(pos-1) */
 694         strcat(dst, "0*");
 695         for (i = 1; i<pos; i++)
 696             strcat(dst, "[0-9]?");
 697     }
 698     else
 699     {
 700         /* match everything greater than 10^pos */
 701         for (i = 0; i <= pos; i++)
 702             strcat(dst, "[0-9]");
 703         strcat(dst, "[0-9]*");
 704     }
 705     strcat(dst, "))");
 706 }
 707
 708 void string_rel_add_char(char **term_p, const char *src, int *indx)
 709 {
 710     if (src[*indx] == '\\')
 711         *(*term_p)++ = src[(*indx)++];
 712     *(*term_p)++ = src[(*indx)++];
 713 }
 714
 715 /*
 716  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 717  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 718  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 719  *              ([^-a].*|a[^-b].*|ab[c-].*)
 720  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 721  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 722  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 723  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 724  */
 725 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 726                            const char **term_sub, char *term_dict,
 727                            oid_value attributeSet,
 728                            int reg_type, int space_split, char *term_dst,
 729                            int *error_code)
 730 {
 731     AttrType relation;
 732     int relation_value;
 733     int i;
 734     char *term_tmp = term_dict + strlen(term_dict);
 735     char term_component[2*IT_MAX_WORD+20];
 736
 737     attr_init_APT(&relation, zapt, 2);
 738     relation_value = attr_find(&relation, NULL);
 739
 740     *error_code = 0;
 741     yaz_log(YLOG_DEBUG, "string relation value=%d", relation_value);
 742     switch (relation_value)
 743     {
 744     case 1:
 745         if (!term_100(zh->reg->zebra_maps, reg_type,
 746                       term_sub, term_component,
 747                       space_split, term_dst))
 748             return 0;
 749         yaz_log(log_level_rpn, "Relation <");
 750
 751         *term_tmp++ = '(';
 752         for (i = 0; term_component[i]; )
 753         {
 754             int j = 0;
 755
 756             if (i)
 757                 *term_tmp++ = '|';
 758             while (j < i)
 759                 string_rel_add_char(&term_tmp, term_component, &j);
 760
 761             *term_tmp++ = '[';
 762
 763             *term_tmp++ = '^';
 764             string_rel_add_char(&term_tmp, term_component, &i);
 765             *term_tmp++ = '-';
 766
 767             *term_tmp++ = ']';
 768             *term_tmp++ = '.';
 769             *term_tmp++ = '*';
 770
 771             if ((term_tmp - term_dict) > IT_MAX_WORD)
 772                 break;
 773         }
 774         *term_tmp++ = ')';
 775         *term_tmp = '\0';
 776         break;
 777     case 2:
 778         if (!term_100(zh->reg->zebra_maps, reg_type,
 779                       term_sub, term_component,
 780                       space_split, term_dst))
 781             return 0;
 782         yaz_log(log_level_rpn, "Relation <=");
 783
 784         *term_tmp++ = '(';
 785         for (i = 0; term_component[i]; )
 786         {
 787             int j = 0;
 788
 789             while (j < i)
 790                 string_rel_add_char(&term_tmp, term_component, &j);
 791             *term_tmp++ = '[';
 792
 793             *term_tmp++ = '^';
 794             string_rel_add_char(&term_tmp, term_component, &i);
 795             *term_tmp++ = '-';
 796
 797             *term_tmp++ = ']';
 798             *term_tmp++ = '.';
 799             *term_tmp++ = '*';
 800
 801             *term_tmp++ = '|';
 802
 803             if ((term_tmp - term_dict) > IT_MAX_WORD)
 804                 break;
 805         }
 806         for (i = 0; term_component[i]; )
 807             string_rel_add_char(&term_tmp, term_component, &i);
 808         *term_tmp++ = ')';
 809         *term_tmp = '\0';
 810         break;
 811     case 5:
 812         if (!term_100 (zh->reg->zebra_maps, reg_type,
 813                        term_sub, term_component, space_split, term_dst))
 814             return 0;
 815         yaz_log(log_level_rpn, "Relation >");
 816
 817         *term_tmp++ = '(';
 818         for (i = 0; term_component[i];)
 819         {
 820             int j = 0;
 821
 822             while (j < i)
 823                 string_rel_add_char(&term_tmp, term_component, &j);
 824             *term_tmp++ = '[';
 825
 826             *term_tmp++ = '^';
 827             *term_tmp++ = '-';
 828             string_rel_add_char(&term_tmp, term_component, &i);
 829
 830             *term_tmp++ = ']';
 831             *term_tmp++ = '.';
 832             *term_tmp++ = '*';
 833
 834             *term_tmp++ = '|';
 835
 836             if ((term_tmp - term_dict) > IT_MAX_WORD)
 837                 break;
 838         }
 839         for (i = 0; term_component[i];)
 840             string_rel_add_char(&term_tmp, term_component, &i);
 841         *term_tmp++ = '.';
 842         *term_tmp++ = '+';
 843         *term_tmp++ = ')';
 844         *term_tmp = '\0';
 845         break;
 846     case 4:
 847         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 848                       term_component, space_split, term_dst))
 849             return 0;
 850         yaz_log(log_level_rpn, "Relation >=");
 851
 852         *term_tmp++ = '(';
 853         for (i = 0; term_component[i];)
 854         {
 855             int j = 0;
 856
 857             if (i)
 858                 *term_tmp++ = '|';
 859             while (j < i)
 860                 string_rel_add_char(&term_tmp, term_component, &j);
 861             *term_tmp++ = '[';
 862
 863             if (term_component[i+1])
 864             {
 865                 *term_tmp++ = '^';
 866                 *term_tmp++ = '-';
 867                 string_rel_add_char(&term_tmp, term_component, &i);
 868             }
 869             else
 870             {
 871                 string_rel_add_char(&term_tmp, term_component, &i);
 872                 *term_tmp++ = '-';
 873             }
 874             *term_tmp++ = ']';
 875             *term_tmp++ = '.';
 876             *term_tmp++ = '*';
 877
 878             if ((term_tmp - term_dict) > IT_MAX_WORD)
 879                 break;
 880         }
 881         *term_tmp++ = ')';
 882         *term_tmp = '\0';
 883         break;
 884     case 3:
 885     case 102:
 886     case -1:
 887         if (!**term_sub)
 888             return 1;
 889         yaz_log(log_level_rpn, "Relation =");
 890         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 891                       term_component, space_split, term_dst))
 892             return 0;
 893         strcat(term_tmp, "(");
 894         strcat(term_tmp, term_component);
 895         strcat(term_tmp, ")");
 896         break;
 897     case 103:
 898         yaz_log(log_level_rpn, "Relation always matches");
 899         /* skip to end of term (we don't care what it is) */
 900         while (**term_sub != '\0')
 901             (*term_sub)++;
 902         break;
 903     default:
 904         *error_code = YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
 905         return 0;
 906     }
 907     return 1;
 908 }
 909
 910 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 911                              const char **term_sub,
 912                              oid_value attributeSet, NMEM stream,
 913                              struct grep_info *grep_info,
 914                              int reg_type, int complete_flag,
 915                              int num_bases, char **basenames,
 916                              char *term_dst,
 917                              const char *xpath_use,
 918                              struct ord_list **ol);
 919
 920 static ZEBRA_RES term_limits_APT(ZebraHandle zh,
 921                                  Z_AttributesPlusTerm *zapt,
 922                                  zint *hits_limit_value,
 923                                  const char **term_ref_id_str,
 924                                  NMEM nmem)
 925 {
 926     AttrType term_ref_id_attr;
 927     AttrType hits_limit_attr;
 928     int term_ref_id_int;
 929
 930     attr_init_APT(&hits_limit_attr, zapt, 9);
 931     *hits_limit_value  = attr_find(&hits_limit_attr, NULL);
 932
 933     attr_init_APT(&term_ref_id_attr, zapt, 10);
 934     term_ref_id_int = attr_find_ex(&term_ref_id_attr, NULL, term_ref_id_str);
 935     if (term_ref_id_int >= 0)
 936     {
 937         char *res = nmem_malloc(nmem, 20);
 938         sprintf(res, "%d", term_ref_id_int);
 939         *term_ref_id_str = res;
 940     }
 941
 942     /* no limit given ? */
 943     if (*hits_limit_value == -1)
 944     {
 945         if (*term_ref_id_str)
 946         {
 947             /* use global if term_ref is present */
 948             *hits_limit_value = zh->approx_limit;
 949         }
 950         else
 951         {
 952             /* no counting if term_ref is not present */
 953             *hits_limit_value = 0;
 954         }
 955     }
 956     else if (*hits_limit_value == 0)
 957     {
 958         /* 0 is the same as global limit */
 959         *hits_limit_value = zh->approx_limit;
 960     }
 961     yaz_log(YLOG_DEBUG, "term_limits_APT ref_id=%s limit=" ZINT_FORMAT,
 962             *term_ref_id_str ? *term_ref_id_str : "none",
 963             *hits_limit_value);
 964     return ZEBRA_OK;
 965 }
 966
 967 static ZEBRA_RES term_trunc(ZebraHandle zh,
 968                             Z_AttributesPlusTerm *zapt,
 969                             const char **term_sub,
 970                             oid_value attributeSet, NMEM stream,
 971                             struct grep_info *grep_info,
 972                             int reg_type, int complete_flag,
 973                             int num_bases, char **basenames,
 974                             char *term_dst,
 975                             const char *rank_type,
 976                             const char *xpath_use,
 977                             NMEM rset_nmem,
 978                             RSET *rset,
 979                             struct rset_key_control *kc)
 980 {
 981     ZEBRA_RES res;
 982     struct ord_list *ol;
 983     zint hits_limit_value;
 984     const char *term_ref_id_str = 0;
 985     *rset = 0;
 986
 987     term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str, stream);
 988     grep_info->isam_p_indx = 0;
 989     res = string_term(zh, zapt, term_sub, attributeSet, stream, grep_info,
 990                       reg_type, complete_flag, num_bases, basenames,
 991                       term_dst, xpath_use, &ol);
 992     if (res != ZEBRA_OK)
 993         return res;
 994     if (!*term_sub)  /* no more terms ? */
 995         return res;
 996     yaz_log(log_level_rpn, "term: %s", term_dst);
 997     *rset = rset_trunc(zh, grep_info->isam_p_buf,
 998                        grep_info->isam_p_indx, term_dst,
 999                        strlen(term_dst), rank_type, 1 /* preserve pos */,
1000                        zapt->term->which, rset_nmem,
1001                        kc, kc->scope, ol, reg_type, hits_limit_value,
1002                        term_ref_id_str);
1003     if (!*rset)
1004         return ZEBRA_FAIL;
1005     return ZEBRA_OK;
1006 }
1007
1008 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1009                              const char **term_sub,
1010                              oid_value attributeSet, NMEM stream,
1011                              struct grep_info *grep_info,
1012                              int reg_type, int complete_flag,
1013                              int num_bases, char **basenames,
1014                              char *term_dst,
1015                              const char *xpath_use,
1016                              struct ord_list **ol)
1017 {
1018     char term_dict[2*IT_MAX_WORD+4000];
1019     int j, r, base_no;
1020     AttrType truncation;
1021     int truncation_value;
1022     const char *termp;
1023     struct rpn_char_map_info rcmi;
1024     int space_split = complete_flag ? 0 : 1;
1025
1026     int bases_ok = 0;     /* no of databases with OK attribute */
1027
1028     *ol = ord_list_create(stream);
1029
1030     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1031     attr_init_APT(&truncation, zapt, 5);
1032     truncation_value = attr_find(&truncation, NULL);
1033     yaz_log(log_level_rpn, "truncation value %d", truncation_value);
1034
1035     for (base_no = 0; base_no < num_bases; base_no++)
1036     {
1037         int ord = -1;
1038         int regex_range = 0;
1039         int max_pos, prefix_len = 0;
1040         int relation_error;
1041         char ord_buf[32];
1042         int ord_len, i;
1043
1044         termp = *term_sub; /* start of term for each database */
1045
1046         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1047         {
1048             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1049                            basenames[base_no]);
1050             return ZEBRA_FAIL;
1051         }
1052
1053         if (zebra_apt_get_ord(zh, zapt, reg_type, xpath_use,
1054                               attributeSet, &ord) != ZEBRA_OK)
1055             continue;
1056
1057         bases_ok++;
1058
1059         *ol = ord_list_append(stream, *ol, ord);
1060         ord_len = key_SU_encode (ord, ord_buf);
1061
1062         term_dict[prefix_len++] = '(';
1063         for (i = 0; i<ord_len; i++)
1064         {
1065             term_dict[prefix_len++] = 1;  /* our internal regexp escape char */
1066             term_dict[prefix_len++] = ord_buf[i];
1067         }
1068         term_dict[prefix_len++] = ')';
1069         term_dict[prefix_len] = '\0';
1070         j = prefix_len;
1071         switch (truncation_value)
1072         {
1073         case -1:         /* not specified */
1074         case 100:        /* do not truncate */
1075             if (!string_relation(zh, zapt, &termp, term_dict,
1076                                  attributeSet,
1077                                  reg_type, space_split, term_dst,
1078                                  &relation_error))
1079             {
1080                 if (relation_error)
1081                 {
1082                     zebra_setError(zh, relation_error, 0);
1083                     return ZEBRA_FAIL;
1084                 }
1085                 *term_sub = 0;
1086                 return ZEBRA_OK;
1087             }
1088             break;
1089         case 1:          /* right truncation */
1090             term_dict[j++] = '(';
1091             if (!term_100(zh->reg->zebra_maps, reg_type,
1092                           &termp, term_dict + j, space_split, term_dst))
1093             {
1094                 *term_sub = 0;
1095                 return ZEBRA_OK;
1096             }
1097             strcat(term_dict, ".*)");
1098             break;
1099         case 2:          /* keft truncation */
1100             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1101             if (!term_100(zh->reg->zebra_maps, reg_type,
1102                           &termp, term_dict + j, space_split, term_dst))
1103             {
1104                 *term_sub = 0;
1105                 return ZEBRA_OK;
1106             }
1107             strcat(term_dict, ")");
1108             break;
1109         case 3:          /* left&right truncation */
1110             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1111             if (!term_100(zh->reg->zebra_maps, reg_type,
1112                           &termp, term_dict + j, space_split, term_dst))
1113             {
1114                 *term_sub = 0;
1115                 return ZEBRA_OK;
1116             }
1117             strcat(term_dict, ".*)");
1118             break;
1119         case 101:        /* process # in term */
1120             term_dict[j++] = '(';
1121             if (!term_101(zh->reg->zebra_maps, reg_type,
1122                           &termp, term_dict + j, space_split, term_dst))
1123             {
1124                 *term_sub = 0;
1125                 return ZEBRA_OK;
1126             }
1127             strcat(term_dict, ")");
1128             break;
1129         case 102:        /* Regexp-1 */
1130             term_dict[j++] = '(';
1131             if (!term_102(zh->reg->zebra_maps, reg_type,
1132                           &termp, term_dict + j, space_split, term_dst))
1133             {
1134                 *term_sub = 0;
1135                 return ZEBRA_OK;
1136             }
1137             strcat(term_dict, ")");
1138             break;
1139         case 103:       /* Regexp-2 */
1140             regex_range = 1;
1141             term_dict[j++] = '(';
1142             if (!term_103(zh->reg->zebra_maps, reg_type,
1143                           &termp, term_dict + j, &regex_range,
1144                           space_split, term_dst))
1145             {
1146                 *term_sub = 0;
1147                 return ZEBRA_OK;
1148             }
1149             strcat(term_dict, ")");
1150             break;
1151         case 104:        /* process # and ! in term */
1152             term_dict[j++] = '(';
1153             if (!term_104(zh->reg->zebra_maps, reg_type,
1154                           &termp, term_dict + j, space_split, term_dst))
1155             {
1156                 *term_sub = 0;
1157                 return ZEBRA_OK;
1158             }
1159             strcat(term_dict, ")");
1160             break;
1161         case 105:        /* process * and ! in term */
1162             term_dict[j++] = '(';
1163             if (!term_105(zh->reg->zebra_maps, reg_type,
1164                           &termp, term_dict + j, space_split, term_dst, 1))
1165             {
1166                 *term_sub = 0;
1167                 return ZEBRA_OK;
1168             }
1169             strcat(term_dict, ")");
1170             break;
1171         case 106:        /* process * and ! in term */
1172             term_dict[j++] = '(';
1173             if (!term_105(zh->reg->zebra_maps, reg_type,
1174                           &termp, term_dict + j, space_split, term_dst, 0))
1175             {
1176                 *term_sub = 0;
1177                 return ZEBRA_OK;
1178             }
1179             strcat(term_dict, ")");
1180             break;
1181         default:
1182             zebra_setError_zint(zh,
1183                                 YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE,
1184                                 truncation_value);
1185             return ZEBRA_FAIL;
1186         }
1187         if (1)
1188         {
1189             char buf[80];
1190             const char *input = term_dict + prefix_len;
1191             esc_str(buf, sizeof(buf), input, strlen(input));
1192         }
1193         yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_dict+prefix_len);
1194         r = dict_lookup_grep(zh->reg->dict, term_dict, regex_range,
1195                              grep_info, &max_pos,
1196                              ord_len /* number of "exact" chars */,
1197                              grep_handle);
1198         if (r)
1199             yaz_log(YLOG_WARN, "dict_lookup_grep fail %d", r);
1200     }
1201     if (!bases_ok)
1202         return ZEBRA_FAIL;
1203     *term_sub = termp;
1204     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1205     return ZEBRA_OK;
1206 }
1207
1208
1209 /* convert APT search term to UTF8 */
1210 static ZEBRA_RES zapt_term_to_utf8(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1211                                    char *termz)
1212 {
1213     size_t sizez;
1214     Z_Term *term = zapt->term;
1215
1216     switch (term->which)
1217     {
1218     case Z_Term_general:
1219         if (zh->iconv_to_utf8 != 0)
1220         {
1221             char *inbuf = (char *) term->u.general->buf;
1222             size_t inleft = term->u.general->len;
1223             char *outbuf = termz;
1224             size_t outleft = IT_MAX_WORD-1;
1225             size_t ret;
1226
1227             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1228                         &outbuf, &outleft);
1229             if (ret == (size_t)(-1))
1230             {
1231                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1232                 zebra_setError(
1233                     zh,
1234                     YAZ_BIB1_QUERY_TERM_INCLUDES_CHARS_THAT_DO_NOT_TRANSLATE_INTO_,
1235                     0);
1236                 return ZEBRA_FAIL;
1237             }
1238             *outbuf = 0;
1239         }
1240         else
1241         {
1242             sizez = term->u.general->len;
1243             if (sizez > IT_MAX_WORD-1)
1244                 sizez = IT_MAX_WORD-1;
1245             memcpy (termz, term->u.general->buf, sizez);
1246             termz[sizez] = '\0';
1247         }
1248         break;
1249     case Z_Term_characterString:
1250         sizez = strlen(term->u.characterString);
1251         if (sizez > IT_MAX_WORD-1)
1252             sizez = IT_MAX_WORD-1;
1253         memcpy (termz, term->u.characterString, sizez);
1254         termz[sizez] = '\0';
1255         break;
1256     default:
1257         zebra_setError(zh, YAZ_BIB1_UNSUPP_CODED_VALUE_FOR_TERM, 0);
1258         return ZEBRA_FAIL;
1259     }
1260     return ZEBRA_OK;
1261 }
1262
1263 /* convert APT SCAN term to internal cmap */
1264 static ZEBRA_RES trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1265                                  char *termz, int reg_type)
1266 {
1267     char termz0[IT_MAX_WORD];
1268
1269     if (zapt_term_to_utf8(zh, zapt, termz0) == ZEBRA_FAIL)
1270         return ZEBRA_FAIL;    /* error */
1271     else
1272     {
1273         const char **map;
1274         const char *cp = (const char *) termz0;
1275         const char *cp_end = cp + strlen(cp);
1276         const char *src;
1277         int i = 0;
1278         const char *space_map = NULL;
1279         int len;
1280
1281         while ((len = (cp_end - cp)) > 0)
1282         {
1283             map = zebra_maps_input(zh->reg->zebra_maps, reg_type, &cp, len, 0);
1284             if (**map == *CHR_SPACE)
1285                 space_map = *map;
1286             else
1287             {
1288                 if (i && space_map)
1289                     for (src = space_map; *src; src++)
1290                         termz[i++] = *src;
1291                 space_map = NULL;
1292                 for (src = *map; *src; src++)
1293                     termz[i++] = *src;
1294             }
1295         }
1296         termz[i] = '\0';
1297     }
1298     return ZEBRA_OK;
1299 }
1300
1301 static void grep_info_delete(struct grep_info *grep_info)
1302 {
1303 #ifdef TERM_COUNT
1304     xfree(grep_info->term_no);
1305 #endif
1306     xfree(grep_info->isam_p_buf);
1307 }
1308
1309 static ZEBRA_RES grep_info_prepare(ZebraHandle zh,
1310                                    Z_AttributesPlusTerm *zapt,
1311                                    struct grep_info *grep_info,
1312                                    int reg_type)
1313 {
1314     AttrType termset;
1315     int termset_value_numeric;
1316     const char *termset_value_string;
1317
1318 #ifdef TERM_COUNT
1319     grep_info->term_no = 0;
1320 #endif
1321     grep_info->isam_p_size = 0;
1322     grep_info->isam_p_buf = NULL;
1323     grep_info->zh = zh;
1324     grep_info->reg_type = reg_type;
1325     grep_info->termset = 0;
1326     if (!zapt)
1327         return ZEBRA_OK;
1328     attr_init_APT(&termset, zapt, 8);
1329     termset_value_numeric =
1330         attr_find_ex(&termset, NULL, &termset_value_string);
1331     if (termset_value_numeric != -1)
1332     {
1333 #if TERMSET_DISABLE
1334         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, "termset");
1335         return ZEBRA_FAIL;
1336 #else
1337         char resname[32];
1338         const char *termset_name = 0;
1339         if (termset_value_numeric != -2)
1340         {
1341
1342             sprintf(resname, "%d", termset_value_numeric);
1343             termset_name = resname;
1344         }
1345         else
1346             termset_name = termset_value_string;
1347         yaz_log(log_level_rpn, "creating termset set %s", termset_name);
1348         grep_info->termset = resultSetAdd(zh, termset_name, 1);
1349         if (!grep_info->termset)
1350         {
1351             zebra_setError(zh, YAZ_BIB1_ILLEGAL_RESULT_SET_NAME, termset_name);
1352             return ZEBRA_FAIL;
1353         }
1354 #endif
1355     }
1356     return ZEBRA_OK;
1357 }
1358
1359 /**
1360   \brief Create result set(s) for list of terms
1361   \param zh Zebra Handle
1362   \param termz term as used in query but converted to UTF-8
1363   \param attributeSet default attribute set
1364   \param stream memory for result
1365   \param reg_type register type ('w', 'p',..)
1366   \param complete_flag whether it's phrases or not
1367   \param rank_type term flags for ranking
1368   \param xpath_use use attribute for X-Path (-1 for no X-path)
1369   \param num_bases number of databases
1370   \param basenames array of databases
1371   \param rset_mem memory for result sets
1372   \param result_sets output result set for each term in list (output)
1373   \param number number of output result sets
1374   \param kc rset key control to be used for created result sets
1375 */
1376 static ZEBRA_RES term_list_trunc(ZebraHandle zh,
1377                                  Z_AttributesPlusTerm *zapt,
1378                                  const char *termz,
1379                                  oid_value attributeSet,
1380                                  NMEM stream,
1381                                  int reg_type, int complete_flag,
1382                                  const char *rank_type,
1383                                  const char *xpath_use,
1384                                  int num_bases, char **basenames,
1385                                  NMEM rset_nmem,
1386                                  RSET **result_sets, int *num_result_sets,
1387                                  struct rset_key_control *kc)
1388 {
1389     char term_dst[IT_MAX_WORD+1];
1390     struct grep_info grep_info;
1391     const char *termp = termz;
1392     int alloc_sets = 0;
1393
1394     *num_result_sets = 0;
1395     *term_dst = 0;
1396     if (grep_info_prepare(zh, zapt, &grep_info, reg_type) == ZEBRA_FAIL)
1397         return ZEBRA_FAIL;
1398     while(1)
1399     {
1400         ZEBRA_RES res;
1401
1402         if (alloc_sets == *num_result_sets)
1403         {
1404             int add = 10;
1405             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1406                                               sizeof(*rnew));
1407             if (alloc_sets)
1408                 memcpy(rnew, *result_sets, alloc_sets * sizeof(*rnew));
1409             alloc_sets = alloc_sets + add;
1410             *result_sets = rnew;
1411         }
1412         res = term_trunc(zh, zapt, &termp, attributeSet,
1413                          stream, &grep_info,
1414                          reg_type, complete_flag,
1415                          num_bases, basenames,
1416                          term_dst, rank_type,
1417                          xpath_use, rset_nmem,
1418                          &(*result_sets)[*num_result_sets],
1419                          kc);
1420         if (res != ZEBRA_OK)
1421         {
1422             int i;
1423             for (i = 0; i < *num_result_sets; i++)
1424                 rset_delete((*result_sets)[i]);
1425             grep_info_delete (&grep_info);
1426             return res;
1427         }
1428         if ((*result_sets)[*num_result_sets] == 0)
1429             break;
1430         (*num_result_sets)++;
1431
1432         if (!*termp)
1433             break;
1434     }
1435     grep_info_delete(&grep_info);
1436     return ZEBRA_OK;
1437 }
1438
1439 static ZEBRA_RES rpn_search_APT_phrase(ZebraHandle zh,
1440                                        Z_AttributesPlusTerm *zapt,
1441                                        const char *termz_org,
1442                                        oid_value attributeSet,
1443                                        NMEM stream,
1444                                        int reg_type, int complete_flag,
1445                                        const char *rank_type,
1446                                        const char *xpath_use,
1447                                        int num_bases, char **basenames,
1448                                        NMEM rset_nmem,
1449                                        RSET *rset,
1450                                        struct rset_key_control *kc)
1451 {
1452     RSET *result_sets = 0;
1453     int num_result_sets = 0;
1454     ZEBRA_RES res =
1455         term_list_trunc(zh, zapt, termz_org, attributeSet,
1456                         stream, reg_type, complete_flag,
1457                         rank_type, xpath_use,
1458                         num_bases, basenames,
1459                         rset_nmem,
1460                         &result_sets, &num_result_sets, kc);
1461     if (res != ZEBRA_OK)
1462         return res;
1463     if (num_result_sets == 0)
1464         *rset = rset_create_null(rset_nmem, kc, 0);
1465     else if (num_result_sets == 1)
1466         *rset = result_sets[0];
1467     else
1468         *rset = rset_create_prox(rset_nmem, kc, kc->scope,
1469                                  num_result_sets, result_sets,
1470                                  1 /* ordered */, 0 /* exclusion */,
1471                                  3 /* relation */, 1 /* distance */);
1472     if (!*rset)
1473         return ZEBRA_FAIL;
1474     return ZEBRA_OK;
1475 }
1476
1477 static ZEBRA_RES rpn_search_APT_or_list(ZebraHandle zh,
1478                                         Z_AttributesPlusTerm *zapt,
1479                                         const char *termz_org,
1480                                         oid_value attributeSet,
1481                                         NMEM stream,
1482                                         int reg_type, int complete_flag,
1483                                         const char *rank_type,
1484                                         const char *xpath_use,
1485                                         int num_bases, char **basenames,
1486                                         NMEM rset_nmem,
1487                                         RSET *rset,
1488                                         struct rset_key_control *kc)
1489 {
1490     RSET *result_sets = 0;
1491     int num_result_sets = 0;
1492     ZEBRA_RES res =
1493         term_list_trunc(zh, zapt, termz_org, attributeSet,
1494                         stream, reg_type, complete_flag,
1495                         rank_type, xpath_use,
1496                         num_bases, basenames,
1497                         rset_nmem,
1498                         &result_sets, &num_result_sets, kc);
1499     if (res != ZEBRA_OK)
1500         return res;
1501     if (num_result_sets == 0)
1502         *rset = rset_create_null(rset_nmem, kc, 0);
1503     else if (num_result_sets == 1)
1504         *rset = result_sets[0];
1505     else
1506         *rset = rset_create_or(rset_nmem, kc, kc->scope, 0 /* termid */,
1507                                num_result_sets, result_sets);
1508     if (!*rset)
1509         return ZEBRA_FAIL;
1510     return ZEBRA_OK;
1511 }
1512
1513 static ZEBRA_RES rpn_search_APT_and_list(ZebraHandle zh,
1514                                          Z_AttributesPlusTerm *zapt,
1515                                          const char *termz_org,
1516                                          oid_value attributeSet,
1517                                          NMEM stream,
1518                                          int reg_type, int complete_flag,
1519                                          const char *rank_type,
1520                                          const char *xpath_use,
1521                                          int num_bases, char **basenames,
1522                                          NMEM rset_nmem,
1523                                          RSET *rset,
1524                                          struct rset_key_control *kc)
1525 {
1526     RSET *result_sets = 0;
1527     int num_result_sets = 0;
1528     ZEBRA_RES res =
1529         term_list_trunc(zh, zapt, termz_org, attributeSet,
1530                         stream, reg_type, complete_flag,
1531                         rank_type, xpath_use,
1532                         num_bases, basenames,
1533                         rset_nmem,
1534                         &result_sets, &num_result_sets,
1535                         kc);
1536     if (res != ZEBRA_OK)
1537         return res;
1538     if (num_result_sets == 0)
1539         *rset = rset_create_null(rset_nmem, kc, 0);
1540     else if (num_result_sets == 1)
1541         *rset = result_sets[0];
1542     else
1543         *rset = rset_create_and(rset_nmem, kc, kc->scope,
1544                                 num_result_sets, result_sets);
1545     if (!*rset)
1546         return ZEBRA_FAIL;
1547     return ZEBRA_OK;
1548 }
1549
1550 static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1551                             const char **term_sub,
1552                             char *term_dict,
1553                             oid_value attributeSet,
1554                             struct grep_info *grep_info,
1555                             int *max_pos,
1556                             int reg_type,
1557                             char *term_dst,
1558                             int *error_code)
1559 {
1560     AttrType relation;
1561     int relation_value;
1562     int term_value;
1563     int r;
1564     char *term_tmp = term_dict + strlen(term_dict);
1565
1566     *error_code = 0;
1567     attr_init_APT(&relation, zapt, 2);
1568     relation_value = attr_find(&relation, NULL);
1569
1570     yaz_log(log_level_rpn, "numeric relation value=%d", relation_value);
1571
1572     switch (relation_value)
1573     {
1574     case 1:
1575         yaz_log(log_level_rpn, "Relation <");
1576         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1577                       term_dst))
1578             return 0;
1579         term_value = atoi (term_tmp);
1580         gen_regular_rel(term_tmp, term_value-1, 1);
1581         break;
1582     case 2:
1583         yaz_log(log_level_rpn, "Relation <=");
1584         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1585                       term_dst))
1586             return 0;
1587         term_value = atoi (term_tmp);
1588         gen_regular_rel(term_tmp, term_value, 1);
1589         break;
1590     case 4:
1591         yaz_log(log_level_rpn, "Relation >=");
1592         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1593                       term_dst))
1594             return 0;
1595         term_value = atoi (term_tmp);
1596         gen_regular_rel(term_tmp, term_value, 0);
1597         break;
1598     case 5:
1599         yaz_log(log_level_rpn, "Relation >");
1600         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1601                       term_dst))
1602             return 0;
1603         term_value = atoi (term_tmp);
1604         gen_regular_rel(term_tmp, term_value+1, 0);
1605         break;
1606     case -1:
1607     case 3:
1608         yaz_log(log_level_rpn, "Relation =");
1609         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1610                       term_dst))
1611             return 0;
1612         term_value = atoi (term_tmp);
1613         sprintf(term_tmp, "(0*%d)", term_value);
1614         break;
1615     case 103:
1616         /* term_tmp untouched.. */
1617         while (**term_sub != '\0')
1618             (*term_sub)++;
1619         break;
1620     default:
1621         *error_code = YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
1622         return 0;
1623     }
1624     yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_tmp);
1625     r = dict_lookup_grep(zh->reg->dict, term_dict, 0, grep_info, max_pos,
1626                           0, grep_handle);
1627     if (r)
1628         yaz_log(YLOG_WARN, "dict_lookup_grep fail, rel = gt: %d", r);
1629     yaz_log(log_level_rpn, "%d positions", grep_info->isam_p_indx);
1630     return 1;
1631 }
1632
1633 static ZEBRA_RES numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1634                               const char **term_sub,
1635                               oid_value attributeSet, NMEM stream,
1636                               struct grep_info *grep_info,
1637                               int reg_type, int complete_flag,
1638                               int num_bases, char **basenames,
1639                               char *term_dst,
1640                               const char *xpath_use,
1641                               struct ord_list **ol)
1642 {
1643     char term_dict[2*IT_MAX_WORD+2];
1644     int base_no;
1645     const char *termp;
1646     struct rpn_char_map_info rcmi;
1647
1648     int bases_ok = 0;     /* no of databases with OK attribute */
1649
1650     *ol = ord_list_create(stream);
1651
1652     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1653
1654     for (base_no = 0; base_no < num_bases; base_no++)
1655     {
1656         int max_pos, prefix_len = 0;
1657         int relation_error = 0;
1658         int ord, ord_len, i;
1659         char ord_buf[32];
1660
1661         termp = *term_sub;
1662
1663         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1664         {
1665             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1666                            basenames[base_no]);
1667             return ZEBRA_FAIL;
1668         }
1669
1670         if (zebra_apt_get_ord(zh, zapt, reg_type, xpath_use,
1671                               attributeSet, &ord) != ZEBRA_OK)
1672             continue;
1673         bases_ok++;
1674
1675         *ol = ord_list_append(stream, *ol, ord);
1676
1677         ord_len = key_SU_encode (ord, ord_buf);
1678
1679         term_dict[prefix_len++] = '(';
1680         for (i = 0; i < ord_len; i++)
1681         {
1682             term_dict[prefix_len++] = 1;
1683             term_dict[prefix_len++] = ord_buf[i];
1684         }
1685         term_dict[prefix_len++] = ')';
1686         term_dict[prefix_len] = '\0';
1687
1688         if (!numeric_relation(zh, zapt, &termp, term_dict,
1689                               attributeSet, grep_info, &max_pos, reg_type,
1690                               term_dst, &relation_error))
1691         {
1692             if (relation_error)
1693             {
1694                 zebra_setError(zh, relation_error, 0);
1695                 return ZEBRA_FAIL;
1696             }
1697             *term_sub = 0;
1698             return ZEBRA_OK;
1699         }
1700     }
1701     if (!bases_ok)
1702         return ZEBRA_FAIL;
1703     *term_sub = termp;
1704     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1705     return ZEBRA_OK;
1706 }
1707
1708
1709 static ZEBRA_RES rpn_search_APT_numeric(ZebraHandle zh,
1710                                         Z_AttributesPlusTerm *zapt,
1711                                         const char *termz,
1712                                         oid_value attributeSet,
1713                                         NMEM stream,
1714                                         int reg_type, int complete_flag,
1715                                         const char *rank_type,
1716                                         const char *xpath_use,
1717                                         int num_bases, char **basenames,
1718                                         NMEM rset_nmem,
1719                                         RSET *rset,
1720                                         struct rset_key_control *kc)
1721 {
1722     char term_dst[IT_MAX_WORD+1];
1723     const char *termp = termz;
1724     RSET *result_sets = 0;
1725     int num_result_sets = 0;
1726     ZEBRA_RES res;
1727     struct grep_info grep_info;
1728     int alloc_sets = 0;
1729     zint hits_limit_value;
1730     const char *term_ref_id_str = 0;
1731
1732     term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str, stream);
1733
1734     yaz_log(log_level_rpn, "APT_numeric t='%s'", termz);
1735     if (grep_info_prepare(zh, zapt, &grep_info, reg_type) == ZEBRA_FAIL)
1736         return ZEBRA_FAIL;
1737     while (1)
1738     {
1739         struct ord_list *ol;
1740         if (alloc_sets == num_result_sets)
1741         {
1742             int add = 10;
1743             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1744                                               sizeof(*rnew));
1745             if (alloc_sets)
1746                 memcpy(rnew, result_sets, alloc_sets * sizeof(*rnew));
1747             alloc_sets = alloc_sets + add;
1748             result_sets = rnew;
1749         }
1750         yaz_log(YLOG_DEBUG, "APT_numeric termp=%s", termp);
1751         grep_info.isam_p_indx = 0;
1752         res = numeric_term(zh, zapt, &termp, attributeSet, stream, &grep_info,
1753                            reg_type, complete_flag, num_bases, basenames,
1754                            term_dst, xpath_use, &ol);
1755         if (res == ZEBRA_FAIL || termp == 0)
1756             break;
1757         yaz_log(YLOG_DEBUG, "term: %s", term_dst);
1758         result_sets[num_result_sets] =
1759             rset_trunc(zh, grep_info.isam_p_buf,
1760                        grep_info.isam_p_indx, term_dst,
1761                        strlen(term_dst), rank_type,
1762                        0 /* preserve position */,
1763                        zapt->term->which, rset_nmem,
1764                        kc, kc->scope, ol, reg_type,
1765                        hits_limit_value,
1766                        term_ref_id_str);
1767         if (!result_sets[num_result_sets])
1768             break;
1769         num_result_sets++;
1770         if (!*termp)
1771             break;
1772     }
1773     grep_info_delete(&grep_info);
1774
1775     if (res != ZEBRA_OK)
1776         return res;
1777     if (num_result_sets == 0)
1778         *rset = rset_create_null(rset_nmem, kc, 0);
1779     else if (num_result_sets == 1)
1780         *rset = result_sets[0];
1781     else
1782         *rset = rset_create_and(rset_nmem, kc, kc->scope,
1783                                 num_result_sets, result_sets);
1784     if (!*rset)
1785         return ZEBRA_FAIL;
1786     return ZEBRA_OK;
1787 }
1788
1789 static ZEBRA_RES rpn_search_APT_local(ZebraHandle zh,
1790                                       Z_AttributesPlusTerm *zapt,
1791                                       const char *termz,
1792                                       oid_value attributeSet,
1793                                       NMEM stream,
1794                                       const char *rank_type, NMEM rset_nmem,
1795                                       RSET *rset,
1796                                       struct rset_key_control *kc)
1797 {
1798     RSFD rsfd;
1799     struct it_key key;
1800     int sys;
1801     *rset = rset_create_temp(rset_nmem, kc, kc->scope,
1802                              res_get (zh->res, "setTmpDir"),0 );
1803     rsfd = rset_open(*rset, RSETF_WRITE);
1804
1805     sys = atoi(termz);
1806     if (sys <= 0)
1807         sys = 1;
1808     key.mem[0] = sys;
1809     key.mem[1] = 1;
1810     key.len = 2;
1811     rset_write (rsfd, &key);
1812     rset_close (rsfd);
1813     return ZEBRA_OK;
1814 }
1815
1816 static ZEBRA_RES rpn_sort_spec(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1817                                oid_value attributeSet, NMEM stream,
1818                                Z_SortKeySpecList *sort_sequence,
1819                                const char *rank_type,
1820                                NMEM rset_nmem,
1821                                RSET *rset,
1822                                struct rset_key_control *kc)
1823 {
1824     int i;
1825     int sort_relation_value;
1826     AttrType sort_relation_type;
1827     Z_SortKeySpec *sks;
1828     Z_SortKey *sk;
1829     int oid[OID_SIZE];
1830     oident oe;
1831     char termz[20];
1832
1833     attr_init_APT(&sort_relation_type, zapt, 7);
1834     sort_relation_value = attr_find(&sort_relation_type, &attributeSet);
1835
1836     if (!sort_sequence->specs)
1837     {
1838         sort_sequence->num_specs = 10;
1839         sort_sequence->specs = (Z_SortKeySpec **)
1840             nmem_malloc(stream, sort_sequence->num_specs *
1841                          sizeof(*sort_sequence->specs));
1842         for (i = 0; i<sort_sequence->num_specs; i++)
1843             sort_sequence->specs[i] = 0;
1844     }
1845     if (zapt->term->which != Z_Term_general)
1846         i = 0;
1847     else
1848         i = atoi_n ((char *) zapt->term->u.general->buf,
1849                     zapt->term->u.general->len);
1850     if (i >= sort_sequence->num_specs)
1851         i = 0;
1852     sprintf(termz, "%d", i);
1853
1854     oe.proto = PROTO_Z3950;
1855     oe.oclass = CLASS_ATTSET;
1856     oe.value = attributeSet;
1857     if (!oid_ent_to_oid (&oe, oid))
1858         return ZEBRA_FAIL;
1859
1860     sks = (Z_SortKeySpec *) nmem_malloc(stream, sizeof(*sks));
1861     sks->sortElement = (Z_SortElement *)
1862         nmem_malloc(stream, sizeof(*sks->sortElement));
1863     sks->sortElement->which = Z_SortElement_generic;
1864     sk = sks->sortElement->u.generic = (Z_SortKey *)
1865         nmem_malloc(stream, sizeof(*sk));
1866     sk->which = Z_SortKey_sortAttributes;
1867     sk->u.sortAttributes = (Z_SortAttributes *)
1868         nmem_malloc(stream, sizeof(*sk->u.sortAttributes));
1869
1870     sk->u.sortAttributes->id = oid;
1871     sk->u.sortAttributes->list = zapt->attributes;
1872
1873     sks->sortRelation = (int *)
1874         nmem_malloc(stream, sizeof(*sks->sortRelation));
1875     if (sort_relation_value == 1)
1876         *sks->sortRelation = Z_SortKeySpec_ascending;
1877     else if (sort_relation_value == 2)
1878         *sks->sortRelation = Z_SortKeySpec_descending;
1879     else
1880         *sks->sortRelation = Z_SortKeySpec_ascending;
1881
1882     sks->caseSensitivity = (int *)
1883         nmem_malloc(stream, sizeof(*sks->caseSensitivity));
1884     *sks->caseSensitivity = 0;
1885
1886     sks->which = Z_SortKeySpec_null;
1887     sks->u.null = odr_nullval ();
1888     sort_sequence->specs[i] = sks;
1889     *rset = rset_create_null(rset_nmem, kc, 0);
1890     return ZEBRA_OK;
1891 }
1892
1893
1894 static int rpn_check_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1895                            oid_value attributeSet,
1896                            struct xpath_location_step *xpath, int max,
1897                            NMEM mem)
1898 {
1899     oid_value curAttributeSet = attributeSet;
1900     AttrType use;
1901     const char *use_string = 0;
1902
1903     attr_init_APT(&use, zapt, 1);
1904     attr_find_ex(&use, &curAttributeSet, &use_string);
1905
1906     if (!use_string || *use_string != '/')
1907         return -1;
1908
1909     return zebra_parse_xpath_str(use_string, xpath, max, mem);
1910 }
1911
1912
1913
1914 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
1915                         int reg_type, const char *term,
1916                         const char *xpath_use,
1917                         NMEM rset_nmem,
1918                         struct rset_key_control *kc)
1919 {
1920     RSET rset;
1921     struct grep_info grep_info;
1922     char term_dict[2048];
1923     char ord_buf[32];
1924     int prefix_len = 0;
1925     int ord = zebraExplain_lookup_attr_str(zh->reg->zei,
1926                                            zinfo_index_category_index,
1927                                            reg_type,
1928                                            xpath_use);
1929     int ord_len, i, r, max_pos;
1930     int term_type = Z_Term_characterString;
1931     const char *flags = "void";
1932
1933     if (grep_info_prepare(zh, 0 /* zapt */, &grep_info, '0') == ZEBRA_FAIL)
1934         return rset_create_null(rset_nmem, kc, 0);
1935
1936     if (ord < 0)
1937         return rset_create_null(rset_nmem, kc, 0);
1938     if (prefix_len)
1939         term_dict[prefix_len++] = '|';
1940     else
1941         term_dict[prefix_len++] = '(';
1942
1943     ord_len = key_SU_encode (ord, ord_buf);
1944     for (i = 0; i<ord_len; i++)
1945     {
1946         term_dict[prefix_len++] = 1;
1947         term_dict[prefix_len++] = ord_buf[i];
1948     }
1949     term_dict[prefix_len++] = ')';
1950     strcpy(term_dict+prefix_len, term);
1951
1952     grep_info.isam_p_indx = 0;
1953     r = dict_lookup_grep(zh->reg->dict, term_dict, 0,
1954                           &grep_info, &max_pos, 0, grep_handle);
1955     yaz_log(YLOG_DEBUG, "%s %d positions", term,
1956              grep_info.isam_p_indx);
1957     rset = rset_trunc(zh, grep_info.isam_p_buf,
1958                       grep_info.isam_p_indx, term, strlen(term),
1959                       flags, 1, term_type,rset_nmem,
1960                       kc, kc->scope, 0, reg_type, 0 /* hits_limit */,
1961                       0 /* term_ref_id_str */);
1962     grep_info_delete(&grep_info);
1963     return rset;
1964 }
1965
1966 static
1967 ZEBRA_RES rpn_search_xpath(ZebraHandle zh,
1968                            int num_bases, char **basenames,
1969                            NMEM stream, const char *rank_type, RSET rset,
1970                            int xpath_len, struct xpath_location_step *xpath,
1971                            NMEM rset_nmem,
1972                            RSET *rset_out,
1973                            struct rset_key_control *kc)
1974 {
1975     int base_no;
1976     int i;
1977     int always_matches = rset ? 0 : 1;
1978
1979     if (xpath_len < 0)
1980     {
1981         *rset_out = rset;
1982         return ZEBRA_OK;
1983     }
1984
1985     yaz_log(YLOG_DEBUG, "xpath len=%d", xpath_len);
1986     for (i = 0; i<xpath_len; i++)
1987     {
1988         yaz_log(log_level_rpn, "XPATH %d %s", i, xpath[i].part);
1989
1990     }
1991
1992     /*
1993       //a    ->    a/.*
1994       //a/b  ->    b/a/.*
1995       /a     ->    a/
1996       /a/b   ->    b/a/
1997
1998       /      ->    none
1999
2000    a[@attr = value]/b[@other = othervalue]
2001
2002  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
2003  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
2004  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
2005  /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y)
2006  /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y)
2007  /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x)
2008
2009     */
2010
2011     dict_grep_cmap (zh->reg->dict, 0, 0);
2012
2013     for (base_no = 0; base_no < num_bases; base_no++)
2014     {
2015         int level = xpath_len;
2016         int first_path = 1;
2017
2018         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2019         {
2020             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
2021                            basenames[base_no]);
2022             *rset_out = rset;
2023             return ZEBRA_FAIL;
2024         }
2025         while (--level >= 0)
2026         {
2027             WRBUF xpath_rev = wrbuf_alloc();
2028             int i;
2029             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
2030
2031             for (i = level; i >= 1; --i)
2032             {
2033                 const char *cp = xpath[i].part;
2034                 if (*cp)
2035                 {
2036                     for (; *cp; cp++)
2037                     {
2038                         if (*cp == '*')
2039                             wrbuf_puts(xpath_rev, "[^/]*");
2040                         else if (*cp == ' ')
2041                             wrbuf_puts(xpath_rev, "\001 ");
2042                         else
2043                             wrbuf_putc(xpath_rev, *cp);
2044
2045                         /* wrbuf_putc does not null-terminate , but
2046                            wrbuf_puts below ensures it does.. so xpath_rev
2047                            is OK iff length is > 0 */
2048                     }
2049                     wrbuf_puts(xpath_rev, "/");
2050                 }
2051                 else if (i == 1)  /* // case */
2052                     wrbuf_puts(xpath_rev, ".*");
2053             }
2054             if (xpath[level].predicate &&
2055                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
2056                 xpath[level].predicate->u.relation.name[0])
2057             {
2058                 WRBUF wbuf = wrbuf_alloc();
2059                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
2060                 if (xpath[level].predicate->u.relation.value)
2061                 {
2062                     const char *cp = xpath[level].predicate->u.relation.value;
2063                     wrbuf_putc(wbuf, '=');
2064
2065                     while (*cp)
2066                     {
2067                         if (strchr(REGEX_CHARS, *cp))
2068                             wrbuf_putc(wbuf, '\\');
2069                         wrbuf_putc(wbuf, *cp);
2070                         cp++;
2071                     }
2072                 }
2073                 wrbuf_puts(wbuf, "");
2074                 rset_attr = xpath_trunc(
2075                     zh, stream, '0', wrbuf_buf(wbuf), ZEBRA_XPATH_ATTR_NAME,
2076                     rset_nmem, kc);
2077                 wrbuf_free(wbuf, 1);
2078             }
2079             else
2080             {
2081                 if (!first_path)
2082                 {
2083                     wrbuf_free(xpath_rev, 1);
2084                     continue;
2085                 }
2086             }
2087             yaz_log(log_level_rpn, "xpath_rev (%d) = %.*s", level,
2088                     wrbuf_len(xpath_rev), wrbuf_buf(xpath_rev));
2089             if (wrbuf_len(xpath_rev))
2090             {
2091                 rset_start_tag = xpath_trunc(zh, stream, '0',
2092                                              wrbuf_buf(xpath_rev),
2093                                              ZEBRA_XPATH_ELM_BEGIN,
2094                                              rset_nmem, kc);
2095                 if (always_matches)
2096                     rset = rset_start_tag;
2097                 else
2098                 {
2099                     rset_end_tag = xpath_trunc(zh, stream, '0',
2100                                                wrbuf_buf(xpath_rev),
2101                                                ZEBRA_XPATH_ELM_END,
2102                                                rset_nmem, kc);
2103
2104                     rset = rset_create_between(rset_nmem, kc, kc->scope,
2105                                                rset_start_tag, rset,
2106                                                rset_end_tag, rset_attr);
2107                 }
2108             }
2109             wrbuf_free(xpath_rev, 1);
2110             first_path = 0;
2111         }
2112     }
2113     *rset_out = rset;
2114     return ZEBRA_OK;
2115 }
2116
2117 #define MAX_XPATH_STEPS 10
2118
2119 static ZEBRA_RES rpn_search_APT(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2120                                 oid_value attributeSet, NMEM stream,
2121                                 Z_SortKeySpecList *sort_sequence,
2122                                 int num_bases, char **basenames,
2123                                 NMEM rset_nmem,
2124                                 RSET *rset,
2125                                 struct rset_key_control *kc)
2126 {
2127     ZEBRA_RES res = ZEBRA_OK;
2128     unsigned reg_id;
2129     char *search_type = NULL;
2130     char rank_type[128];
2131     int complete_flag;
2132     int sort_flag;
2133     char termz[IT_MAX_WORD+1];
2134     int xpath_len;
2135     const char *xpath_use = 0;
2136     struct xpath_location_step xpath[MAX_XPATH_STEPS];
2137
2138     if (!log_level_set)
2139     {
2140         log_level_rpn = yaz_log_module_level("rpn");
2141         log_level_set = 1;
2142     }
2143     zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2144                     rank_type, &complete_flag, &sort_flag);
2145
2146     yaz_log(YLOG_DEBUG, "reg_id=%c", reg_id);
2147     yaz_log(YLOG_DEBUG, "complete_flag=%d", complete_flag);
2148     yaz_log(YLOG_DEBUG, "search_type=%s", search_type);
2149     yaz_log(YLOG_DEBUG, "rank_type=%s", rank_type);
2150
2151     if (zapt_term_to_utf8(zh, zapt, termz) == ZEBRA_FAIL)
2152         return ZEBRA_FAIL;
2153
2154     if (sort_flag)
2155         return rpn_sort_spec(zh, zapt, attributeSet, stream, sort_sequence,
2156                              rank_type, rset_nmem, rset, kc);
2157     /* consider if an X-Path query is used */
2158     xpath_len = rpn_check_xpath(zh, zapt, attributeSet,
2159                                 xpath, MAX_XPATH_STEPS, stream);
2160     if (xpath_len >= 0)
2161     {
2162         if (xpath[xpath_len-1].part[0] == '@')
2163             xpath_use = ZEBRA_XPATH_ATTR_CDATA;  /* last step is attribute  */
2164         else
2165             xpath_use = ZEBRA_XPATH_CDATA;  /* searching for cdata */
2166
2167         if (1)
2168         {
2169             AttrType relation;
2170             int relation_value;
2171
2172             attr_init_APT(&relation, zapt, 2);
2173             relation_value = attr_find(&relation, NULL);
2174
2175             if (relation_value == 103) /* alwaysmatches */
2176             {
2177                 *rset = 0; /* signal no "term" set */
2178                 return rpn_search_xpath(zh, num_bases, basenames,
2179                                         stream, rank_type, *rset,
2180                                         xpath_len, xpath, rset_nmem, rset, kc);
2181             }
2182         }
2183     }
2184
2185     /* search using one of the various search type strategies
2186        termz is our UTF-8 search term
2187        attributeSet is top-level default attribute set
2188        stream is ODR for search
2189        reg_id is the register type
2190        complete_flag is 1 for complete subfield, 0 for incomplete
2191        xpath_use is use-attribute to be used for X-Path search, 0 for none
2192     */
2193     if (!strcmp(search_type, "phrase"))
2194     {
2195         res = rpn_search_APT_phrase(zh, zapt, termz, attributeSet, stream,
2196                                     reg_id, complete_flag, rank_type,
2197                                     xpath_use,
2198                                     num_bases, basenames, rset_nmem,
2199                                     rset, kc);
2200     }
2201     else if (!strcmp(search_type, "and-list"))
2202     {
2203         res = rpn_search_APT_and_list(zh, zapt, termz, attributeSet, stream,
2204                                       reg_id, complete_flag, rank_type,
2205                                       xpath_use,
2206                                       num_bases, basenames, rset_nmem,
2207                                       rset, kc);
2208     }
2209     else if (!strcmp(search_type, "or-list"))
2210     {
2211         res = rpn_search_APT_or_list(zh, zapt, termz, attributeSet, stream,
2212                                      reg_id, complete_flag, rank_type,
2213                                      xpath_use,
2214                                      num_bases, basenames, rset_nmem,
2215                                      rset, kc);
2216     }
2217     else if (!strcmp(search_type, "local"))
2218     {
2219         res = rpn_search_APT_local(zh, zapt, termz, attributeSet, stream,
2220                                    rank_type, rset_nmem, rset, kc);
2221     }
2222     else if (!strcmp(search_type, "numeric"))
2223     {
2224         res = rpn_search_APT_numeric(zh, zapt, termz, attributeSet, stream,
2225                                      reg_id, complete_flag, rank_type,
2226                                      xpath_use,
2227                                      num_bases, basenames, rset_nmem,
2228                                      rset, kc);
2229     }
2230     else
2231     {
2232         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2233         res = ZEBRA_FAIL;
2234     }
2235     if (res != ZEBRA_OK)
2236         return res;
2237     if (!*rset)
2238         return ZEBRA_FAIL;
2239     return rpn_search_xpath(zh, num_bases, basenames,
2240                             stream, rank_type, *rset,
2241                             xpath_len, xpath, rset_nmem, rset, kc);
2242 }
2243
2244 static ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2245                                       oid_value attributeSet,
2246                                       NMEM stream, NMEM rset_nmem,
2247                                       Z_SortKeySpecList *sort_sequence,
2248                                       int num_bases, char **basenames,
2249                                       RSET **result_sets, int *num_result_sets,
2250                                       Z_Operator *parent_op,
2251                                       struct rset_key_control *kc);
2252
2253 ZEBRA_RES rpn_search_top(ZebraHandle zh, Z_RPNStructure *zs,
2254                          oid_value attributeSet,
2255                          NMEM stream, NMEM rset_nmem,
2256                          Z_SortKeySpecList *sort_sequence,
2257                          int num_bases, char **basenames,
2258                          RSET *result_set)
2259 {
2260     RSET *result_sets = 0;
2261     int num_result_sets = 0;
2262     ZEBRA_RES res;
2263     struct rset_key_control *kc = zebra_key_control_create(zh);
2264
2265     res = rpn_search_structure(zh, zs, attributeSet,
2266                                stream, rset_nmem,
2267                                sort_sequence,
2268                                num_bases, basenames,
2269                                &result_sets, &num_result_sets,
2270                                0 /* no parent op */,
2271                                kc);
2272     if (res != ZEBRA_OK)
2273     {
2274         int i;
2275         for (i = 0; i<num_result_sets; i++)
2276             rset_delete(result_sets[i]);
2277         *result_set = 0;
2278     }
2279     else
2280     {
2281         assert(num_result_sets == 1);
2282         assert(result_sets);
2283         assert(*result_sets);
2284         *result_set = *result_sets;
2285     }
2286     (*kc->dec)(kc);
2287     return res;
2288 }
2289
2290 ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2291                                oid_value attributeSet,
2292                                NMEM stream, NMEM rset_nmem,
2293                                Z_SortKeySpecList *sort_sequence,
2294                                int num_bases, char **basenames,
2295                                RSET **result_sets, int *num_result_sets,
2296                                Z_Operator *parent_op,
2297                                struct rset_key_control *kc)
2298 {
2299     *num_result_sets = 0;
2300     if (zs->which == Z_RPNStructure_complex)
2301     {
2302         ZEBRA_RES res;
2303         Z_Operator *zop = zs->u.complex->roperator;
2304         RSET *result_sets_l = 0;
2305         int num_result_sets_l = 0;
2306         RSET *result_sets_r = 0;
2307         int num_result_sets_r = 0;
2308
2309         res = rpn_search_structure(zh, zs->u.complex->s1,
2310                                    attributeSet, stream, rset_nmem,
2311                                    sort_sequence,
2312                                    num_bases, basenames,
2313                                    &result_sets_l, &num_result_sets_l,
2314                                    zop, kc);
2315         if (res != ZEBRA_OK)
2316         {
2317             int i;
2318             for (i = 0; i<num_result_sets_l; i++)
2319                 rset_delete(result_sets_l[i]);
2320             return res;
2321         }
2322         res = rpn_search_structure(zh, zs->u.complex->s2,
2323                                    attributeSet, stream, rset_nmem,
2324                                    sort_sequence,
2325                                    num_bases, basenames,
2326                                    &result_sets_r, &num_result_sets_r,
2327                                    zop, kc);
2328         if (res != ZEBRA_OK)
2329         {
2330             int i;
2331             for (i = 0; i<num_result_sets_l; i++)
2332                 rset_delete(result_sets_l[i]);
2333             for (i = 0; i<num_result_sets_r; i++)
2334                 rset_delete(result_sets_r[i]);
2335             return res;
2336         }
2337
2338         /* make a new list of result for all children */
2339         *num_result_sets = num_result_sets_l + num_result_sets_r;
2340         *result_sets = nmem_malloc(stream, *num_result_sets *
2341                                    sizeof(**result_sets));
2342         memcpy(*result_sets, result_sets_l,
2343                num_result_sets_l * sizeof(**result_sets));
2344         memcpy(*result_sets + num_result_sets_l, result_sets_r,
2345                num_result_sets_r * sizeof(**result_sets));
2346
2347         if (!parent_op || parent_op->which != zop->which
2348             || (zop->which != Z_Operator_and &&
2349                 zop->which != Z_Operator_or))
2350         {
2351             /* parent node different from this one (or non-present) */
2352             /* we must combine result sets now */
2353             RSET rset;
2354             switch (zop->which)
2355             {
2356             case Z_Operator_and:
2357                 rset = rset_create_and(rset_nmem, kc,
2358                                        kc->scope,
2359                                        *num_result_sets, *result_sets);
2360                 break;
2361             case Z_Operator_or:
2362                 rset = rset_create_or(rset_nmem, kc,
2363                                       kc->scope, 0, /* termid */
2364                                       *num_result_sets, *result_sets);
2365                 break;
2366             case Z_Operator_and_not:
2367                 rset = rset_create_not(rset_nmem, kc,
2368                                        kc->scope,
2369                                        (*result_sets)[0],
2370                                        (*result_sets)[1]);
2371                 break;
2372             case Z_Operator_prox:
2373                 if (zop->u.prox->which != Z_ProximityOperator_known)
2374                 {
2375                     zebra_setError(zh,
2376                                    YAZ_BIB1_UNSUPP_PROX_UNIT_CODE,
2377                                    0);
2378                     return ZEBRA_FAIL;
2379                 }
2380                 if (*zop->u.prox->u.known != Z_ProxUnit_word)
2381                 {
2382                     zebra_setError_zint(zh,
2383                                         YAZ_BIB1_UNSUPP_PROX_UNIT_CODE,
2384                                         *zop->u.prox->u.known);
2385                     return ZEBRA_FAIL;
2386                 }
2387                 else
2388                 {
2389                     rset = rset_create_prox(rset_nmem, kc,
2390                                             kc->scope,
2391                                             *num_result_sets, *result_sets,
2392                                             *zop->u.prox->ordered,
2393                                             (!zop->u.prox->exclusion ?
2394                                              0 : *zop->u.prox->exclusion),
2395                                             *zop->u.prox->relationType,
2396                                             *zop->u.prox->distance );
2397                 }
2398                 break;
2399             default:
2400                 zebra_setError(zh, YAZ_BIB1_OPERATOR_UNSUPP, 0);
2401                 return ZEBRA_FAIL;
2402             }
2403             *num_result_sets = 1;
2404             *result_sets = nmem_malloc(stream, *num_result_sets *
2405                                        sizeof(**result_sets));
2406             (*result_sets)[0] = rset;
2407         }
2408     }
2409     else if (zs->which == Z_RPNStructure_simple)
2410     {
2411         RSET rset;
2412         ZEBRA_RES res;
2413
2414         if (zs->u.simple->which == Z_Operand_APT)
2415         {
2416             yaz_log(YLOG_DEBUG, "rpn_search_APT");
2417             res = rpn_search_APT(zh, zs->u.simple->u.attributesPlusTerm,
2418                                  attributeSet, stream, sort_sequence,
2419                                  num_bases, basenames, rset_nmem, &rset,
2420                                  kc);
2421             if (res != ZEBRA_OK)
2422                 return res;
2423         }
2424         else if (zs->u.simple->which == Z_Operand_resultSetId)
2425         {
2426             yaz_log(YLOG_DEBUG, "rpn_search_ref");
2427             rset = resultSetRef(zh, zs->u.simple->u.resultSetId);
2428             if (!rset)
2429             {
2430                 zebra_setError(zh,
2431                                YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
2432                                zs->u.simple->u.resultSetId);
2433                 return ZEBRA_FAIL;
2434             }
2435             rset_dup(rset);
2436         }
2437         else
2438         {
2439             zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2440             return ZEBRA_FAIL;
2441         }
2442         *num_result_sets = 1;
2443         *result_sets = nmem_malloc(stream, *num_result_sets *
2444                                    sizeof(**result_sets));
2445         (*result_sets)[0] = rset;
2446     }
2447     else
2448     {
2449         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2450         return ZEBRA_FAIL;
2451     }
2452     return ZEBRA_OK;
2453 }
2454
2455 struct scan_info_entry {
2456     char *term;
2457     ISAM_P isam_p;
2458 };
2459
2460 struct scan_info {
2461     struct scan_info_entry *list;
2462     ODR odr;
2463     int before, after;
2464     char prefix[20];
2465 };
2466
2467 static int scan_handle (char *name, const char *info, int pos, void *client)
2468 {
2469     int len_prefix, idx;
2470     struct scan_info *scan_info = (struct scan_info *) client;
2471
2472     len_prefix = strlen(scan_info->prefix);
2473     if (memcmp (name, scan_info->prefix, len_prefix))
2474         return 1;
2475     if (pos > 0)
2476         idx = scan_info->after - pos + scan_info->before;
2477     else
2478         idx = - pos - 1;
2479
2480     if (idx < 0)
2481         return 0;
2482     scan_info->list[idx].term = (char *)
2483         odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
2484     strcpy(scan_info->list[idx].term, name + len_prefix);
2485     assert (*info == sizeof(ISAM_P));
2486     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
2487     return 0;
2488 }
2489
2490 void zebra_term_untrans_iconv(ZebraHandle zh, NMEM stream, int reg_type,
2491                               char **dst, const char *src)
2492 {
2493     char term_src[IT_MAX_WORD];
2494     char term_dst[IT_MAX_WORD];
2495
2496     zebra_term_untrans (zh, reg_type, term_src, src);
2497
2498     if (zh->iconv_from_utf8 != 0)
2499     {
2500         int len;
2501         char *inbuf = term_src;
2502         size_t inleft = strlen(term_src);
2503         char *outbuf = term_dst;
2504         size_t outleft = sizeof(term_dst)-1;
2505         size_t ret;
2506
2507         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2508                          &outbuf, &outleft);
2509         if (ret == (size_t)(-1))
2510             len = 0;
2511         else
2512             len = outbuf - term_dst;
2513         *dst = nmem_malloc(stream, len + 1);
2514         if (len > 0)
2515             memcpy (*dst, term_dst, len);
2516         (*dst)[len] = '\0';
2517     }
2518     else
2519         *dst = nmem_strdup(stream, term_src);
2520 }
2521
2522 static void count_set(ZebraHandle zh, RSET rset, zint *count)
2523 {
2524     zint psysno = 0;
2525     struct it_key key;
2526     RSFD rfd;
2527
2528     yaz_log(YLOG_DEBUG, "count_set");
2529
2530     rset->hits_limit = zh->approx_limit;
2531
2532     *count = 0;
2533     rfd = rset_open(rset, RSETF_READ);
2534     while (rset_read(rfd, &key,0 /* never mind terms */))
2535     {
2536         if (key.mem[0] != psysno)
2537         {
2538             psysno = key.mem[0];
2539             if (rfd->counted_items >= rset->hits_limit)
2540                 break;
2541         }
2542     }
2543     rset_close (rfd);
2544     *count = rset->hits_count;
2545 }
2546
2547 #define RPN_MAX_ORDS 32
2548
2549 ZEBRA_RES rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2550                    oid_value attributeset,
2551                    int num_bases, char **basenames,
2552                    int *position, int *num_entries, ZebraScanEntry **list,
2553                    int *is_partial, RSET limit_set, int return_zero)
2554 {
2555     int i;
2556     int pos = *position;
2557     int num = *num_entries;
2558     int before;
2559     int after;
2560     int base_no;
2561     char termz[IT_MAX_WORD+20];
2562     struct scan_info *scan_info_array;
2563     ZebraScanEntry *glist;
2564     int ords[RPN_MAX_ORDS], ord_no = 0;
2565     int ptr[RPN_MAX_ORDS];
2566
2567     unsigned index_type;
2568     char *search_type = NULL;
2569     char rank_type[128];
2570     int complete_flag;
2571     int sort_flag;
2572     NMEM rset_nmem = NULL;
2573     struct rset_key_control *kc = 0;
2574
2575     *list = 0;
2576     *is_partial = 0;
2577
2578     if (attributeset == VAL_NONE)
2579         attributeset = VAL_BIB1;
2580
2581     if (!limit_set)
2582     {
2583         AttrType termset;
2584         int termset_value_numeric;
2585         const char *termset_value_string;
2586         attr_init_APT(&termset, zapt, 8);
2587         termset_value_numeric =
2588             attr_find_ex(&termset, NULL, &termset_value_string);
2589         if (termset_value_numeric != -1)
2590         {
2591             char resname[32];
2592             const char *termset_name = 0;
2593
2594             if (termset_value_numeric != -2)
2595             {
2596
2597                 sprintf(resname, "%d", termset_value_numeric);
2598                 termset_name = resname;
2599             }
2600             else
2601                 termset_name = termset_value_string;
2602
2603             limit_set = resultSetRef (zh, termset_name);
2604         }
2605     }
2606
2607     yaz_log(YLOG_DEBUG, "position = %d, num = %d set=%d",
2608             pos, num, attributeset);
2609
2610     if (zebra_maps_attr(zh->reg->zebra_maps, zapt, &index_type, &search_type,
2611                         rank_type, &complete_flag, &sort_flag))
2612     {
2613         *num_entries = 0;
2614         zebra_setError(zh, YAZ_BIB1_UNSUPP_ATTRIBUTE_TYPE, 0);
2615         return ZEBRA_FAIL;
2616     }
2617     for (base_no = 0; base_no < num_bases && ord_no < RPN_MAX_ORDS; base_no++)
2618     {
2619         int ord;
2620
2621         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2622         {
2623             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
2624                            basenames[base_no]);
2625             *num_entries = 0;
2626             return ZEBRA_FAIL;
2627         }
2628         if (zebra_apt_get_ord(zh, zapt, index_type, 0, attributeset, &ord)
2629             != ZEBRA_OK)
2630             continue;
2631         ords[ord_no++] = ord;
2632     }
2633     if (ord_no == 0)
2634     {
2635         *num_entries = 0;
2636         return ZEBRA_OK;
2637     }
2638     /* prepare dictionary scanning */
2639     if (num < 1)
2640     {
2641         *num_entries = 0;
2642         return ZEBRA_OK;
2643     }
2644     before = pos-1;
2645     if (before < 0)
2646         before = 0;
2647     after = 1+num-pos;
2648     if (after < 0)
2649         after = 0;
2650     yaz_log(YLOG_DEBUG, "rpn_scan pos=%d num=%d before=%d "
2651             "after=%d before+after=%d",
2652             pos, num, before, after, before+after);
2653     scan_info_array = (struct scan_info *)
2654         odr_malloc(stream, ord_no * sizeof(*scan_info_array));
2655     for (i = 0; i < ord_no; i++)
2656     {
2657         int j, prefix_len = 0;
2658         int before_tmp = before, after_tmp = after;
2659         struct scan_info *scan_info = scan_info_array + i;
2660         struct rpn_char_map_info rcmi;
2661
2662         rpn_char_map_prepare (zh->reg, index_type, &rcmi);
2663
2664         scan_info->before = before;
2665         scan_info->after = after;
2666         scan_info->odr = stream;
2667
2668         scan_info->list = (struct scan_info_entry *)
2669             odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
2670         for (j = 0; j<before+after; j++)
2671             scan_info->list[j].term = NULL;
2672
2673         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2674         termz[prefix_len] = 0;
2675         strcpy(scan_info->prefix, termz);
2676
2677         if (trans_scan_term(zh, zapt, termz+prefix_len, index_type) ==
2678             ZEBRA_FAIL)
2679             return ZEBRA_FAIL;
2680
2681         dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
2682                   scan_info, scan_handle);
2683     }
2684     glist = (ZebraScanEntry *)
2685         odr_malloc(stream, (before+after)*sizeof(*glist));
2686
2687     rset_nmem = nmem_create();
2688     kc = zebra_key_control_create(zh);
2689
2690     /* consider terms after main term */
2691     for (i = 0; i < ord_no; i++)
2692         ptr[i] = before;
2693
2694     *is_partial = 0;
2695     for (i = 0; i<after; i++)
2696     {
2697         int j, j0 = -1;
2698         const char *mterm = NULL;
2699         const char *tst;
2700         RSET rset = 0;
2701         int lo = i + pos-1; /* offset in result list */
2702
2703         /* find: j0 is the first of the minimal values */
2704         for (j = 0; j < ord_no; j++)
2705         {
2706             if (ptr[j] < before+after && ptr[j] >= 0 &&
2707                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2708                 (!mterm || strcmp (tst, mterm) < 0))
2709             {
2710                 j0 = j;
2711                 mterm = tst;
2712             }
2713         }
2714         if (j0 == -1)
2715             break;  /* no value found, stop */
2716
2717         /* get result set for first one , but only if it's within bounds */
2718         if (lo >= 0)
2719         {
2720             /* get result set for first term */
2721             zebra_term_untrans_iconv(zh, stream->mem, index_type,
2722                                      &glist[lo].term, mterm);
2723             rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2724                               glist[lo].term, strlen(glist[lo].term),
2725                               NULL, 0, zapt->term->which, rset_nmem,
2726                               kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2727                               0 /* term_ref_id_str */);
2728         }
2729         ptr[j0]++; /* move index for this set .. */
2730         /* get result set for remaining scan terms */
2731         for (j = j0+1; j<ord_no; j++)
2732         {
2733             if (ptr[j] < before+after && ptr[j] >= 0 &&
2734                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2735                 !strcmp (tst, mterm))
2736             {
2737                 if (lo >= 0)
2738                 {
2739                     RSET rsets[2];
2740
2741                     rsets[0] = rset;
2742                     rsets[1] =
2743                         rset_trunc(
2744                             zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
2745                             glist[lo].term,
2746                             strlen(glist[lo].term), NULL, 0,
2747                             zapt->term->which,rset_nmem,
2748                             kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2749                             0 /* term_ref_id_str */ );
2750                     rset = rset_create_or(rset_nmem, kc,
2751                                           kc->scope, 0 /* termid */,
2752                                           2, rsets);
2753                 }
2754                 ptr[j]++;
2755             }
2756         }
2757         if (lo >= 0)
2758         {
2759             zint count;
2760             /* merge with limit_set if given */
2761             if (limit_set)
2762             {
2763                 RSET rsets[2];
2764                 rsets[0] = rset;
2765                 rsets[1] = rset_dup(limit_set);
2766
2767                 rset = rset_create_and(rset_nmem, kc, kc->scope, 2, rsets);
2768             }
2769             /* count it */
2770             count_set(zh, rset, &count);
2771             glist[lo].occurrences = count;
2772             rset_delete(rset);
2773         }
2774     }
2775     if (i < after)
2776     {
2777         *num_entries -= (after-i);
2778         *is_partial = 1;
2779         if (*num_entries < 0)
2780         {
2781             (*kc->dec)(kc);
2782             nmem_destroy(rset_nmem);
2783             *num_entries = 0;
2784             return ZEBRA_OK;
2785         }
2786     }
2787     /* consider terms before main term */
2788     for (i = 0; i<ord_no; i++)
2789         ptr[i] = 0;
2790
2791     for (i = 0; i<before; i++)
2792     {
2793         int j, j0 = -1;
2794         const char *mterm = NULL;
2795         const char *tst;
2796         RSET rset;
2797         int lo = before-1-i; /* offset in result list */
2798         zint count;
2799
2800         for (j = 0; j <ord_no; j++)
2801         {
2802             if (ptr[j] < before && ptr[j] >= 0 &&
2803                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2804                 (!mterm || strcmp (tst, mterm) > 0))
2805             {
2806                 j0 = j;
2807                     mterm = tst;
2808             }
2809         }
2810         if (j0 == -1)
2811             break;
2812
2813         zebra_term_untrans_iconv(zh, stream->mem, index_type,
2814                                  &glist[lo].term, mterm);
2815
2816         rset = rset_trunc
2817             (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
2818              glist[lo].term, strlen(glist[lo].term),
2819              NULL, 0, zapt->term->which, rset_nmem,
2820              kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2821              0 /* term_ref_id_str */);
2822
2823         ptr[j0]++;
2824
2825         for (j = j0+1; j<ord_no; j++)
2826         {
2827             if (ptr[j] < before && ptr[j] >= 0 &&
2828                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2829                 !strcmp (tst, mterm))
2830             {
2831                 RSET rsets[2];
2832
2833                 rsets[0] = rset;
2834                 rsets[1] = rset_trunc(
2835                     zh,
2836                     &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
2837                     glist[lo].term,
2838                     strlen(glist[lo].term), NULL, 0,
2839                     zapt->term->which, rset_nmem,
2840                     kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2841                     0 /* term_ref_id_str */);
2842                 rset = rset_create_or(rset_nmem, kc,
2843                                       kc->scope, 0 /* termid */, 2, rsets);
2844
2845                 ptr[j]++;
2846             }
2847         }
2848         if (limit_set)
2849         {
2850             RSET rsets[2];
2851             rsets[0] = rset;
2852             rsets[1] = rset_dup(limit_set);
2853
2854             rset = rset_create_and(rset_nmem, kc, kc->scope, 2, rsets);
2855         }
2856         count_set(zh, rset, &count);
2857         glist[lo].occurrences = count;
2858         rset_delete (rset);
2859     }
2860     (*kc->dec)(kc);
2861     nmem_destroy(rset_nmem);
2862     i = before-i;
2863     if (i)
2864     {
2865         *is_partial = 1;
2866         *position -= i;
2867         *num_entries -= i;
2868         if (*num_entries <= 0)
2869         {
2870             *num_entries = 0;
2871             return ZEBRA_OK;
2872         }
2873     }
2874
2875     *list = glist + i;               /* list is set to first 'real' entry */
2876
2877     yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
2878             *position, *num_entries);
2879     return ZEBRA_OK;
2880 }
2881
2882 /*
2883  * Local variables:
2884  * c-basic-offset: 4
2885  * indent-tabs-mode: nil
2886  * End:
2887  * vim: shiftwidth=4 tabstop=8 expandtab
2888  */
2889