index/zrpn.c

   1 /* $Id: zrpn.c,v 1.222 2006-07-03 10:43:43 adam Exp $
   2    Copyright (C) 1995-2006
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with Zebra; see the file LICENSE.zebra.  If not, write to the
  19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  20 02111-1307, USA.
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #ifdef WIN32
  26 #include <io.h>
  27 #endif
  28 #if HAVE_UNISTD_H
  29 #include <unistd.h>
  30 #endif
  31 #include <ctype.h>
  32
  33 #include <yaz/diagbib1.h>
  34 #include "index.h"
  35 #include <zebra_xpath.h>
  36 #include <attrfind.h>
  37 #include <charmap.h>
  38 #include <rset.h>
  39
  40 struct rpn_char_map_info
  41 {
  42     ZebraMaps zm;
  43     int reg_type;
  44 };
  45
  46 static int log_level_set = 0;
  47 static int log_level_rpn = 0;
  48
  49
  50
  51 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  52 {
  53     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  54     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  55 #if 0
  56     if (out && *out)
  57     {
  58         const char *outp = *out;
  59         yaz_log(YLOG_LOG, "---");
  60         while (*outp)
  61         {
  62             yaz_log(YLOG_LOG, "%02X", *outp);
  63             outp++;
  64         }
  65     }
  66 #endif
  67     return out;
  68 }
  69
  70 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  71                                  struct rpn_char_map_info *map_info)
  72 {
  73     map_info->zm = reg->zebra_maps;
  74     map_info->reg_type = reg_type;
  75     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  76 }
  77
  78 #define TERM_COUNT
  79
  80 struct grep_info {
  81 #ifdef TERM_COUNT
  82     int *term_no;
  83 #endif
  84     ISAM_P *isam_p_buf;
  85     int isam_p_size;
  86     int isam_p_indx;
  87     ZebraHandle zh;
  88     int reg_type;
  89     ZebraSet termset;
  90 };
  91
  92 void zebra_term_untrans(ZebraHandle zh, int reg_type,
  93                         char *dst, const char *src)
  94 {
  95     int len = 0;
  96     while (*src)
  97     {
  98         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
  99                                            reg_type, &src);
 100         if (!cp)
 101         {
 102             if (len < IT_MAX_WORD-1)
 103                 dst[len++] = *src;
 104             src++;
 105         }
 106         else
 107             while (*cp && len < IT_MAX_WORD-1)
 108                 dst[len++] = *cp++;
 109     }
 110     dst[len] = '\0';
 111 }
 112
 113 static void add_isam_p(const char *name, const char *info,
 114                        struct grep_info *p)
 115 {
 116     if (!log_level_set)
 117     {
 118         log_level_rpn = yaz_log_module_level("rpn");
 119         log_level_set = 1;
 120     }
 121     if (p->isam_p_indx == p->isam_p_size)
 122     {
 123         ISAM_P *new_isam_p_buf;
 124 #ifdef TERM_COUNT
 125         int *new_term_no;
 126 #endif
 127         p->isam_p_size = 2*p->isam_p_size + 100;
 128         new_isam_p_buf = (ISAM_P *) xmalloc(sizeof(*new_isam_p_buf) *
 129                                             p->isam_p_size);
 130         if (p->isam_p_buf)
 131         {
 132             memcpy(new_isam_p_buf, p->isam_p_buf,
 133                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 134             xfree(p->isam_p_buf);
 135         }
 136         p->isam_p_buf = new_isam_p_buf;
 137
 138 #ifdef TERM_COUNT
 139         new_term_no = (int *) xmalloc(sizeof(*new_term_no) * p->isam_p_size);
 140         if (p->term_no)
 141         {
 142             memcpy(new_term_no, p->isam_p_buf,
 143                     p->isam_p_indx * sizeof(*p->term_no));
 144             xfree(p->term_no);
 145         }
 146         p->term_no = new_term_no;
 147 #endif
 148     }
 149     assert(*info == sizeof(*p->isam_p_buf));
 150     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 151
 152 #if 1
 153     if (p->termset)
 154     {
 155         const char *db;
 156         char term_tmp[IT_MAX_WORD];
 157         int ord = 0;
 158         const char *index_name;
 159         int len = key_SU_decode (&ord, (const unsigned char *) name);
 160
 161         zebra_term_untrans  (p->zh, p->reg_type, term_tmp, name+len);
 162         yaz_log(log_level_rpn, "grep: %d %c %s", ord, name[len], term_tmp);
 163         zebraExplain_lookup_ord(p->zh->reg->zei,
 164                                 ord, 0 /* index_type */, &db, &index_name);
 165         yaz_log(log_level_rpn, "grep:  db=%s index=%s", db, index_name);
 166
 167         resultSetAddTerm(p->zh, p->termset, name[len], db,
 168                          index_name, term_tmp);
 169     }
 170 #endif
 171     (p->isam_p_indx)++;
 172 }
 173
 174 static int grep_handle(char *name, const char *info, void *p)
 175 {
 176     add_isam_p(name, info, (struct grep_info *) p);
 177     return 0;
 178 }
 179
 180 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 181                     const char *ct1, const char *ct2, int first)
 182 {
 183     const char *s1, *s0 = *src;
 184     const char **map;
 185
 186     /* skip white space */
 187     while (*s0)
 188     {
 189         if (ct1 && strchr(ct1, *s0))
 190             break;
 191         if (ct2 && strchr(ct2, *s0))
 192             break;
 193         s1 = s0;
 194         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 195         if (**map != *CHR_SPACE)
 196             break;
 197         s0 = s1;
 198     }
 199     *src = s0;
 200     return *s0;
 201 }
 202
 203
 204 static void esc_str(char *out_buf, size_t out_size,
 205                     const char *in_buf, int in_size)
 206 {
 207     int k;
 208
 209     assert(out_buf);
 210     assert(in_buf);
 211     assert(out_size > 20);
 212     *out_buf = '\0';
 213     for (k = 0; k<in_size; k++)
 214     {
 215         int c = in_buf[k] & 0xff;
 216         int pc;
 217         if (c < 32 || c > 126)
 218             pc = '?';
 219         else
 220             pc = c;
 221         sprintf(out_buf +strlen(out_buf), "%02X:%c  ", c, pc);
 222         if (strlen(out_buf) > out_size-20)
 223         {
 224             strcat(out_buf, "..");
 225             break;
 226         }
 227     }
 228 }
 229
 230 #define REGEX_CHARS " []()|.*+?!"
 231
 232 /* term_100: handle term, where trunc = none(no operators at all) */
 233 static int term_100(ZebraMaps zebra_maps, int reg_type,
 234                     const char **src, char *dst, int space_split,
 235                     char *dst_term)
 236 {
 237     const char *s0;
 238     const char **map;
 239     int i = 0;
 240     int j = 0;
 241
 242     const char *space_start = 0;
 243     const char *space_end = 0;
 244
 245     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 246         return 0;
 247     s0 = *src;
 248     while (*s0)
 249     {
 250         const char *s1 = s0;
 251         int q_map_match = 0;
 252         map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 253                                 &q_map_match);
 254         if (space_split)
 255         {
 256             if (**map == *CHR_SPACE)
 257                 break;
 258         }
 259         else  /* complete subfield only. */
 260         {
 261             if (**map == *CHR_SPACE)
 262             {   /* save space mapping for later  .. */
 263                 space_start = s1;
 264                 space_end = s0;
 265                 continue;
 266             }
 267             else if (space_start)
 268             {   /* reload last space */
 269                 while (space_start < space_end)
 270                 {
 271                     if (strchr(REGEX_CHARS, *space_start))
 272                         dst[i++] = '\\';
 273                     dst_term[j++] = *space_start;
 274                     dst[i++] = *space_start++;
 275                 }
 276                 /* and reset */
 277                 space_start = space_end = 0;
 278             }
 279         }
 280         /* add non-space char */
 281         memcpy(dst_term+j, s1, s0 - s1);
 282         j += (s0 - s1);
 283         if (!q_map_match)
 284         {
 285             while (s1 < s0)
 286             {
 287                 if (strchr(REGEX_CHARS, *s1))
 288                     dst[i++] = '\\';
 289                 dst[i++] = *s1++;
 290             }
 291         }
 292         else
 293         {
 294             char tmpbuf[80];
 295             esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 296
 297             strcpy(dst + i, map[0]);
 298             i += strlen(map[0]);
 299         }
 300     }
 301     dst[i] = '\0';
 302     dst_term[j] = '\0';
 303     *src = s0;
 304     return i;
 305 }
 306
 307 /* term_101: handle term, where trunc = Process # */
 308 static int term_101(ZebraMaps zebra_maps, int reg_type,
 309                     const char **src, char *dst, int space_split,
 310                     char *dst_term)
 311 {
 312     const char *s0;
 313     const char **map;
 314     int i = 0;
 315     int j = 0;
 316
 317     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 318         return 0;
 319     s0 = *src;
 320     while (*s0)
 321     {
 322         if (*s0 == '#')
 323         {
 324             dst[i++] = '.';
 325             dst[i++] = '*';
 326             dst_term[j++] = *s0++;
 327         }
 328         else
 329         {
 330             const char *s1 = s0;
 331             int q_map_match = 0;
 332             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 333                                     &q_map_match);
 334             if (space_split && **map == *CHR_SPACE)
 335                 break;
 336
 337             /* add non-space char */
 338             memcpy(dst_term+j, s1, s0 - s1);
 339             j += (s0 - s1);
 340             if (!q_map_match)
 341             {
 342                 while (s1 < s0)
 343                 {
 344                     if (strchr(REGEX_CHARS, *s1))
 345                         dst[i++] = '\\';
 346                     dst[i++] = *s1++;
 347                 }
 348             }
 349             else
 350             {
 351                 char tmpbuf[80];
 352                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 353
 354                 strcpy(dst + i, map[0]);
 355                 i += strlen(map[0]);
 356             }
 357         }
 358     }
 359     dst[i] = '\0';
 360     dst_term[j++] = '\0';
 361     *src = s0;
 362     return i;
 363 }
 364
 365 /* term_103: handle term, where trunc = re-2 (regular expressions) */
 366 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 367                     char *dst, int *errors, int space_split,
 368                     char *dst_term)
 369 {
 370     int i = 0;
 371     int j = 0;
 372     const char *s0;
 373     const char **map;
 374
 375     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 376         return 0;
 377     s0 = *src;
 378     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 379         isdigit(((const unsigned char *)s0)[1]))
 380     {
 381         *errors = s0[1] - '0';
 382         s0 += 3;
 383         if (*errors > 3)
 384             *errors = 3;
 385     }
 386     while (*s0)
 387     {
 388         if (strchr("^\\()[].*+?|-", *s0))
 389         {
 390             dst_term[j++] = *s0;
 391             dst[i++] = *s0++;
 392         }
 393         else
 394         {
 395             const char *s1 = s0;
 396             int q_map_match = 0;
 397             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 398                                     &q_map_match);
 399             if (space_split && **map == *CHR_SPACE)
 400                 break;
 401
 402             /* add non-space char */
 403             memcpy(dst_term+j, s1, s0 - s1);
 404             j += (s0 - s1);
 405             if (!q_map_match)
 406             {
 407                 while (s1 < s0)
 408                 {
 409                     if (strchr(REGEX_CHARS, *s1))
 410                         dst[i++] = '\\';
 411                     dst[i++] = *s1++;
 412                 }
 413             }
 414             else
 415             {
 416                 char tmpbuf[80];
 417                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 418
 419                 strcpy(dst + i, map[0]);
 420                 i += strlen(map[0]);
 421             }
 422         }
 423     }
 424     dst[i] = '\0';
 425     dst_term[j] = '\0';
 426     *src = s0;
 427
 428     return i;
 429 }
 430
 431 /* term_103: handle term, where trunc = re-1 (regular expressions) */
 432 static int term_102(ZebraMaps zebra_maps, int reg_type, const char **src,
 433                     char *dst, int space_split, char *dst_term)
 434 {
 435     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 436                     dst_term);
 437 }
 438
 439
 440 /* term_104: handle term, where trunc = Process # and ! */
 441 static int term_104(ZebraMaps zebra_maps, int reg_type,
 442                     const char **src, char *dst, int space_split,
 443                     char *dst_term)
 444 {
 445     const char *s0;
 446     const char **map;
 447     int i = 0;
 448     int j = 0;
 449
 450     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 451         return 0;
 452     s0 = *src;
 453     while (*s0)
 454     {
 455         if (*s0 == '?')
 456         {
 457             dst_term[j++] = *s0++;
 458             if (*s0 >= '0' && *s0 <= '9')
 459             {
 460                 int limit = 0;
 461                 while (*s0 >= '0' && *s0 <= '9')
 462                 {
 463                     limit = limit * 10 + (*s0 - '0');
 464                     dst_term[j++] = *s0++;
 465                 }
 466                 if (limit > 20)
 467                     limit = 20;
 468                 while (--limit >= 0)
 469                 {
 470                     dst[i++] = '.';
 471                     dst[i++] = '?';
 472                 }
 473             }
 474             else
 475             {
 476                 dst[i++] = '.';
 477                 dst[i++] = '*';
 478             }
 479         }
 480         else if (*s0 == '*')
 481         {
 482             dst[i++] = '.';
 483             dst[i++] = '*';
 484             dst_term[j++] = *s0++;
 485         }
 486         else if (*s0 == '#')
 487         {
 488             dst[i++] = '.';
 489             dst_term[j++] = *s0++;
 490         }
 491         else
 492         {
 493             const char *s1 = s0;
 494             int q_map_match = 0;
 495             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 496                                     &q_map_match);
 497             if (space_split && **map == *CHR_SPACE)
 498                 break;
 499
 500             /* add non-space char */
 501             memcpy(dst_term+j, s1, s0 - s1);
 502             j += (s0 - s1);
 503             if (!q_map_match)
 504             {
 505                 while (s1 < s0)
 506                 {
 507                     if (strchr(REGEX_CHARS, *s1))
 508                         dst[i++] = '\\';
 509                     dst[i++] = *s1++;
 510                 }
 511             }
 512             else
 513             {
 514                 char tmpbuf[80];
 515                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 516
 517                 strcpy(dst + i, map[0]);
 518                 i += strlen(map[0]);
 519             }
 520         }
 521     }
 522     dst[i] = '\0';
 523     dst_term[j++] = '\0';
 524     *src = s0;
 525     return i;
 526 }
 527
 528 /* term_105/106: handle term, where trunc = Process * and ! and right trunc */
 529 static int term_105(ZebraMaps zebra_maps, int reg_type,
 530                     const char **src, char *dst, int space_split,
 531                     char *dst_term, int right_truncate)
 532 {
 533     const char *s0;
 534     const char **map;
 535     int i = 0;
 536     int j = 0;
 537
 538     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 539         return 0;
 540     s0 = *src;
 541     while (*s0)
 542     {
 543         if (*s0 == '*')
 544         {
 545             dst[i++] = '.';
 546             dst[i++] = '*';
 547             dst_term[j++] = *s0++;
 548         }
 549         else if (*s0 == '!')
 550         {
 551             dst[i++] = '.';
 552             dst_term[j++] = *s0++;
 553         }
 554         else
 555         {
 556             const char *s1 = s0;
 557             int q_map_match = 0;
 558             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 559                                     &q_map_match);
 560             if (space_split && **map == *CHR_SPACE)
 561                 break;
 562
 563             /* add non-space char */
 564             memcpy(dst_term+j, s1, s0 - s1);
 565             j += (s0 - s1);
 566             if (!q_map_match)
 567             {
 568                 while (s1 < s0)
 569                 {
 570                     if (strchr(REGEX_CHARS, *s1))
 571                         dst[i++] = '\\';
 572                     dst[i++] = *s1++;
 573                 }
 574             }
 575             else
 576             {
 577                 char tmpbuf[80];
 578                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 579
 580                 strcpy(dst + i, map[0]);
 581                 i += strlen(map[0]);
 582             }
 583         }
 584     }
 585     if (right_truncate)
 586     {
 587         dst[i++] = '.';
 588         dst[i++] = '*';
 589     }
 590     dst[i] = '\0';
 591
 592     dst_term[j++] = '\0';
 593     *src = s0;
 594     return i;
 595 }
 596
 597
 598 /* gen_regular_rel - generate regular expression from relation
 599  *  val:     border value (inclusive)
 600  *  islt:    1 if <=; 0 if >=.
 601  */
 602 static void gen_regular_rel(char *dst, int val, int islt)
 603 {
 604     int dst_p;
 605     int w, d, i;
 606     int pos = 0;
 607     char numstr[20];
 608
 609     yaz_log(YLOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 610     if (val >= 0)
 611     {
 612         if (islt)
 613             strcpy(dst, "(-[0-9]+|(");
 614         else
 615             strcpy(dst, "((");
 616     }
 617     else
 618     {
 619         if (!islt)
 620         {
 621             strcpy(dst, "([0-9]+|-(");
 622             dst_p = strlen(dst);
 623             islt = 1;
 624         }
 625         else
 626         {
 627             strcpy(dst, "(-(");
 628             islt = 0;
 629         }
 630         val = -val;
 631     }
 632     dst_p = strlen(dst);
 633     sprintf(numstr, "%d", val);
 634     for (w = strlen(numstr); --w >= 0; pos++)
 635     {
 636         d = numstr[w];
 637         if (pos > 0)
 638         {
 639             if (islt)
 640             {
 641                 if (d == '0')
 642                     continue;
 643                 d--;
 644             }
 645             else
 646             {
 647                 if (d == '9')
 648                     continue;
 649                 d++;
 650             }
 651         }
 652
 653         strcpy(dst + dst_p, numstr);
 654         dst_p = strlen(dst) - pos - 1;
 655
 656         if (islt)
 657         {
 658             if (d != '0')
 659             {
 660                 dst[dst_p++] = '[';
 661                 dst[dst_p++] = '0';
 662                 dst[dst_p++] = '-';
 663                 dst[dst_p++] = d;
 664                 dst[dst_p++] = ']';
 665             }
 666             else
 667                 dst[dst_p++] = d;
 668         }
 669         else
 670         {
 671             if (d != '9')
 672             {
 673                 dst[dst_p++] = '[';
 674                 dst[dst_p++] = d;
 675                 dst[dst_p++] = '-';
 676                 dst[dst_p++] = '9';
 677                 dst[dst_p++] = ']';
 678             }
 679             else
 680                 dst[dst_p++] = d;
 681         }
 682         for (i = 0; i<pos; i++)
 683         {
 684             dst[dst_p++] = '[';
 685             dst[dst_p++] = '0';
 686             dst[dst_p++] = '-';
 687             dst[dst_p++] = '9';
 688             dst[dst_p++] = ']';
 689         }
 690         dst[dst_p++] = '|';
 691     }
 692     dst[dst_p] = '\0';
 693     if (islt)
 694     {
 695         /* match everything less than 10^(pos-1) */
 696         strcat(dst, "0*");
 697         for (i = 1; i<pos; i++)
 698             strcat(dst, "[0-9]?");
 699     }
 700     else
 701     {
 702         /* match everything greater than 10^pos */
 703         for (i = 0; i <= pos; i++)
 704             strcat(dst, "[0-9]");
 705         strcat(dst, "[0-9]*");
 706     }
 707     strcat(dst, "))");
 708 }
 709
 710 void string_rel_add_char(char **term_p, const char *src, int *indx)
 711 {
 712     if (src[*indx] == '\\')
 713         *(*term_p)++ = src[(*indx)++];
 714     *(*term_p)++ = src[(*indx)++];
 715 }
 716
 717 /*
 718  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 719  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 720  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 721  *              ([^-a].*|a[^-b].*|ab[c-].*)
 722  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 723  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 724  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 725  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 726  */
 727 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 728                            const char **term_sub, char *term_dict,
 729                            oid_value attributeSet,
 730                            int reg_type, int space_split, char *term_dst,
 731                            int *error_code)
 732 {
 733     AttrType relation;
 734     int relation_value;
 735     int i;
 736     char *term_tmp = term_dict + strlen(term_dict);
 737     char term_component[2*IT_MAX_WORD+20];
 738
 739     attr_init_APT(&relation, zapt, 2);
 740     relation_value = attr_find(&relation, NULL);
 741
 742     *error_code = 0;
 743     yaz_log(YLOG_DEBUG, "string relation value=%d", relation_value);
 744     switch (relation_value)
 745     {
 746     case 1:
 747         if (!term_100(zh->reg->zebra_maps, reg_type,
 748                       term_sub, term_component,
 749                       space_split, term_dst))
 750             return 0;
 751         yaz_log(log_level_rpn, "Relation <");
 752
 753         *term_tmp++ = '(';
 754         for (i = 0; term_component[i]; )
 755         {
 756             int j = 0;
 757
 758             if (i)
 759                 *term_tmp++ = '|';
 760             while (j < i)
 761                 string_rel_add_char(&term_tmp, term_component, &j);
 762
 763             *term_tmp++ = '[';
 764
 765             *term_tmp++ = '^';
 766             string_rel_add_char(&term_tmp, term_component, &i);
 767             *term_tmp++ = '-';
 768
 769             *term_tmp++ = ']';
 770             *term_tmp++ = '.';
 771             *term_tmp++ = '*';
 772
 773             if ((term_tmp - term_dict) > IT_MAX_WORD)
 774                 break;
 775         }
 776         *term_tmp++ = ')';
 777         *term_tmp = '\0';
 778         break;
 779     case 2:
 780         if (!term_100(zh->reg->zebra_maps, reg_type,
 781                       term_sub, term_component,
 782                       space_split, term_dst))
 783             return 0;
 784         yaz_log(log_level_rpn, "Relation <=");
 785
 786         *term_tmp++ = '(';
 787         for (i = 0; term_component[i]; )
 788         {
 789             int j = 0;
 790
 791             while (j < i)
 792                 string_rel_add_char(&term_tmp, term_component, &j);
 793             *term_tmp++ = '[';
 794
 795             *term_tmp++ = '^';
 796             string_rel_add_char(&term_tmp, term_component, &i);
 797             *term_tmp++ = '-';
 798
 799             *term_tmp++ = ']';
 800             *term_tmp++ = '.';
 801             *term_tmp++ = '*';
 802
 803             *term_tmp++ = '|';
 804
 805             if ((term_tmp - term_dict) > IT_MAX_WORD)
 806                 break;
 807         }
 808         for (i = 0; term_component[i]; )
 809             string_rel_add_char(&term_tmp, term_component, &i);
 810         *term_tmp++ = ')';
 811         *term_tmp = '\0';
 812         break;
 813     case 5:
 814         if (!term_100 (zh->reg->zebra_maps, reg_type,
 815                        term_sub, term_component, space_split, term_dst))
 816             return 0;
 817         yaz_log(log_level_rpn, "Relation >");
 818
 819         *term_tmp++ = '(';
 820         for (i = 0; term_component[i];)
 821         {
 822             int j = 0;
 823
 824             while (j < i)
 825                 string_rel_add_char(&term_tmp, term_component, &j);
 826             *term_tmp++ = '[';
 827
 828             *term_tmp++ = '^';
 829             *term_tmp++ = '-';
 830             string_rel_add_char(&term_tmp, term_component, &i);
 831
 832             *term_tmp++ = ']';
 833             *term_tmp++ = '.';
 834             *term_tmp++ = '*';
 835
 836             *term_tmp++ = '|';
 837
 838             if ((term_tmp - term_dict) > IT_MAX_WORD)
 839                 break;
 840         }
 841         for (i = 0; term_component[i];)
 842             string_rel_add_char(&term_tmp, term_component, &i);
 843         *term_tmp++ = '.';
 844         *term_tmp++ = '+';
 845         *term_tmp++ = ')';
 846         *term_tmp = '\0';
 847         break;
 848     case 4:
 849         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 850                       term_component, space_split, term_dst))
 851             return 0;
 852         yaz_log(log_level_rpn, "Relation >=");
 853
 854         *term_tmp++ = '(';
 855         for (i = 0; term_component[i];)
 856         {
 857             int j = 0;
 858
 859             if (i)
 860                 *term_tmp++ = '|';
 861             while (j < i)
 862                 string_rel_add_char(&term_tmp, term_component, &j);
 863             *term_tmp++ = '[';
 864
 865             if (term_component[i+1])
 866             {
 867                 *term_tmp++ = '^';
 868                 *term_tmp++ = '-';
 869                 string_rel_add_char(&term_tmp, term_component, &i);
 870             }
 871             else
 872             {
 873                 string_rel_add_char(&term_tmp, term_component, &i);
 874                 *term_tmp++ = '-';
 875             }
 876             *term_tmp++ = ']';
 877             *term_tmp++ = '.';
 878             *term_tmp++ = '*';
 879
 880             if ((term_tmp - term_dict) > IT_MAX_WORD)
 881                 break;
 882         }
 883         *term_tmp++ = ')';
 884         *term_tmp = '\0';
 885         break;
 886     case 3:
 887     case 102:
 888     case -1:
 889         if (!**term_sub)
 890             return 1;
 891         yaz_log(log_level_rpn, "Relation =");
 892         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 893                       term_component, space_split, term_dst))
 894             return 0;
 895         strcat(term_tmp, "(");
 896         strcat(term_tmp, term_component);
 897         strcat(term_tmp, ")");
 898         break;
 899     case 103:
 900         yaz_log(log_level_rpn, "Relation always matches");
 901         /* skip to end of term (we don't care what it is) */
 902         while (**term_sub != '\0')
 903             (*term_sub)++;
 904         break;
 905     default:
 906         *error_code = YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
 907         return 0;
 908     }
 909     return 1;
 910 }
 911
 912 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 913                              const char **term_sub,
 914                              oid_value attributeSet, NMEM stream,
 915                              struct grep_info *grep_info,
 916                              int reg_type, int complete_flag,
 917                              int num_bases, char **basenames,
 918                              char *term_dst,
 919                              const char *xpath_use,
 920                              struct ord_list **ol);
 921
 922 static ZEBRA_RES term_limits_APT(ZebraHandle zh,
 923                                  Z_AttributesPlusTerm *zapt,
 924                                  zint *hits_limit_value,
 925                                  const char **term_ref_id_str,
 926                                  NMEM nmem)
 927 {
 928     AttrType term_ref_id_attr;
 929     AttrType hits_limit_attr;
 930     int term_ref_id_int;
 931
 932     attr_init_APT(&hits_limit_attr, zapt, 9);
 933     *hits_limit_value  = attr_find(&hits_limit_attr, NULL);
 934
 935     attr_init_APT(&term_ref_id_attr, zapt, 10);
 936     term_ref_id_int = attr_find_ex(&term_ref_id_attr, NULL, term_ref_id_str);
 937     if (term_ref_id_int >= 0)
 938     {
 939         char *res = nmem_malloc(nmem, 20);
 940         sprintf(res, "%d", term_ref_id_int);
 941         *term_ref_id_str = res;
 942     }
 943
 944     /* no limit given ? */
 945     if (*hits_limit_value == -1)
 946     {
 947         if (*term_ref_id_str)
 948         {
 949             /* use global if term_ref is present */
 950             *hits_limit_value = zh->approx_limit;
 951         }
 952         else
 953         {
 954             /* no counting if term_ref is not present */
 955             *hits_limit_value = 0;
 956         }
 957     }
 958     else if (*hits_limit_value == 0)
 959     {
 960         /* 0 is the same as global limit */
 961         *hits_limit_value = zh->approx_limit;
 962     }
 963     yaz_log(YLOG_DEBUG, "term_limits_APT ref_id=%s limit=" ZINT_FORMAT,
 964             *term_ref_id_str ? *term_ref_id_str : "none",
 965             *hits_limit_value);
 966     return ZEBRA_OK;
 967 }
 968
 969 static ZEBRA_RES term_trunc(ZebraHandle zh,
 970                             Z_AttributesPlusTerm *zapt,
 971                             const char **term_sub,
 972                             oid_value attributeSet, NMEM stream,
 973                             struct grep_info *grep_info,
 974                             int reg_type, int complete_flag,
 975                             int num_bases, char **basenames,
 976                             char *term_dst,
 977                             const char *rank_type,
 978                             const char *xpath_use,
 979                             NMEM rset_nmem,
 980                             RSET *rset,
 981                             struct rset_key_control *kc)
 982 {
 983     ZEBRA_RES res;
 984     struct ord_list *ol;
 985     zint hits_limit_value;
 986     const char *term_ref_id_str = 0;
 987     *rset = 0;
 988
 989     term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str, stream);
 990     grep_info->isam_p_indx = 0;
 991     res = string_term(zh, zapt, term_sub, attributeSet, stream, grep_info,
 992                       reg_type, complete_flag, num_bases, basenames,
 993                       term_dst, xpath_use, &ol);
 994     if (res != ZEBRA_OK)
 995         return res;
 996     if (!*term_sub)  /* no more terms ? */
 997         return res;
 998     yaz_log(log_level_rpn, "term: %s", term_dst);
 999     *rset = rset_trunc(zh, grep_info->isam_p_buf,
1000                        grep_info->isam_p_indx, term_dst,
1001                        strlen(term_dst), rank_type, 1 /* preserve pos */,
1002                        zapt->term->which, rset_nmem,
1003                        kc, kc->scope, ol, reg_type, hits_limit_value,
1004                        term_ref_id_str);
1005     if (!*rset)
1006         return ZEBRA_FAIL;
1007     return ZEBRA_OK;
1008 }
1009
1010 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1011                              const char **term_sub,
1012                              oid_value attributeSet, NMEM stream,
1013                              struct grep_info *grep_info,
1014                              int reg_type, int complete_flag,
1015                              int num_bases, char **basenames,
1016                              char *term_dst,
1017                              const char *xpath_use,
1018                              struct ord_list **ol)
1019 {
1020     char term_dict[2*IT_MAX_WORD+4000];
1021     int j, r, base_no;
1022     AttrType truncation;
1023     int truncation_value;
1024     const char *termp;
1025     struct rpn_char_map_info rcmi;
1026     int space_split = complete_flag ? 0 : 1;
1027
1028     int bases_ok = 0;     /* no of databases with OK attribute */
1029
1030     *ol = ord_list_create(stream);
1031
1032     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1033     attr_init_APT(&truncation, zapt, 5);
1034     truncation_value = attr_find(&truncation, NULL);
1035     yaz_log(log_level_rpn, "truncation value %d", truncation_value);
1036
1037     for (base_no = 0; base_no < num_bases; base_no++)
1038     {
1039         int ord = -1;
1040         int regex_range = 0;
1041         int max_pos, prefix_len = 0;
1042         int relation_error;
1043         char ord_buf[32];
1044         int ord_len, i;
1045
1046         termp = *term_sub; /* start of term for each database */
1047
1048         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1049         {
1050             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1051                            basenames[base_no]);
1052             return ZEBRA_FAIL;
1053         }
1054
1055         if (zebra_apt_get_ord(zh, zapt, reg_type, xpath_use,
1056                               attributeSet, &ord) != ZEBRA_OK)
1057             continue;
1058
1059         bases_ok++;
1060
1061         *ol = ord_list_append(stream, *ol, ord);
1062         ord_len = key_SU_encode (ord, ord_buf);
1063
1064         term_dict[prefix_len++] = '(';
1065         for (i = 0; i<ord_len; i++)
1066         {
1067             term_dict[prefix_len++] = 1;  /* our internal regexp escape char */
1068             term_dict[prefix_len++] = ord_buf[i];
1069         }
1070         term_dict[prefix_len++] = ')';
1071         term_dict[prefix_len] = '\0';
1072         j = prefix_len;
1073         switch (truncation_value)
1074         {
1075         case -1:         /* not specified */
1076         case 100:        /* do not truncate */
1077             if (!string_relation(zh, zapt, &termp, term_dict,
1078                                  attributeSet,
1079                                  reg_type, space_split, term_dst,
1080                                  &relation_error))
1081             {
1082                 if (relation_error)
1083                 {
1084                     zebra_setError(zh, relation_error, 0);
1085                     return ZEBRA_FAIL;
1086                 }
1087                 *term_sub = 0;
1088                 return ZEBRA_OK;
1089             }
1090             break;
1091         case 1:          /* right truncation */
1092             term_dict[j++] = '(';
1093             if (!term_100(zh->reg->zebra_maps, reg_type,
1094                           &termp, term_dict + j, space_split, term_dst))
1095             {
1096                 *term_sub = 0;
1097                 return ZEBRA_OK;
1098             }
1099             strcat(term_dict, ".*)");
1100             break;
1101         case 2:          /* keft truncation */
1102             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1103             if (!term_100(zh->reg->zebra_maps, reg_type,
1104                           &termp, term_dict + j, space_split, term_dst))
1105             {
1106                 *term_sub = 0;
1107                 return ZEBRA_OK;
1108             }
1109             strcat(term_dict, ")");
1110             break;
1111         case 3:          /* left&right truncation */
1112             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1113             if (!term_100(zh->reg->zebra_maps, reg_type,
1114                           &termp, term_dict + j, space_split, term_dst))
1115             {
1116                 *term_sub = 0;
1117                 return ZEBRA_OK;
1118             }
1119             strcat(term_dict, ".*)");
1120             break;
1121         case 101:        /* process # in term */
1122             term_dict[j++] = '(';
1123             if (!term_101(zh->reg->zebra_maps, reg_type,
1124                           &termp, term_dict + j, space_split, term_dst))
1125             {
1126                 *term_sub = 0;
1127                 return ZEBRA_OK;
1128             }
1129             strcat(term_dict, ")");
1130             break;
1131         case 102:        /* Regexp-1 */
1132             term_dict[j++] = '(';
1133             if (!term_102(zh->reg->zebra_maps, reg_type,
1134                           &termp, term_dict + j, space_split, term_dst))
1135             {
1136                 *term_sub = 0;
1137                 return ZEBRA_OK;
1138             }
1139             strcat(term_dict, ")");
1140             break;
1141         case 103:       /* Regexp-2 */
1142             regex_range = 1;
1143             term_dict[j++] = '(';
1144             if (!term_103(zh->reg->zebra_maps, reg_type,
1145                           &termp, term_dict + j, &regex_range,
1146                           space_split, term_dst))
1147             {
1148                 *term_sub = 0;
1149                 return ZEBRA_OK;
1150             }
1151             strcat(term_dict, ")");
1152             break;
1153         case 104:        /* process # and ! in term */
1154             term_dict[j++] = '(';
1155             if (!term_104(zh->reg->zebra_maps, reg_type,
1156                           &termp, term_dict + j, space_split, term_dst))
1157             {
1158                 *term_sub = 0;
1159                 return ZEBRA_OK;
1160             }
1161             strcat(term_dict, ")");
1162             break;
1163         case 105:        /* process * and ! in term */
1164             term_dict[j++] = '(';
1165             if (!term_105(zh->reg->zebra_maps, reg_type,
1166                           &termp, term_dict + j, space_split, term_dst, 1))
1167             {
1168                 *term_sub = 0;
1169                 return ZEBRA_OK;
1170             }
1171             strcat(term_dict, ")");
1172             break;
1173         case 106:        /* process * and ! in term */
1174             term_dict[j++] = '(';
1175             if (!term_105(zh->reg->zebra_maps, reg_type,
1176                           &termp, term_dict + j, space_split, term_dst, 0))
1177             {
1178                 *term_sub = 0;
1179                 return ZEBRA_OK;
1180             }
1181             strcat(term_dict, ")");
1182             break;
1183         default:
1184             zebra_setError_zint(zh,
1185                                 YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE,
1186                                 truncation_value);
1187             return ZEBRA_FAIL;
1188         }
1189         if (1)
1190         {
1191             char buf[80];
1192             const char *input = term_dict + prefix_len;
1193             esc_str(buf, sizeof(buf), input, strlen(input));
1194         }
1195         yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_dict+prefix_len);
1196         r = dict_lookup_grep(zh->reg->dict, term_dict, regex_range,
1197                              grep_info, &max_pos,
1198                              ord_len /* number of "exact" chars */,
1199                              grep_handle);
1200         if (r)
1201             yaz_log(YLOG_WARN, "dict_lookup_grep fail %d", r);
1202     }
1203     if (!bases_ok)
1204         return ZEBRA_FAIL;
1205     *term_sub = termp;
1206     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1207     return ZEBRA_OK;
1208 }
1209
1210
1211 /* convert APT search term to UTF8 */
1212 static ZEBRA_RES zapt_term_to_utf8(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1213                                    char *termz)
1214 {
1215     size_t sizez;
1216     Z_Term *term = zapt->term;
1217
1218     switch (term->which)
1219     {
1220     case Z_Term_general:
1221         if (zh->iconv_to_utf8 != 0)
1222         {
1223             char *inbuf = (char *) term->u.general->buf;
1224             size_t inleft = term->u.general->len;
1225             char *outbuf = termz;
1226             size_t outleft = IT_MAX_WORD-1;
1227             size_t ret;
1228
1229             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1230                         &outbuf, &outleft);
1231             if (ret == (size_t)(-1))
1232             {
1233                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1234                 zebra_setError(
1235                     zh,
1236                     YAZ_BIB1_QUERY_TERM_INCLUDES_CHARS_THAT_DO_NOT_TRANSLATE_INTO_,
1237                     0);
1238                 return ZEBRA_FAIL;
1239             }
1240             *outbuf = 0;
1241         }
1242         else
1243         {
1244             sizez = term->u.general->len;
1245             if (sizez > IT_MAX_WORD-1)
1246                 sizez = IT_MAX_WORD-1;
1247             memcpy (termz, term->u.general->buf, sizez);
1248             termz[sizez] = '\0';
1249         }
1250         break;
1251     case Z_Term_characterString:
1252         sizez = strlen(term->u.characterString);
1253         if (sizez > IT_MAX_WORD-1)
1254             sizez = IT_MAX_WORD-1;
1255         memcpy (termz, term->u.characterString, sizez);
1256         termz[sizez] = '\0';
1257         break;
1258     default:
1259         zebra_setError(zh, YAZ_BIB1_UNSUPP_CODED_VALUE_FOR_TERM, 0);
1260         return ZEBRA_FAIL;
1261     }
1262     return ZEBRA_OK;
1263 }
1264
1265 /* convert APT SCAN term to internal cmap */
1266 static ZEBRA_RES trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1267                                  char *termz, int reg_type)
1268 {
1269     char termz0[IT_MAX_WORD];
1270
1271     if (zapt_term_to_utf8(zh, zapt, termz0) == ZEBRA_FAIL)
1272         return ZEBRA_FAIL;    /* error */
1273     else
1274     {
1275         const char **map;
1276         const char *cp = (const char *) termz0;
1277         const char *cp_end = cp + strlen(cp);
1278         const char *src;
1279         int i = 0;
1280         const char *space_map = NULL;
1281         int len;
1282
1283         while ((len = (cp_end - cp)) > 0)
1284         {
1285             map = zebra_maps_input(zh->reg->zebra_maps, reg_type, &cp, len, 0);
1286             if (**map == *CHR_SPACE)
1287                 space_map = *map;
1288             else
1289             {
1290                 if (i && space_map)
1291                     for (src = space_map; *src; src++)
1292                         termz[i++] = *src;
1293                 space_map = NULL;
1294                 for (src = *map; *src; src++)
1295                     termz[i++] = *src;
1296             }
1297         }
1298         termz[i] = '\0';
1299     }
1300     return ZEBRA_OK;
1301 }
1302
1303 static void grep_info_delete(struct grep_info *grep_info)
1304 {
1305 #ifdef TERM_COUNT
1306     xfree(grep_info->term_no);
1307 #endif
1308     xfree(grep_info->isam_p_buf);
1309 }
1310
1311 static ZEBRA_RES grep_info_prepare(ZebraHandle zh,
1312                                    Z_AttributesPlusTerm *zapt,
1313                                    struct grep_info *grep_info,
1314                                    int reg_type)
1315 {
1316     AttrType termset;
1317     int termset_value_numeric;
1318     const char *termset_value_string;
1319
1320 #ifdef TERM_COUNT
1321     grep_info->term_no = 0;
1322 #endif
1323     grep_info->isam_p_size = 0;
1324     grep_info->isam_p_buf = NULL;
1325     grep_info->zh = zh;
1326     grep_info->reg_type = reg_type;
1327     grep_info->termset = 0;
1328
1329     if (!zapt)
1330         return ZEBRA_OK;
1331     attr_init_APT(&termset, zapt, 8);
1332     termset_value_numeric =
1333         attr_find_ex(&termset, NULL, &termset_value_string);
1334     if (termset_value_numeric != -1)
1335     {
1336         char resname[32];
1337         const char *termset_name = 0;
1338         if (termset_value_numeric != -2)
1339         {
1340
1341             sprintf(resname, "%d", termset_value_numeric);
1342             termset_name = resname;
1343         }
1344         else
1345             termset_name = termset_value_string;
1346         yaz_log(log_level_rpn, "creating termset set %s", termset_name);
1347         grep_info->termset = resultSetAdd(zh, termset_name, 1);
1348         if (!grep_info->termset)
1349         {
1350             zebra_setError(zh, YAZ_BIB1_ILLEGAL_RESULT_SET_NAME, termset_name);
1351             return ZEBRA_FAIL;
1352         }
1353     }
1354     return ZEBRA_OK;
1355 }
1356
1357 /**
1358   \brief Create result set(s) for list of terms
1359   \param zh Zebra Handle
1360   \param termz term as used in query but converted to UTF-8
1361   \param attributeSet default attribute set
1362   \param stream memory for result
1363   \param reg_type register type ('w', 'p',..)
1364   \param complete_flag whether it's phrases or not
1365   \param rank_type term flags for ranking
1366   \param xpath_use use attribute for X-Path (-1 for no X-path)
1367   \param num_bases number of databases
1368   \param basenames array of databases
1369   \param rset_mem memory for result sets
1370   \param result_sets output result set for each term in list (output)
1371   \param number number of output result sets
1372   \param kc rset key control to be used for created result sets
1373 */
1374 static ZEBRA_RES term_list_trunc(ZebraHandle zh,
1375                                  Z_AttributesPlusTerm *zapt,
1376                                  const char *termz,
1377                                  oid_value attributeSet,
1378                                  NMEM stream,
1379                                  int reg_type, int complete_flag,
1380                                  const char *rank_type,
1381                                  const char *xpath_use,
1382                                  int num_bases, char **basenames,
1383                                  NMEM rset_nmem,
1384                                  RSET **result_sets, int *num_result_sets,
1385                                  struct rset_key_control *kc)
1386 {
1387     char term_dst[IT_MAX_WORD+1];
1388     struct grep_info grep_info;
1389     const char *termp = termz;
1390     int alloc_sets = 0;
1391
1392     *num_result_sets = 0;
1393     *term_dst = 0;
1394     if (grep_info_prepare(zh, zapt, &grep_info, reg_type) == ZEBRA_FAIL)
1395         return ZEBRA_FAIL;
1396     while(1)
1397     {
1398         ZEBRA_RES res;
1399
1400         if (alloc_sets == *num_result_sets)
1401         {
1402             int add = 10;
1403             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1404                                               sizeof(*rnew));
1405             if (alloc_sets)
1406                 memcpy(rnew, *result_sets, alloc_sets * sizeof(*rnew));
1407             alloc_sets = alloc_sets + add;
1408             *result_sets = rnew;
1409         }
1410         res = term_trunc(zh, zapt, &termp, attributeSet,
1411                          stream, &grep_info,
1412                          reg_type, complete_flag,
1413                          num_bases, basenames,
1414                          term_dst, rank_type,
1415                          xpath_use, rset_nmem,
1416                          &(*result_sets)[*num_result_sets],
1417                          kc);
1418         if (res != ZEBRA_OK)
1419         {
1420             int i;
1421             for (i = 0; i < *num_result_sets; i++)
1422                 rset_delete((*result_sets)[i]);
1423             grep_info_delete (&grep_info);
1424             return res;
1425         }
1426         if ((*result_sets)[*num_result_sets] == 0)
1427             break;
1428         (*num_result_sets)++;
1429
1430         if (!*termp)
1431             break;
1432     }
1433     grep_info_delete(&grep_info);
1434     return ZEBRA_OK;
1435 }
1436
1437 static ZEBRA_RES rpn_search_APT_phrase(ZebraHandle zh,
1438                                        Z_AttributesPlusTerm *zapt,
1439                                        const char *termz_org,
1440                                        oid_value attributeSet,
1441                                        NMEM stream,
1442                                        int reg_type, int complete_flag,
1443                                        const char *rank_type,
1444                                        const char *xpath_use,
1445                                        int num_bases, char **basenames,
1446                                        NMEM rset_nmem,
1447                                        RSET *rset,
1448                                        struct rset_key_control *kc)
1449 {
1450     RSET *result_sets = 0;
1451     int num_result_sets = 0;
1452     ZEBRA_RES res =
1453         term_list_trunc(zh, zapt, termz_org, attributeSet,
1454                         stream, reg_type, complete_flag,
1455                         rank_type, xpath_use,
1456                         num_bases, basenames,
1457                         rset_nmem,
1458                         &result_sets, &num_result_sets, kc);
1459     if (res != ZEBRA_OK)
1460         return res;
1461     if (num_result_sets == 0)
1462         *rset = rset_create_null(rset_nmem, kc, 0);
1463     else if (num_result_sets == 1)
1464         *rset = result_sets[0];
1465     else
1466         *rset = rset_create_prox(rset_nmem, kc, kc->scope,
1467                                  num_result_sets, result_sets,
1468                                  1 /* ordered */, 0 /* exclusion */,
1469                                  3 /* relation */, 1 /* distance */);
1470     if (!*rset)
1471         return ZEBRA_FAIL;
1472     return ZEBRA_OK;
1473 }
1474
1475 static ZEBRA_RES rpn_search_APT_or_list(ZebraHandle zh,
1476                                         Z_AttributesPlusTerm *zapt,
1477                                         const char *termz_org,
1478                                         oid_value attributeSet,
1479                                         NMEM stream,
1480                                         int reg_type, int complete_flag,
1481                                         const char *rank_type,
1482                                         const char *xpath_use,
1483                                         int num_bases, char **basenames,
1484                                         NMEM rset_nmem,
1485                                         RSET *rset,
1486                                         struct rset_key_control *kc)
1487 {
1488     RSET *result_sets = 0;
1489     int num_result_sets = 0;
1490     ZEBRA_RES res =
1491         term_list_trunc(zh, zapt, termz_org, attributeSet,
1492                         stream, reg_type, complete_flag,
1493                         rank_type, xpath_use,
1494                         num_bases, basenames,
1495                         rset_nmem,
1496                         &result_sets, &num_result_sets, kc);
1497     if (res != ZEBRA_OK)
1498         return res;
1499     if (num_result_sets == 0)
1500         *rset = rset_create_null(rset_nmem, kc, 0);
1501     else if (num_result_sets == 1)
1502         *rset = result_sets[0];
1503     else
1504         *rset = rset_create_or(rset_nmem, kc, kc->scope, 0 /* termid */,
1505                                num_result_sets, result_sets);
1506     if (!*rset)
1507         return ZEBRA_FAIL;
1508     return ZEBRA_OK;
1509 }
1510
1511 static ZEBRA_RES rpn_search_APT_and_list(ZebraHandle zh,
1512                                          Z_AttributesPlusTerm *zapt,
1513                                          const char *termz_org,
1514                                          oid_value attributeSet,
1515                                          NMEM stream,
1516                                          int reg_type, int complete_flag,
1517                                          const char *rank_type,
1518                                          const char *xpath_use,
1519                                          int num_bases, char **basenames,
1520                                          NMEM rset_nmem,
1521                                          RSET *rset,
1522                                          struct rset_key_control *kc)
1523 {
1524     RSET *result_sets = 0;
1525     int num_result_sets = 0;
1526     ZEBRA_RES res =
1527         term_list_trunc(zh, zapt, termz_org, attributeSet,
1528                         stream, reg_type, complete_flag,
1529                         rank_type, xpath_use,
1530                         num_bases, basenames,
1531                         rset_nmem,
1532                         &result_sets, &num_result_sets,
1533                         kc);
1534     if (res != ZEBRA_OK)
1535         return res;
1536     if (num_result_sets == 0)
1537         *rset = rset_create_null(rset_nmem, kc, 0);
1538     else if (num_result_sets == 1)
1539         *rset = result_sets[0];
1540     else
1541         *rset = rset_create_and(rset_nmem, kc, kc->scope,
1542                                 num_result_sets, result_sets);
1543     if (!*rset)
1544         return ZEBRA_FAIL;
1545     return ZEBRA_OK;
1546 }
1547
1548 static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1549                             const char **term_sub,
1550                             char *term_dict,
1551                             oid_value attributeSet,
1552                             struct grep_info *grep_info,
1553                             int *max_pos,
1554                             int reg_type,
1555                             char *term_dst,
1556                             int *error_code)
1557 {
1558     AttrType relation;
1559     int relation_value;
1560     int term_value;
1561     int r;
1562     char *term_tmp = term_dict + strlen(term_dict);
1563
1564     *error_code = 0;
1565     attr_init_APT(&relation, zapt, 2);
1566     relation_value = attr_find(&relation, NULL);
1567
1568     yaz_log(log_level_rpn, "numeric relation value=%d", relation_value);
1569
1570     switch (relation_value)
1571     {
1572     case 1:
1573         yaz_log(log_level_rpn, "Relation <");
1574         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1575                       term_dst))
1576             return 0;
1577         term_value = atoi (term_tmp);
1578         gen_regular_rel(term_tmp, term_value-1, 1);
1579         break;
1580     case 2:
1581         yaz_log(log_level_rpn, "Relation <=");
1582         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1583                       term_dst))
1584             return 0;
1585         term_value = atoi (term_tmp);
1586         gen_regular_rel(term_tmp, term_value, 1);
1587         break;
1588     case 4:
1589         yaz_log(log_level_rpn, "Relation >=");
1590         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1591                       term_dst))
1592             return 0;
1593         term_value = atoi (term_tmp);
1594         gen_regular_rel(term_tmp, term_value, 0);
1595         break;
1596     case 5:
1597         yaz_log(log_level_rpn, "Relation >");
1598         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1599                       term_dst))
1600             return 0;
1601         term_value = atoi (term_tmp);
1602         gen_regular_rel(term_tmp, term_value+1, 0);
1603         break;
1604     case -1:
1605     case 3:
1606         yaz_log(log_level_rpn, "Relation =");
1607         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1608                       term_dst))
1609             return 0;
1610         term_value = atoi (term_tmp);
1611         sprintf(term_tmp, "(0*%d)", term_value);
1612         break;
1613     case 103:
1614         /* term_tmp untouched.. */
1615         while (**term_sub != '\0')
1616             (*term_sub)++;
1617         break;
1618     default:
1619         *error_code = YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
1620         return 0;
1621     }
1622     yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_tmp);
1623     r = dict_lookup_grep(zh->reg->dict, term_dict, 0, grep_info, max_pos,
1624                           0, grep_handle);
1625     if (r)
1626         yaz_log(YLOG_WARN, "dict_lookup_grep fail, rel = gt: %d", r);
1627     yaz_log(log_level_rpn, "%d positions", grep_info->isam_p_indx);
1628     return 1;
1629 }
1630
1631 static ZEBRA_RES numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1632                               const char **term_sub,
1633                               oid_value attributeSet, NMEM stream,
1634                               struct grep_info *grep_info,
1635                               int reg_type, int complete_flag,
1636                               int num_bases, char **basenames,
1637                               char *term_dst,
1638                               const char *xpath_use,
1639                               struct ord_list **ol)
1640 {
1641     char term_dict[2*IT_MAX_WORD+2];
1642     int base_no;
1643     const char *termp;
1644     struct rpn_char_map_info rcmi;
1645
1646     int bases_ok = 0;     /* no of databases with OK attribute */
1647
1648     *ol = ord_list_create(stream);
1649
1650     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1651
1652     for (base_no = 0; base_no < num_bases; base_no++)
1653     {
1654         int max_pos, prefix_len = 0;
1655         int relation_error = 0;
1656         int ord, ord_len, i;
1657         char ord_buf[32];
1658
1659         termp = *term_sub;
1660
1661         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1662         {
1663             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1664                            basenames[base_no]);
1665             return ZEBRA_FAIL;
1666         }
1667
1668         if (zebra_apt_get_ord(zh, zapt, reg_type, xpath_use,
1669                               attributeSet, &ord) != ZEBRA_OK)
1670             continue;
1671         bases_ok++;
1672
1673         *ol = ord_list_append(stream, *ol, ord);
1674
1675         ord_len = key_SU_encode (ord, ord_buf);
1676
1677         term_dict[prefix_len++] = '(';
1678         for (i = 0; i < ord_len; i++)
1679         {
1680             term_dict[prefix_len++] = 1;
1681             term_dict[prefix_len++] = ord_buf[i];
1682         }
1683         term_dict[prefix_len++] = ')';
1684         term_dict[prefix_len] = '\0';
1685
1686         if (!numeric_relation(zh, zapt, &termp, term_dict,
1687                               attributeSet, grep_info, &max_pos, reg_type,
1688                               term_dst, &relation_error))
1689         {
1690             if (relation_error)
1691             {
1692                 zebra_setError(zh, relation_error, 0);
1693                 return ZEBRA_FAIL;
1694             }
1695             *term_sub = 0;
1696             return ZEBRA_OK;
1697         }
1698     }
1699     if (!bases_ok)
1700         return ZEBRA_FAIL;
1701     *term_sub = termp;
1702     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1703     return ZEBRA_OK;
1704 }
1705
1706
1707 static ZEBRA_RES rpn_search_APT_numeric(ZebraHandle zh,
1708                                         Z_AttributesPlusTerm *zapt,
1709                                         const char *termz,
1710                                         oid_value attributeSet,
1711                                         NMEM stream,
1712                                         int reg_type, int complete_flag,
1713                                         const char *rank_type,
1714                                         const char *xpath_use,
1715                                         int num_bases, char **basenames,
1716                                         NMEM rset_nmem,
1717                                         RSET *rset,
1718                                         struct rset_key_control *kc)
1719 {
1720     char term_dst[IT_MAX_WORD+1];
1721     const char *termp = termz;
1722     RSET *result_sets = 0;
1723     int num_result_sets = 0;
1724     ZEBRA_RES res;
1725     struct grep_info grep_info;
1726     int alloc_sets = 0;
1727     zint hits_limit_value;
1728     const char *term_ref_id_str = 0;
1729
1730     term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str, stream);
1731
1732     yaz_log(log_level_rpn, "APT_numeric t='%s'", termz);
1733     if (grep_info_prepare(zh, zapt, &grep_info, reg_type) == ZEBRA_FAIL)
1734         return ZEBRA_FAIL;
1735     while (1)
1736     {
1737         struct ord_list *ol;
1738         if (alloc_sets == num_result_sets)
1739         {
1740             int add = 10;
1741             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1742                                               sizeof(*rnew));
1743             if (alloc_sets)
1744                 memcpy(rnew, result_sets, alloc_sets * sizeof(*rnew));
1745             alloc_sets = alloc_sets + add;
1746             result_sets = rnew;
1747         }
1748         yaz_log(YLOG_DEBUG, "APT_numeric termp=%s", termp);
1749         grep_info.isam_p_indx = 0;
1750         res = numeric_term(zh, zapt, &termp, attributeSet, stream, &grep_info,
1751                            reg_type, complete_flag, num_bases, basenames,
1752                            term_dst, xpath_use, &ol);
1753         if (res == ZEBRA_FAIL || termp == 0)
1754             break;
1755         yaz_log(YLOG_DEBUG, "term: %s", term_dst);
1756         result_sets[num_result_sets] =
1757             rset_trunc(zh, grep_info.isam_p_buf,
1758                        grep_info.isam_p_indx, term_dst,
1759                        strlen(term_dst), rank_type,
1760                        0 /* preserve position */,
1761                        zapt->term->which, rset_nmem,
1762                        kc, kc->scope, ol, reg_type,
1763                        hits_limit_value,
1764                        term_ref_id_str);
1765         if (!result_sets[num_result_sets])
1766             break;
1767         num_result_sets++;
1768         if (!*termp)
1769             break;
1770     }
1771     grep_info_delete(&grep_info);
1772
1773     if (res != ZEBRA_OK)
1774         return res;
1775     if (num_result_sets == 0)
1776         *rset = rset_create_null(rset_nmem, kc, 0);
1777     else if (num_result_sets == 1)
1778         *rset = result_sets[0];
1779     else
1780         *rset = rset_create_and(rset_nmem, kc, kc->scope,
1781                                 num_result_sets, result_sets);
1782     if (!*rset)
1783         return ZEBRA_FAIL;
1784     return ZEBRA_OK;
1785 }
1786
1787 static ZEBRA_RES rpn_search_APT_local(ZebraHandle zh,
1788                                       Z_AttributesPlusTerm *zapt,
1789                                       const char *termz,
1790                                       oid_value attributeSet,
1791                                       NMEM stream,
1792                                       const char *rank_type, NMEM rset_nmem,
1793                                       RSET *rset,
1794                                       struct rset_key_control *kc)
1795 {
1796     RSFD rsfd;
1797     struct it_key key;
1798     int sys;
1799     *rset = rset_create_temp(rset_nmem, kc, kc->scope,
1800                              res_get (zh->res, "setTmpDir"),0 );
1801     rsfd = rset_open(*rset, RSETF_WRITE);
1802
1803     sys = atoi(termz);
1804     if (sys <= 0)
1805         sys = 1;
1806     key.mem[0] = sys;
1807     key.mem[1] = 1;
1808     key.len = 2;
1809     rset_write (rsfd, &key);
1810     rset_close (rsfd);
1811     return ZEBRA_OK;
1812 }
1813
1814 static ZEBRA_RES rpn_sort_spec(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1815                                oid_value attributeSet, NMEM stream,
1816                                Z_SortKeySpecList *sort_sequence,
1817                                const char *rank_type,
1818                                NMEM rset_nmem,
1819                                RSET *rset,
1820                                struct rset_key_control *kc)
1821 {
1822     int i;
1823     int sort_relation_value;
1824     AttrType sort_relation_type;
1825     Z_SortKeySpec *sks;
1826     Z_SortKey *sk;
1827     int oid[OID_SIZE];
1828     oident oe;
1829     char termz[20];
1830
1831     attr_init_APT(&sort_relation_type, zapt, 7);
1832     sort_relation_value = attr_find(&sort_relation_type, &attributeSet);
1833
1834     if (!sort_sequence->specs)
1835     {
1836         sort_sequence->num_specs = 10;
1837         sort_sequence->specs = (Z_SortKeySpec **)
1838             nmem_malloc(stream, sort_sequence->num_specs *
1839                          sizeof(*sort_sequence->specs));
1840         for (i = 0; i<sort_sequence->num_specs; i++)
1841             sort_sequence->specs[i] = 0;
1842     }
1843     if (zapt->term->which != Z_Term_general)
1844         i = 0;
1845     else
1846         i = atoi_n ((char *) zapt->term->u.general->buf,
1847                     zapt->term->u.general->len);
1848     if (i >= sort_sequence->num_specs)
1849         i = 0;
1850     sprintf(termz, "%d", i);
1851
1852     oe.proto = PROTO_Z3950;
1853     oe.oclass = CLASS_ATTSET;
1854     oe.value = attributeSet;
1855     if (!oid_ent_to_oid (&oe, oid))
1856         return ZEBRA_FAIL;
1857
1858     sks = (Z_SortKeySpec *) nmem_malloc(stream, sizeof(*sks));
1859     sks->sortElement = (Z_SortElement *)
1860         nmem_malloc(stream, sizeof(*sks->sortElement));
1861     sks->sortElement->which = Z_SortElement_generic;
1862     sk = sks->sortElement->u.generic = (Z_SortKey *)
1863         nmem_malloc(stream, sizeof(*sk));
1864     sk->which = Z_SortKey_sortAttributes;
1865     sk->u.sortAttributes = (Z_SortAttributes *)
1866         nmem_malloc(stream, sizeof(*sk->u.sortAttributes));
1867
1868     sk->u.sortAttributes->id = oid;
1869     sk->u.sortAttributes->list = zapt->attributes;
1870
1871     sks->sortRelation = (int *)
1872         nmem_malloc(stream, sizeof(*sks->sortRelation));
1873     if (sort_relation_value == 1)
1874         *sks->sortRelation = Z_SortKeySpec_ascending;
1875     else if (sort_relation_value == 2)
1876         *sks->sortRelation = Z_SortKeySpec_descending;
1877     else
1878         *sks->sortRelation = Z_SortKeySpec_ascending;
1879
1880     sks->caseSensitivity = (int *)
1881         nmem_malloc(stream, sizeof(*sks->caseSensitivity));
1882     *sks->caseSensitivity = 0;
1883
1884     sks->which = Z_SortKeySpec_null;
1885     sks->u.null = odr_nullval ();
1886     sort_sequence->specs[i] = sks;
1887     *rset = rset_create_null(rset_nmem, kc, 0);
1888     return ZEBRA_OK;
1889 }
1890
1891
1892 static int rpn_check_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1893                            oid_value attributeSet,
1894                            struct xpath_location_step *xpath, int max,
1895                            NMEM mem)
1896 {
1897     oid_value curAttributeSet = attributeSet;
1898     AttrType use;
1899     const char *use_string = 0;
1900
1901     attr_init_APT(&use, zapt, 1);
1902     attr_find_ex(&use, &curAttributeSet, &use_string);
1903
1904     if (!use_string || *use_string != '/')
1905         return -1;
1906
1907     return zebra_parse_xpath_str(use_string, xpath, max, mem);
1908 }
1909
1910
1911
1912 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
1913                         int reg_type, const char *term,
1914                         const char *xpath_use,
1915                         NMEM rset_nmem,
1916                         struct rset_key_control *kc)
1917 {
1918     RSET rset;
1919     struct grep_info grep_info;
1920     char term_dict[2048];
1921     char ord_buf[32];
1922     int prefix_len = 0;
1923     int ord = zebraExplain_lookup_attr_str(zh->reg->zei,
1924                                            zinfo_index_category_index,
1925                                            reg_type,
1926                                            xpath_use);
1927     int ord_len, i, r, max_pos;
1928     int term_type = Z_Term_characterString;
1929     const char *flags = "void";
1930
1931     if (grep_info_prepare(zh, 0 /* zapt */, &grep_info, '0') == ZEBRA_FAIL)
1932         return rset_create_null(rset_nmem, kc, 0);
1933
1934     if (ord < 0)
1935         return rset_create_null(rset_nmem, kc, 0);
1936     if (prefix_len)
1937         term_dict[prefix_len++] = '|';
1938     else
1939         term_dict[prefix_len++] = '(';
1940
1941     ord_len = key_SU_encode (ord, ord_buf);
1942     for (i = 0; i<ord_len; i++)
1943     {
1944         term_dict[prefix_len++] = 1;
1945         term_dict[prefix_len++] = ord_buf[i];
1946     }
1947     term_dict[prefix_len++] = ')';
1948     strcpy(term_dict+prefix_len, term);
1949
1950     grep_info.isam_p_indx = 0;
1951     r = dict_lookup_grep(zh->reg->dict, term_dict, 0,
1952                           &grep_info, &max_pos, 0, grep_handle);
1953     yaz_log(YLOG_DEBUG, "%s %d positions", term,
1954              grep_info.isam_p_indx);
1955     rset = rset_trunc(zh, grep_info.isam_p_buf,
1956                       grep_info.isam_p_indx, term, strlen(term),
1957                       flags, 1, term_type,rset_nmem,
1958                       kc, kc->scope, 0, reg_type, 0 /* hits_limit */,
1959                       0 /* term_ref_id_str */);
1960     grep_info_delete(&grep_info);
1961     return rset;
1962 }
1963
1964 static
1965 ZEBRA_RES rpn_search_xpath(ZebraHandle zh,
1966                            int num_bases, char **basenames,
1967                            NMEM stream, const char *rank_type, RSET rset,
1968                            int xpath_len, struct xpath_location_step *xpath,
1969                            NMEM rset_nmem,
1970                            RSET *rset_out,
1971                            struct rset_key_control *kc)
1972 {
1973     int base_no;
1974     int i;
1975     int always_matches = rset ? 0 : 1;
1976
1977     if (xpath_len < 0)
1978     {
1979         *rset_out = rset;
1980         return ZEBRA_OK;
1981     }
1982
1983     yaz_log(YLOG_DEBUG, "xpath len=%d", xpath_len);
1984     for (i = 0; i<xpath_len; i++)
1985     {
1986         yaz_log(log_level_rpn, "XPATH %d %s", i, xpath[i].part);
1987
1988     }
1989
1990     /*
1991       //a    ->    a/.*
1992       //a/b  ->    b/a/.*
1993       /a     ->    a/
1994       /a/b   ->    b/a/
1995
1996       /      ->    none
1997
1998    a[@attr = value]/b[@other = othervalue]
1999
2000  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
2001  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
2002  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
2003  /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y)
2004  /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y)
2005  /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x)
2006
2007     */
2008
2009     dict_grep_cmap (zh->reg->dict, 0, 0);
2010
2011     for (base_no = 0; base_no < num_bases; base_no++)
2012     {
2013         int level = xpath_len;
2014         int first_path = 1;
2015
2016         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2017         {
2018             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
2019                            basenames[base_no]);
2020             *rset_out = rset;
2021             return ZEBRA_FAIL;
2022         }
2023         while (--level >= 0)
2024         {
2025             WRBUF xpath_rev = wrbuf_alloc();
2026             int i;
2027             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
2028
2029             for (i = level; i >= 1; --i)
2030             {
2031                 const char *cp = xpath[i].part;
2032                 if (*cp)
2033                 {
2034                     for (; *cp; cp++)
2035                     {
2036                         if (*cp == '*')
2037                             wrbuf_puts(xpath_rev, "[^/]*");
2038                         else if (*cp == ' ')
2039                             wrbuf_puts(xpath_rev, "\001 ");
2040                         else
2041                             wrbuf_putc(xpath_rev, *cp);
2042
2043                         /* wrbuf_putc does not null-terminate , but
2044                            wrbuf_puts below ensures it does.. so xpath_rev
2045                            is OK iff length is > 0 */
2046                     }
2047                     wrbuf_puts(xpath_rev, "/");
2048                 }
2049                 else if (i == 1)  /* // case */
2050                     wrbuf_puts(xpath_rev, ".*");
2051             }
2052             if (xpath[level].predicate &&
2053                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
2054                 xpath[level].predicate->u.relation.name[0])
2055             {
2056                 WRBUF wbuf = wrbuf_alloc();
2057                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
2058                 if (xpath[level].predicate->u.relation.value)
2059                 {
2060                     const char *cp = xpath[level].predicate->u.relation.value;
2061                     wrbuf_putc(wbuf, '=');
2062
2063                     while (*cp)
2064                     {
2065                         if (strchr(REGEX_CHARS, *cp))
2066                             wrbuf_putc(wbuf, '\\');
2067                         wrbuf_putc(wbuf, *cp);
2068                         cp++;
2069                     }
2070                 }
2071                 wrbuf_puts(wbuf, "");
2072                 rset_attr = xpath_trunc(
2073                     zh, stream, '0', wrbuf_buf(wbuf), ZEBRA_XPATH_ATTR_NAME,
2074                     rset_nmem, kc);
2075                 wrbuf_free(wbuf, 1);
2076             }
2077             else
2078             {
2079                 if (!first_path)
2080                 {
2081                     wrbuf_free(xpath_rev, 1);
2082                     continue;
2083                 }
2084             }
2085             yaz_log(log_level_rpn, "xpath_rev (%d) = %.*s", level,
2086                     wrbuf_len(xpath_rev), wrbuf_buf(xpath_rev));
2087             if (wrbuf_len(xpath_rev))
2088             {
2089                 rset_start_tag = xpath_trunc(zh, stream, '0',
2090                                              wrbuf_buf(xpath_rev),
2091                                              ZEBRA_XPATH_ELM_BEGIN,
2092                                              rset_nmem, kc);
2093                 if (always_matches)
2094                     rset = rset_start_tag;
2095                 else
2096                 {
2097                     rset_end_tag = xpath_trunc(zh, stream, '0',
2098                                                wrbuf_buf(xpath_rev),
2099                                                ZEBRA_XPATH_ELM_END,
2100                                                rset_nmem, kc);
2101
2102                     rset = rset_create_between(rset_nmem, kc, kc->scope,
2103                                                rset_start_tag, rset,
2104                                                rset_end_tag, rset_attr);
2105                 }
2106             }
2107             wrbuf_free(xpath_rev, 1);
2108             first_path = 0;
2109         }
2110     }
2111     *rset_out = rset;
2112     return ZEBRA_OK;
2113 }
2114
2115 #define MAX_XPATH_STEPS 10
2116
2117 static ZEBRA_RES rpn_search_APT(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2118                                 oid_value attributeSet, NMEM stream,
2119                                 Z_SortKeySpecList *sort_sequence,
2120                                 int num_bases, char **basenames,
2121                                 NMEM rset_nmem,
2122                                 RSET *rset,
2123                                 struct rset_key_control *kc)
2124 {
2125     ZEBRA_RES res = ZEBRA_OK;
2126     unsigned reg_id;
2127     char *search_type = NULL;
2128     char rank_type[128];
2129     int complete_flag;
2130     int sort_flag;
2131     char termz[IT_MAX_WORD+1];
2132     int xpath_len;
2133     const char *xpath_use = 0;
2134     struct xpath_location_step xpath[MAX_XPATH_STEPS];
2135
2136     if (!log_level_set)
2137     {
2138         log_level_rpn = yaz_log_module_level("rpn");
2139         log_level_set = 1;
2140     }
2141     zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2142                     rank_type, &complete_flag, &sort_flag);
2143
2144     yaz_log(YLOG_DEBUG, "reg_id=%c", reg_id);
2145     yaz_log(YLOG_DEBUG, "complete_flag=%d", complete_flag);
2146     yaz_log(YLOG_DEBUG, "search_type=%s", search_type);
2147     yaz_log(YLOG_DEBUG, "rank_type=%s", rank_type);
2148
2149     if (zapt_term_to_utf8(zh, zapt, termz) == ZEBRA_FAIL)
2150         return ZEBRA_FAIL;
2151
2152     if (sort_flag)
2153         return rpn_sort_spec(zh, zapt, attributeSet, stream, sort_sequence,
2154                              rank_type, rset_nmem, rset, kc);
2155     /* consider if an X-Path query is used */
2156     xpath_len = rpn_check_xpath(zh, zapt, attributeSet,
2157                                 xpath, MAX_XPATH_STEPS, stream);
2158     if (xpath_len >= 0)
2159     {
2160         if (xpath[xpath_len-1].part[0] == '@')
2161             xpath_use = ZEBRA_XPATH_ATTR_CDATA;  /* last step is attribute  */
2162         else
2163             xpath_use = ZEBRA_XPATH_CDATA;  /* searching for cdata */
2164
2165         if (1)
2166         {
2167             AttrType relation;
2168             int relation_value;
2169
2170             attr_init_APT(&relation, zapt, 2);
2171             relation_value = attr_find(&relation, NULL);
2172
2173             if (relation_value == 103) /* alwaysmatches */
2174             {
2175                 *rset = 0; /* signal no "term" set */
2176                 return rpn_search_xpath(zh, num_bases, basenames,
2177                                         stream, rank_type, *rset,
2178                                         xpath_len, xpath, rset_nmem, rset, kc);
2179             }
2180         }
2181     }
2182
2183     /* search using one of the various search type strategies
2184        termz is our UTF-8 search term
2185        attributeSet is top-level default attribute set
2186        stream is ODR for search
2187        reg_id is the register type
2188        complete_flag is 1 for complete subfield, 0 for incomplete
2189        xpath_use is use-attribute to be used for X-Path search, 0 for none
2190     */
2191     if (!strcmp(search_type, "phrase"))
2192     {
2193         res = rpn_search_APT_phrase(zh, zapt, termz, attributeSet, stream,
2194                                     reg_id, complete_flag, rank_type,
2195                                     xpath_use,
2196                                     num_bases, basenames, rset_nmem,
2197                                     rset, kc);
2198     }
2199     else if (!strcmp(search_type, "and-list"))
2200     {
2201         res = rpn_search_APT_and_list(zh, zapt, termz, attributeSet, stream,
2202                                       reg_id, complete_flag, rank_type,
2203                                       xpath_use,
2204                                       num_bases, basenames, rset_nmem,
2205                                       rset, kc);
2206     }
2207     else if (!strcmp(search_type, "or-list"))
2208     {
2209         res = rpn_search_APT_or_list(zh, zapt, termz, attributeSet, stream,
2210                                      reg_id, complete_flag, rank_type,
2211                                      xpath_use,
2212                                      num_bases, basenames, rset_nmem,
2213                                      rset, kc);
2214     }
2215     else if (!strcmp(search_type, "local"))
2216     {
2217         res = rpn_search_APT_local(zh, zapt, termz, attributeSet, stream,
2218                                    rank_type, rset_nmem, rset, kc);
2219     }
2220     else if (!strcmp(search_type, "numeric"))
2221     {
2222         res = rpn_search_APT_numeric(zh, zapt, termz, attributeSet, stream,
2223                                      reg_id, complete_flag, rank_type,
2224                                      xpath_use,
2225                                      num_bases, basenames, rset_nmem,
2226                                      rset, kc);
2227     }
2228     else
2229     {
2230         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2231         res = ZEBRA_FAIL;
2232     }
2233     if (res != ZEBRA_OK)
2234         return res;
2235     if (!*rset)
2236         return ZEBRA_FAIL;
2237     return rpn_search_xpath(zh, num_bases, basenames,
2238                             stream, rank_type, *rset,
2239                             xpath_len, xpath, rset_nmem, rset, kc);
2240 }
2241
2242 static ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2243                                       oid_value attributeSet,
2244                                       NMEM stream, NMEM rset_nmem,
2245                                       Z_SortKeySpecList *sort_sequence,
2246                                       int num_bases, char **basenames,
2247                                       RSET **result_sets, int *num_result_sets,
2248                                       Z_Operator *parent_op,
2249                                       struct rset_key_control *kc);
2250
2251 ZEBRA_RES rpn_search_top(ZebraHandle zh, Z_RPNStructure *zs,
2252                          oid_value attributeSet,
2253                          NMEM stream, NMEM rset_nmem,
2254                          Z_SortKeySpecList *sort_sequence,
2255                          int num_bases, char **basenames,
2256                          RSET *result_set)
2257 {
2258     RSET *result_sets = 0;
2259     int num_result_sets = 0;
2260     ZEBRA_RES res;
2261     struct rset_key_control *kc = zebra_key_control_create(zh);
2262
2263     res = rpn_search_structure(zh, zs, attributeSet,
2264                                stream, rset_nmem,
2265                                sort_sequence,
2266                                num_bases, basenames,
2267                                &result_sets, &num_result_sets,
2268                                0 /* no parent op */,
2269                                kc);
2270     if (res != ZEBRA_OK)
2271     {
2272         int i;
2273         for (i = 0; i<num_result_sets; i++)
2274             rset_delete(result_sets[i]);
2275         *result_set = 0;
2276     }
2277     else
2278     {
2279         assert(num_result_sets == 1);
2280         assert(result_sets);
2281         assert(*result_sets);
2282         *result_set = *result_sets;
2283     }
2284     (*kc->dec)(kc);
2285     return res;
2286 }
2287
2288 ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2289                                oid_value attributeSet,
2290                                NMEM stream, NMEM rset_nmem,
2291                                Z_SortKeySpecList *sort_sequence,
2292                                int num_bases, char **basenames,
2293                                RSET **result_sets, int *num_result_sets,
2294                                Z_Operator *parent_op,
2295                                struct rset_key_control *kc)
2296 {
2297     *num_result_sets = 0;
2298     if (zs->which == Z_RPNStructure_complex)
2299     {
2300         ZEBRA_RES res;
2301         Z_Operator *zop = zs->u.complex->roperator;
2302         RSET *result_sets_l = 0;
2303         int num_result_sets_l = 0;
2304         RSET *result_sets_r = 0;
2305         int num_result_sets_r = 0;
2306
2307         res = rpn_search_structure(zh, zs->u.complex->s1,
2308                                    attributeSet, stream, rset_nmem,
2309                                    sort_sequence,
2310                                    num_bases, basenames,
2311                                    &result_sets_l, &num_result_sets_l,
2312                                    zop, kc);
2313         if (res != ZEBRA_OK)
2314         {
2315             int i;
2316             for (i = 0; i<num_result_sets_l; i++)
2317                 rset_delete(result_sets_l[i]);
2318             return res;
2319         }
2320         res = rpn_search_structure(zh, zs->u.complex->s2,
2321                                    attributeSet, stream, rset_nmem,
2322                                    sort_sequence,
2323                                    num_bases, basenames,
2324                                    &result_sets_r, &num_result_sets_r,
2325                                    zop, kc);
2326         if (res != ZEBRA_OK)
2327         {
2328             int i;
2329             for (i = 0; i<num_result_sets_l; i++)
2330                 rset_delete(result_sets_l[i]);
2331             for (i = 0; i<num_result_sets_r; i++)
2332                 rset_delete(result_sets_r[i]);
2333             return res;
2334         }
2335
2336         /* make a new list of result for all children */
2337         *num_result_sets = num_result_sets_l + num_result_sets_r;
2338         *result_sets = nmem_malloc(stream, *num_result_sets *
2339                                    sizeof(**result_sets));
2340         memcpy(*result_sets, result_sets_l,
2341                num_result_sets_l * sizeof(**result_sets));
2342         memcpy(*result_sets + num_result_sets_l, result_sets_r,
2343                num_result_sets_r * sizeof(**result_sets));
2344
2345         if (!parent_op || parent_op->which != zop->which
2346             || (zop->which != Z_Operator_and &&
2347                 zop->which != Z_Operator_or))
2348         {
2349             /* parent node different from this one (or non-present) */
2350             /* we must combine result sets now */
2351             RSET rset;
2352             switch (zop->which)
2353             {
2354             case Z_Operator_and:
2355                 rset = rset_create_and(rset_nmem, kc,
2356                                        kc->scope,
2357                                        *num_result_sets, *result_sets);
2358                 break;
2359             case Z_Operator_or:
2360                 rset = rset_create_or(rset_nmem, kc,
2361                                       kc->scope, 0, /* termid */
2362                                       *num_result_sets, *result_sets);
2363                 break;
2364             case Z_Operator_and_not:
2365                 rset = rset_create_not(rset_nmem, kc,
2366                                        kc->scope,
2367                                        (*result_sets)[0],
2368                                        (*result_sets)[1]);
2369                 break;
2370             case Z_Operator_prox:
2371                 if (zop->u.prox->which != Z_ProximityOperator_known)
2372                 {
2373                     zebra_setError(zh,
2374                                    YAZ_BIB1_UNSUPP_PROX_UNIT_CODE,
2375                                    0);
2376                     return ZEBRA_FAIL;
2377                 }
2378                 if (*zop->u.prox->u.known != Z_ProxUnit_word)
2379                 {
2380                     zebra_setError_zint(zh,
2381                                         YAZ_BIB1_UNSUPP_PROX_UNIT_CODE,
2382                                         *zop->u.prox->u.known);
2383                     return ZEBRA_FAIL;
2384                 }
2385                 else
2386                 {
2387                     rset = rset_create_prox(rset_nmem, kc,
2388                                             kc->scope,
2389                                             *num_result_sets, *result_sets,
2390                                             *zop->u.prox->ordered,
2391                                             (!zop->u.prox->exclusion ?
2392                                              0 : *zop->u.prox->exclusion),
2393                                             *zop->u.prox->relationType,
2394                                             *zop->u.prox->distance );
2395                 }
2396                 break;
2397             default:
2398                 zebra_setError(zh, YAZ_BIB1_OPERATOR_UNSUPP, 0);
2399                 return ZEBRA_FAIL;
2400             }
2401             *num_result_sets = 1;
2402             *result_sets = nmem_malloc(stream, *num_result_sets *
2403                                        sizeof(**result_sets));
2404             (*result_sets)[0] = rset;
2405         }
2406     }
2407     else if (zs->which == Z_RPNStructure_simple)
2408     {
2409         RSET rset;
2410         ZEBRA_RES res;
2411
2412         if (zs->u.simple->which == Z_Operand_APT)
2413         {
2414             yaz_log(YLOG_DEBUG, "rpn_search_APT");
2415             res = rpn_search_APT(zh, zs->u.simple->u.attributesPlusTerm,
2416                                  attributeSet, stream, sort_sequence,
2417                                  num_bases, basenames, rset_nmem, &rset,
2418                                  kc);
2419             if (res != ZEBRA_OK)
2420                 return res;
2421         }
2422         else if (zs->u.simple->which == Z_Operand_resultSetId)
2423         {
2424             yaz_log(YLOG_DEBUG, "rpn_search_ref");
2425             rset = resultSetRef(zh, zs->u.simple->u.resultSetId);
2426             if (!rset)
2427             {
2428                 zebra_setError(zh,
2429                                YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
2430                                zs->u.simple->u.resultSetId);
2431                 return ZEBRA_FAIL;
2432             }
2433             rset_dup(rset);
2434         }
2435         else
2436         {
2437             zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2438             return ZEBRA_FAIL;
2439         }
2440         *num_result_sets = 1;
2441         *result_sets = nmem_malloc(stream, *num_result_sets *
2442                                    sizeof(**result_sets));
2443         (*result_sets)[0] = rset;
2444     }
2445     else
2446     {
2447         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2448         return ZEBRA_FAIL;
2449     }
2450     return ZEBRA_OK;
2451 }
2452
2453 struct scan_info_entry {
2454     char *term;
2455     ISAM_P isam_p;
2456 };
2457
2458 struct scan_info {
2459     struct scan_info_entry *list;
2460     ODR odr;
2461     int before, after;
2462     char prefix[20];
2463 };
2464
2465 static int scan_handle (char *name, const char *info, int pos, void *client)
2466 {
2467     int len_prefix, idx;
2468     struct scan_info *scan_info = (struct scan_info *) client;
2469
2470     len_prefix = strlen(scan_info->prefix);
2471     if (memcmp (name, scan_info->prefix, len_prefix))
2472         return 1;
2473     if (pos > 0)
2474         idx = scan_info->after - pos + scan_info->before;
2475     else
2476         idx = - pos - 1;
2477
2478     if (idx < 0)
2479         return 0;
2480     scan_info->list[idx].term = (char *)
2481         odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
2482     strcpy(scan_info->list[idx].term, name + len_prefix);
2483     assert (*info == sizeof(ISAM_P));
2484     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
2485     return 0;
2486 }
2487
2488 void zebra_term_untrans_iconv(ZebraHandle zh, NMEM stream, int reg_type,
2489                               char **dst, const char *src)
2490 {
2491     char term_src[IT_MAX_WORD];
2492     char term_dst[IT_MAX_WORD];
2493
2494     zebra_term_untrans (zh, reg_type, term_src, src);
2495
2496     if (zh->iconv_from_utf8 != 0)
2497     {
2498         int len;
2499         char *inbuf = term_src;
2500         size_t inleft = strlen(term_src);
2501         char *outbuf = term_dst;
2502         size_t outleft = sizeof(term_dst)-1;
2503         size_t ret;
2504
2505         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2506                          &outbuf, &outleft);
2507         if (ret == (size_t)(-1))
2508             len = 0;
2509         else
2510             len = outbuf - term_dst;
2511         *dst = nmem_malloc(stream, len + 1);
2512         if (len > 0)
2513             memcpy (*dst, term_dst, len);
2514         (*dst)[len] = '\0';
2515     }
2516     else
2517         *dst = nmem_strdup(stream, term_src);
2518 }
2519
2520 static void count_set(ZebraHandle zh, RSET rset, zint *count)
2521 {
2522     zint psysno = 0;
2523     struct it_key key;
2524     RSFD rfd;
2525
2526     yaz_log(YLOG_DEBUG, "count_set");
2527
2528     rset->hits_limit = zh->approx_limit;
2529
2530     *count = 0;
2531     rfd = rset_open(rset, RSETF_READ);
2532     while (rset_read(rfd, &key,0 /* never mind terms */))
2533     {
2534         if (key.mem[0] != psysno)
2535         {
2536             psysno = key.mem[0];
2537             if (rfd->counted_items >= rset->hits_limit)
2538                 break;
2539         }
2540     }
2541     rset_close (rfd);
2542     *count = rset->hits_count;
2543 }
2544
2545 #define RPN_MAX_ORDS 32
2546
2547 ZEBRA_RES rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2548                    oid_value attributeset,
2549                    int num_bases, char **basenames,
2550                    int *position, int *num_entries, ZebraScanEntry **list,
2551                    int *is_partial, RSET limit_set, int return_zero)
2552 {
2553     int i;
2554     int pos = *position;
2555     int num = *num_entries;
2556     int before;
2557     int after;
2558     int base_no;
2559     char termz[IT_MAX_WORD+20];
2560     struct scan_info *scan_info_array;
2561     ZebraScanEntry *glist;
2562     int ords[RPN_MAX_ORDS], ord_no = 0;
2563     int ptr[RPN_MAX_ORDS];
2564
2565     unsigned index_type;
2566     char *search_type = NULL;
2567     char rank_type[128];
2568     int complete_flag;
2569     int sort_flag;
2570     NMEM rset_nmem = NULL;
2571     struct rset_key_control *kc = 0;
2572
2573     *list = 0;
2574     *is_partial = 0;
2575
2576     if (attributeset == VAL_NONE)
2577         attributeset = VAL_BIB1;
2578
2579     if (!limit_set)
2580     {
2581         AttrType termset;
2582         int termset_value_numeric;
2583         const char *termset_value_string;
2584         attr_init_APT(&termset, zapt, 8);
2585         termset_value_numeric =
2586             attr_find_ex(&termset, NULL, &termset_value_string);
2587         if (termset_value_numeric != -1)
2588         {
2589             char resname[32];
2590             const char *termset_name = 0;
2591
2592             if (termset_value_numeric != -2)
2593             {
2594
2595                 sprintf(resname, "%d", termset_value_numeric);
2596                 termset_name = resname;
2597             }
2598             else
2599                 termset_name = termset_value_string;
2600
2601             limit_set = resultSetRef (zh, termset_name);
2602         }
2603     }
2604
2605     yaz_log(YLOG_DEBUG, "position = %d, num = %d set=%d",
2606             pos, num, attributeset);
2607
2608     if (zebra_maps_attr(zh->reg->zebra_maps, zapt, &index_type, &search_type,
2609                         rank_type, &complete_flag, &sort_flag))
2610     {
2611         *num_entries = 0;
2612         zebra_setError(zh, YAZ_BIB1_UNSUPP_ATTRIBUTE_TYPE, 0);
2613         return ZEBRA_FAIL;
2614     }
2615     for (base_no = 0; base_no < num_bases && ord_no < RPN_MAX_ORDS; base_no++)
2616     {
2617         int ord;
2618
2619         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2620         {
2621             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
2622                            basenames[base_no]);
2623             *num_entries = 0;
2624             return ZEBRA_FAIL;
2625         }
2626         if (zebra_apt_get_ord(zh, zapt, index_type, 0, attributeset, &ord)
2627             != ZEBRA_OK)
2628             continue;
2629         ords[ord_no++] = ord;
2630     }
2631     if (ord_no == 0)
2632     {
2633         *num_entries = 0;
2634         return ZEBRA_OK;
2635     }
2636     /* prepare dictionary scanning */
2637     if (num < 1)
2638     {
2639         *num_entries = 0;
2640         return ZEBRA_OK;
2641     }
2642     before = pos-1;
2643     if (before < 0)
2644         before = 0;
2645     after = 1+num-pos;
2646     if (after < 0)
2647         after = 0;
2648     yaz_log(YLOG_DEBUG, "rpn_scan pos=%d num=%d before=%d "
2649             "after=%d before+after=%d",
2650             pos, num, before, after, before+after);
2651     scan_info_array = (struct scan_info *)
2652         odr_malloc(stream, ord_no * sizeof(*scan_info_array));
2653     for (i = 0; i < ord_no; i++)
2654     {
2655         int j, prefix_len = 0;
2656         int before_tmp = before, after_tmp = after;
2657         struct scan_info *scan_info = scan_info_array + i;
2658         struct rpn_char_map_info rcmi;
2659
2660         rpn_char_map_prepare (zh->reg, index_type, &rcmi);
2661
2662         scan_info->before = before;
2663         scan_info->after = after;
2664         scan_info->odr = stream;
2665
2666         scan_info->list = (struct scan_info_entry *)
2667             odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
2668         for (j = 0; j<before+after; j++)
2669             scan_info->list[j].term = NULL;
2670
2671         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2672         termz[prefix_len] = 0;
2673         strcpy(scan_info->prefix, termz);
2674
2675         if (trans_scan_term(zh, zapt, termz+prefix_len, index_type) ==
2676             ZEBRA_FAIL)
2677             return ZEBRA_FAIL;
2678
2679         dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
2680                   scan_info, scan_handle);
2681     }
2682     glist = (ZebraScanEntry *)
2683         odr_malloc(stream, (before+after)*sizeof(*glist));
2684
2685     rset_nmem = nmem_create();
2686     kc = zebra_key_control_create(zh);
2687
2688     /* consider terms after main term */
2689     for (i = 0; i < ord_no; i++)
2690         ptr[i] = before;
2691
2692     *is_partial = 0;
2693     for (i = 0; i<after; i++)
2694     {
2695         int j, j0 = -1;
2696         const char *mterm = NULL;
2697         const char *tst;
2698         RSET rset = 0;
2699         int lo = i + pos-1; /* offset in result list */
2700
2701         /* find: j0 is the first of the minimal values */
2702         for (j = 0; j < ord_no; j++)
2703         {
2704             if (ptr[j] < before+after && ptr[j] >= 0 &&
2705                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2706                 (!mterm || strcmp (tst, mterm) < 0))
2707             {
2708                 j0 = j;
2709                 mterm = tst;
2710             }
2711         }
2712         if (j0 == -1)
2713             break;  /* no value found, stop */
2714
2715         /* get result set for first one , but only if it's within bounds */
2716         if (lo >= 0)
2717         {
2718             /* get result set for first term */
2719             zebra_term_untrans_iconv(zh, stream->mem, index_type,
2720                                      &glist[lo].term, mterm);
2721             rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2722                               glist[lo].term, strlen(glist[lo].term),
2723                               NULL, 0, zapt->term->which, rset_nmem,
2724                               kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2725                               0 /* term_ref_id_str */);
2726         }
2727         ptr[j0]++; /* move index for this set .. */
2728         /* get result set for remaining scan terms */
2729         for (j = j0+1; j<ord_no; j++)
2730         {
2731             if (ptr[j] < before+after && ptr[j] >= 0 &&
2732                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2733                 !strcmp (tst, mterm))
2734             {
2735                 if (lo >= 0)
2736                 {
2737                     RSET rsets[2];
2738
2739                     rsets[0] = rset;
2740                     rsets[1] =
2741                         rset_trunc(
2742                             zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
2743                             glist[lo].term,
2744                             strlen(glist[lo].term), NULL, 0,
2745                             zapt->term->which,rset_nmem,
2746                             kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2747                             0 /* term_ref_id_str */ );
2748                     rset = rset_create_or(rset_nmem, kc,
2749                                           kc->scope, 0 /* termid */,
2750                                           2, rsets);
2751                 }
2752                 ptr[j]++;
2753             }
2754         }
2755         if (lo >= 0)
2756         {
2757             zint count;
2758             /* merge with limit_set if given */
2759             if (limit_set)
2760             {
2761                 RSET rsets[2];
2762                 rsets[0] = rset;
2763                 rsets[1] = rset_dup(limit_set);
2764
2765                 rset = rset_create_and(rset_nmem, kc, kc->scope, 2, rsets);
2766             }
2767             /* count it */
2768             count_set(zh, rset, &count);
2769             glist[lo].occurrences = count;
2770             rset_delete(rset);
2771         }
2772     }
2773     if (i < after)
2774     {
2775         *num_entries -= (after-i);
2776         *is_partial = 1;
2777         if (*num_entries < 0)
2778         {
2779             (*kc->dec)(kc);
2780             nmem_destroy(rset_nmem);
2781             *num_entries = 0;
2782             return ZEBRA_OK;
2783         }
2784     }
2785     /* consider terms before main term */
2786     for (i = 0; i<ord_no; i++)
2787         ptr[i] = 0;
2788
2789     for (i = 0; i<before; i++)
2790     {
2791         int j, j0 = -1;
2792         const char *mterm = NULL;
2793         const char *tst;
2794         RSET rset;
2795         int lo = before-1-i; /* offset in result list */
2796         zint count;
2797
2798         for (j = 0; j <ord_no; j++)
2799         {
2800             if (ptr[j] < before && ptr[j] >= 0 &&
2801                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2802                 (!mterm || strcmp (tst, mterm) > 0))
2803             {
2804                 j0 = j;
2805                     mterm = tst;
2806             }
2807         }
2808         if (j0 == -1)
2809             break;
2810
2811         zebra_term_untrans_iconv(zh, stream->mem, index_type,
2812                                  &glist[lo].term, mterm);
2813
2814         rset = rset_trunc
2815             (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
2816              glist[lo].term, strlen(glist[lo].term),
2817              NULL, 0, zapt->term->which, rset_nmem,
2818              kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2819              0 /* term_ref_id_str */);
2820
2821         ptr[j0]++;
2822
2823         for (j = j0+1; j<ord_no; j++)
2824         {
2825             if (ptr[j] < before && ptr[j] >= 0 &&
2826                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2827                 !strcmp (tst, mterm))
2828             {
2829                 RSET rsets[2];
2830
2831                 rsets[0] = rset;
2832                 rsets[1] = rset_trunc(
2833                     zh,
2834                     &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
2835                     glist[lo].term,
2836                     strlen(glist[lo].term), NULL, 0,
2837                     zapt->term->which, rset_nmem,
2838                     kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2839                     0 /* term_ref_id_str */);
2840                 rset = rset_create_or(rset_nmem, kc,
2841                                       kc->scope, 0 /* termid */, 2, rsets);
2842
2843                 ptr[j]++;
2844             }
2845         }
2846         if (limit_set)
2847         {
2848             RSET rsets[2];
2849             rsets[0] = rset;
2850             rsets[1] = rset_dup(limit_set);
2851
2852             rset = rset_create_and(rset_nmem, kc, kc->scope, 2, rsets);
2853         }
2854         count_set(zh, rset, &count);
2855         glist[lo].occurrences = count;
2856         rset_delete (rset);
2857     }
2858     (*kc->dec)(kc);
2859     nmem_destroy(rset_nmem);
2860     i = before-i;
2861     if (i)
2862     {
2863         *is_partial = 1;
2864         *position -= i;
2865         *num_entries -= i;
2866         if (*num_entries <= 0)
2867         {
2868             *num_entries = 0;
2869             return ZEBRA_OK;
2870         }
2871     }
2872
2873     *list = glist + i;               /* list is set to first 'real' entry */
2874
2875     yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
2876             *position, *num_entries);
2877     return ZEBRA_OK;
2878 }
2879
2880 /*
2881  * Local variables:
2882  * c-basic-offset: 4
2883  * indent-tabs-mode: nil
2884  * End:
2885  * vim: shiftwidth=4 tabstop=8 expandtab
2886  */
2887