index/zrpn.c

   1 /* $Id: zrpn.c,v 1.221 2006-06-23 11:21:38 adam Exp $
   2    Copyright (C) 1995-2006
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with Zebra; see the file LICENSE.zebra.  If not, write to the
  19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  20 02111-1307, USA.
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #ifdef WIN32
  26 #include <io.h>
  27 #endif
  28 #if HAVE_UNISTD_H
  29 #include <unistd.h>
  30 #endif
  31 #include <ctype.h>
  32
  33 #include <yaz/diagbib1.h>
  34 #include "index.h"
  35 #include <zebra_xpath.h>
  36 #include <attrfind.h>
  37 #include <charmap.h>
  38 #include <rset.h>
  39
  40 struct rpn_char_map_info
  41 {
  42     ZebraMaps zm;
  43     int reg_type;
  44 };
  45
  46 static int log_level_set = 0;
  47 static int log_level_rpn = 0;
  48
  49 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  50 {
  51     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  52     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  53 #if 0
  54     if (out && *out)
  55     {
  56         const char *outp = *out;
  57         yaz_log(YLOG_LOG, "---");
  58         while (*outp)
  59         {
  60             yaz_log(YLOG_LOG, "%02X", *outp);
  61             outp++;
  62         }
  63     }
  64 #endif
  65     return out;
  66 }
  67
  68 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  69                                  struct rpn_char_map_info *map_info)
  70 {
  71     map_info->zm = reg->zebra_maps;
  72     map_info->reg_type = reg_type;
  73     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  74 }
  75
  76 #define TERM_COUNT
  77
  78 struct grep_info {
  79 #ifdef TERM_COUNT
  80     int *term_no;
  81 #endif
  82     ISAM_P *isam_p_buf;
  83     int isam_p_size;
  84     int isam_p_indx;
  85     ZebraHandle zh;
  86     int reg_type;
  87     ZebraSet termset;
  88 };
  89
  90 void zebra_term_untrans(ZebraHandle zh, int reg_type,
  91                         char *dst, const char *src)
  92 {
  93     int len = 0;
  94     while (*src)
  95     {
  96         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
  97                                            reg_type, &src);
  98         if (!cp)
  99         {
 100             if (len < IT_MAX_WORD-1)
 101                 dst[len++] = *src;
 102             src++;
 103         }
 104         else
 105             while (*cp && len < IT_MAX_WORD-1)
 106                 dst[len++] = *cp++;
 107     }
 108     dst[len] = '\0';
 109 }
 110
 111 static void add_isam_p(const char *name, const char *info,
 112                        struct grep_info *p)
 113 {
 114     if (!log_level_set)
 115     {
 116         log_level_rpn = yaz_log_module_level("rpn");
 117         log_level_set = 1;
 118     }
 119     if (p->isam_p_indx == p->isam_p_size)
 120     {
 121         ISAM_P *new_isam_p_buf;
 122 #ifdef TERM_COUNT
 123         int *new_term_no;
 124 #endif
 125         p->isam_p_size = 2*p->isam_p_size + 100;
 126         new_isam_p_buf = (ISAM_P *) xmalloc(sizeof(*new_isam_p_buf) *
 127                                             p->isam_p_size);
 128         if (p->isam_p_buf)
 129         {
 130             memcpy(new_isam_p_buf, p->isam_p_buf,
 131                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 132             xfree(p->isam_p_buf);
 133         }
 134         p->isam_p_buf = new_isam_p_buf;
 135
 136 #ifdef TERM_COUNT
 137         new_term_no = (int *) xmalloc(sizeof(*new_term_no) * p->isam_p_size);
 138         if (p->term_no)
 139         {
 140             memcpy(new_term_no, p->isam_p_buf,
 141                     p->isam_p_indx * sizeof(*p->term_no));
 142             xfree(p->term_no);
 143         }
 144         p->term_no = new_term_no;
 145 #endif
 146     }
 147     assert(*info == sizeof(*p->isam_p_buf));
 148     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 149
 150 #if 1
 151     if (p->termset)
 152     {
 153         const char *db;
 154         char term_tmp[IT_MAX_WORD];
 155         int ord = 0;
 156         const char *index_name;
 157         int len = key_SU_decode (&ord, (const unsigned char *) name);
 158
 159         zebra_term_untrans  (p->zh, p->reg_type, term_tmp, name+len+1);
 160         yaz_log(log_level_rpn, "grep: %d %c %s", ord, name[len], term_tmp);
 161         zebraExplain_lookup_ord(p->zh->reg->zei,
 162                                 ord, 0 /* index_type */, &db, &index_name);
 163         yaz_log(log_level_rpn, "grep:  db=%s index=%s", db, index_name);
 164
 165         resultSetAddTerm(p->zh, p->termset, name[len], db,
 166                          index_name, term_tmp);
 167     }
 168 #endif
 169     (p->isam_p_indx)++;
 170 }
 171
 172 static int grep_handle(char *name, const char *info, void *p)
 173 {
 174     add_isam_p(name, info, (struct grep_info *) p);
 175     return 0;
 176 }
 177
 178 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 179                     const char *ct1, const char *ct2, int first)
 180 {
 181     const char *s1, *s0 = *src;
 182     const char **map;
 183
 184     /* skip white space */
 185     while (*s0)
 186     {
 187         if (ct1 && strchr(ct1, *s0))
 188             break;
 189         if (ct2 && strchr(ct2, *s0))
 190             break;
 191         s1 = s0;
 192         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 193         if (**map != *CHR_SPACE)
 194             break;
 195         s0 = s1;
 196     }
 197     *src = s0;
 198     return *s0;
 199 }
 200
 201
 202 static void esc_str(char *out_buf, size_t out_size,
 203                     const char *in_buf, int in_size)
 204 {
 205     int k;
 206
 207     assert(out_buf);
 208     assert(in_buf);
 209     assert(out_size > 20);
 210     *out_buf = '\0';
 211     for (k = 0; k<in_size; k++)
 212     {
 213         int c = in_buf[k] & 0xff;
 214         int pc;
 215         if (c < 32 || c > 126)
 216             pc = '?';
 217         else
 218             pc = c;
 219         sprintf(out_buf +strlen(out_buf), "%02X:%c  ", c, pc);
 220         if (strlen(out_buf) > out_size-20)
 221         {
 222             strcat(out_buf, "..");
 223             break;
 224         }
 225     }
 226 }
 227
 228 #define REGEX_CHARS " []()|.*+?!"
 229
 230 /* term_100: handle term, where trunc = none(no operators at all) */
 231 static int term_100(ZebraMaps zebra_maps, int reg_type,
 232                     const char **src, char *dst, int space_split,
 233                     char *dst_term)
 234 {
 235     const char *s0;
 236     const char **map;
 237     int i = 0;
 238     int j = 0;
 239
 240     const char *space_start = 0;
 241     const char *space_end = 0;
 242
 243     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 244         return 0;
 245     s0 = *src;
 246     while (*s0)
 247     {
 248         const char *s1 = s0;
 249         int q_map_match = 0;
 250         map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 251                                 &q_map_match);
 252         if (space_split)
 253         {
 254             if (**map == *CHR_SPACE)
 255                 break;
 256         }
 257         else  /* complete subfield only. */
 258         {
 259             if (**map == *CHR_SPACE)
 260             {   /* save space mapping for later  .. */
 261                 space_start = s1;
 262                 space_end = s0;
 263                 continue;
 264             }
 265             else if (space_start)
 266             {   /* reload last space */
 267                 while (space_start < space_end)
 268                 {
 269                     if (strchr(REGEX_CHARS, *space_start))
 270                         dst[i++] = '\\';
 271                     dst_term[j++] = *space_start;
 272                     dst[i++] = *space_start++;
 273                 }
 274                 /* and reset */
 275                 space_start = space_end = 0;
 276             }
 277         }
 278         /* add non-space char */
 279         memcpy(dst_term+j, s1, s0 - s1);
 280         j += (s0 - s1);
 281         if (!q_map_match)
 282         {
 283             while (s1 < s0)
 284             {
 285                 if (strchr(REGEX_CHARS, *s1))
 286                     dst[i++] = '\\';
 287                 dst[i++] = *s1++;
 288             }
 289         }
 290         else
 291         {
 292             char tmpbuf[80];
 293             esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 294
 295             strcpy(dst + i, map[0]);
 296             i += strlen(map[0]);
 297         }
 298     }
 299     dst[i] = '\0';
 300     dst_term[j] = '\0';
 301     *src = s0;
 302     return i;
 303 }
 304
 305 /* term_101: handle term, where trunc = Process # */
 306 static int term_101(ZebraMaps zebra_maps, int reg_type,
 307                     const char **src, char *dst, int space_split,
 308                     char *dst_term)
 309 {
 310     const char *s0;
 311     const char **map;
 312     int i = 0;
 313     int j = 0;
 314
 315     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 316         return 0;
 317     s0 = *src;
 318     while (*s0)
 319     {
 320         if (*s0 == '#')
 321         {
 322             dst[i++] = '.';
 323             dst[i++] = '*';
 324             dst_term[j++] = *s0++;
 325         }
 326         else
 327         {
 328             const char *s1 = s0;
 329             int q_map_match = 0;
 330             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 331                                     &q_map_match);
 332             if (space_split && **map == *CHR_SPACE)
 333                 break;
 334
 335             /* add non-space char */
 336             memcpy(dst_term+j, s1, s0 - s1);
 337             j += (s0 - s1);
 338             if (!q_map_match)
 339             {
 340                 while (s1 < s0)
 341                 {
 342                     if (strchr(REGEX_CHARS, *s1))
 343                         dst[i++] = '\\';
 344                     dst[i++] = *s1++;
 345                 }
 346             }
 347             else
 348             {
 349                 char tmpbuf[80];
 350                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 351
 352                 strcpy(dst + i, map[0]);
 353                 i += strlen(map[0]);
 354             }
 355         }
 356     }
 357     dst[i] = '\0';
 358     dst_term[j++] = '\0';
 359     *src = s0;
 360     return i;
 361 }
 362
 363 /* term_103: handle term, where trunc = re-2 (regular expressions) */
 364 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 365                     char *dst, int *errors, int space_split,
 366                     char *dst_term)
 367 {
 368     int i = 0;
 369     int j = 0;
 370     const char *s0;
 371     const char **map;
 372
 373     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 374         return 0;
 375     s0 = *src;
 376     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 377         isdigit(((const unsigned char *)s0)[1]))
 378     {
 379         *errors = s0[1] - '0';
 380         s0 += 3;
 381         if (*errors > 3)
 382             *errors = 3;
 383     }
 384     while (*s0)
 385     {
 386         if (strchr("^\\()[].*+?|-", *s0))
 387         {
 388             dst_term[j++] = *s0;
 389             dst[i++] = *s0++;
 390         }
 391         else
 392         {
 393             const char *s1 = s0;
 394             int q_map_match = 0;
 395             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 396                                     &q_map_match);
 397             if (space_split && **map == *CHR_SPACE)
 398                 break;
 399
 400             /* add non-space char */
 401             memcpy(dst_term+j, s1, s0 - s1);
 402             j += (s0 - s1);
 403             if (!q_map_match)
 404             {
 405                 while (s1 < s0)
 406                 {
 407                     if (strchr(REGEX_CHARS, *s1))
 408                         dst[i++] = '\\';
 409                     dst[i++] = *s1++;
 410                 }
 411             }
 412             else
 413             {
 414                 char tmpbuf[80];
 415                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 416
 417                 strcpy(dst + i, map[0]);
 418                 i += strlen(map[0]);
 419             }
 420         }
 421     }
 422     dst[i] = '\0';
 423     dst_term[j] = '\0';
 424     *src = s0;
 425
 426     return i;
 427 }
 428
 429 /* term_103: handle term, where trunc = re-1 (regular expressions) */
 430 static int term_102(ZebraMaps zebra_maps, int reg_type, const char **src,
 431                     char *dst, int space_split, char *dst_term)
 432 {
 433     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 434                     dst_term);
 435 }
 436
 437
 438 /* term_104: handle term, where trunc = Process # and ! */
 439 static int term_104(ZebraMaps zebra_maps, int reg_type,
 440                     const char **src, char *dst, int space_split,
 441                     char *dst_term)
 442 {
 443     const char *s0;
 444     const char **map;
 445     int i = 0;
 446     int j = 0;
 447
 448     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 449         return 0;
 450     s0 = *src;
 451     while (*s0)
 452     {
 453         if (*s0 == '?')
 454         {
 455             dst_term[j++] = *s0++;
 456             if (*s0 >= '0' && *s0 <= '9')
 457             {
 458                 int limit = 0;
 459                 while (*s0 >= '0' && *s0 <= '9')
 460                 {
 461                     limit = limit * 10 + (*s0 - '0');
 462                     dst_term[j++] = *s0++;
 463                 }
 464                 if (limit > 20)
 465                     limit = 20;
 466                 while (--limit >= 0)
 467                 {
 468                     dst[i++] = '.';
 469                     dst[i++] = '?';
 470                 }
 471             }
 472             else
 473             {
 474                 dst[i++] = '.';
 475                 dst[i++] = '*';
 476             }
 477         }
 478         else if (*s0 == '*')
 479         {
 480             dst[i++] = '.';
 481             dst[i++] = '*';
 482             dst_term[j++] = *s0++;
 483         }
 484         else if (*s0 == '#')
 485         {
 486             dst[i++] = '.';
 487             dst_term[j++] = *s0++;
 488         }
 489         else
 490         {
 491             const char *s1 = s0;
 492             int q_map_match = 0;
 493             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 494                                     &q_map_match);
 495             if (space_split && **map == *CHR_SPACE)
 496                 break;
 497
 498             /* add non-space char */
 499             memcpy(dst_term+j, s1, s0 - s1);
 500             j += (s0 - s1);
 501             if (!q_map_match)
 502             {
 503                 while (s1 < s0)
 504                 {
 505                     if (strchr(REGEX_CHARS, *s1))
 506                         dst[i++] = '\\';
 507                     dst[i++] = *s1++;
 508                 }
 509             }
 510             else
 511             {
 512                 char tmpbuf[80];
 513                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 514
 515                 strcpy(dst + i, map[0]);
 516                 i += strlen(map[0]);
 517             }
 518         }
 519     }
 520     dst[i] = '\0';
 521     dst_term[j++] = '\0';
 522     *src = s0;
 523     return i;
 524 }
 525
 526 /* term_105/106: handle term, where trunc = Process * and ! and right trunc */
 527 static int term_105(ZebraMaps zebra_maps, int reg_type,
 528                     const char **src, char *dst, int space_split,
 529                     char *dst_term, int right_truncate)
 530 {
 531     const char *s0;
 532     const char **map;
 533     int i = 0;
 534     int j = 0;
 535
 536     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 537         return 0;
 538     s0 = *src;
 539     while (*s0)
 540     {
 541         if (*s0 == '*')
 542         {
 543             dst[i++] = '.';
 544             dst[i++] = '*';
 545             dst_term[j++] = *s0++;
 546         }
 547         else if (*s0 == '!')
 548         {
 549             dst[i++] = '.';
 550             dst_term[j++] = *s0++;
 551         }
 552         else
 553         {
 554             const char *s1 = s0;
 555             int q_map_match = 0;
 556             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 557                                     &q_map_match);
 558             if (space_split && **map == *CHR_SPACE)
 559                 break;
 560
 561             /* add non-space char */
 562             memcpy(dst_term+j, s1, s0 - s1);
 563             j += (s0 - s1);
 564             if (!q_map_match)
 565             {
 566                 while (s1 < s0)
 567                 {
 568                     if (strchr(REGEX_CHARS, *s1))
 569                         dst[i++] = '\\';
 570                     dst[i++] = *s1++;
 571                 }
 572             }
 573             else
 574             {
 575                 char tmpbuf[80];
 576                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 577
 578                 strcpy(dst + i, map[0]);
 579                 i += strlen(map[0]);
 580             }
 581         }
 582     }
 583     if (right_truncate)
 584     {
 585         dst[i++] = '.';
 586         dst[i++] = '*';
 587     }
 588     dst[i] = '\0';
 589
 590     dst_term[j++] = '\0';
 591     *src = s0;
 592     return i;
 593 }
 594
 595
 596 /* gen_regular_rel - generate regular expression from relation
 597  *  val:     border value (inclusive)
 598  *  islt:    1 if <=; 0 if >=.
 599  */
 600 static void gen_regular_rel(char *dst, int val, int islt)
 601 {
 602     int dst_p;
 603     int w, d, i;
 604     int pos = 0;
 605     char numstr[20];
 606
 607     yaz_log(YLOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 608     if (val >= 0)
 609     {
 610         if (islt)
 611             strcpy(dst, "(-[0-9]+|(");
 612         else
 613             strcpy(dst, "((");
 614     }
 615     else
 616     {
 617         if (!islt)
 618         {
 619             strcpy(dst, "([0-9]+|-(");
 620             dst_p = strlen(dst);
 621             islt = 1;
 622         }
 623         else
 624         {
 625             strcpy(dst, "(-(");
 626             islt = 0;
 627         }
 628         val = -val;
 629     }
 630     dst_p = strlen(dst);
 631     sprintf(numstr, "%d", val);
 632     for (w = strlen(numstr); --w >= 0; pos++)
 633     {
 634         d = numstr[w];
 635         if (pos > 0)
 636         {
 637             if (islt)
 638             {
 639                 if (d == '0')
 640                     continue;
 641                 d--;
 642             }
 643             else
 644             {
 645                 if (d == '9')
 646                     continue;
 647                 d++;
 648             }
 649         }
 650
 651         strcpy(dst + dst_p, numstr);
 652         dst_p = strlen(dst) - pos - 1;
 653
 654         if (islt)
 655         {
 656             if (d != '0')
 657             {
 658                 dst[dst_p++] = '[';
 659                 dst[dst_p++] = '0';
 660                 dst[dst_p++] = '-';
 661                 dst[dst_p++] = d;
 662                 dst[dst_p++] = ']';
 663             }
 664             else
 665                 dst[dst_p++] = d;
 666         }
 667         else
 668         {
 669             if (d != '9')
 670             {
 671                 dst[dst_p++] = '[';
 672                 dst[dst_p++] = d;
 673                 dst[dst_p++] = '-';
 674                 dst[dst_p++] = '9';
 675                 dst[dst_p++] = ']';
 676             }
 677             else
 678                 dst[dst_p++] = d;
 679         }
 680         for (i = 0; i<pos; i++)
 681         {
 682             dst[dst_p++] = '[';
 683             dst[dst_p++] = '0';
 684             dst[dst_p++] = '-';
 685             dst[dst_p++] = '9';
 686             dst[dst_p++] = ']';
 687         }
 688         dst[dst_p++] = '|';
 689     }
 690     dst[dst_p] = '\0';
 691     if (islt)
 692     {
 693         /* match everything less than 10^(pos-1) */
 694         strcat(dst, "0*");
 695         for (i = 1; i<pos; i++)
 696             strcat(dst, "[0-9]?");
 697     }
 698     else
 699     {
 700         /* match everything greater than 10^pos */
 701         for (i = 0; i <= pos; i++)
 702             strcat(dst, "[0-9]");
 703         strcat(dst, "[0-9]*");
 704     }
 705     strcat(dst, "))");
 706 }
 707
 708 void string_rel_add_char(char **term_p, const char *src, int *indx)
 709 {
 710     if (src[*indx] == '\\')
 711         *(*term_p)++ = src[(*indx)++];
 712     *(*term_p)++ = src[(*indx)++];
 713 }
 714
 715 /*
 716  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 717  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 718  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 719  *              ([^-a].*|a[^-b].*|ab[c-].*)
 720  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 721  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 722  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 723  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 724  */
 725 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 726                            const char **term_sub, char *term_dict,
 727                            oid_value attributeSet,
 728                            int reg_type, int space_split, char *term_dst,
 729                            int *error_code)
 730 {
 731     AttrType relation;
 732     int relation_value;
 733     int i;
 734     char *term_tmp = term_dict + strlen(term_dict);
 735     char term_component[2*IT_MAX_WORD+20];
 736
 737     attr_init_APT(&relation, zapt, 2);
 738     relation_value = attr_find(&relation, NULL);
 739
 740     *error_code = 0;
 741     yaz_log(YLOG_DEBUG, "string relation value=%d", relation_value);
 742     switch (relation_value)
 743     {
 744     case 1:
 745         if (!term_100(zh->reg->zebra_maps, reg_type,
 746                       term_sub, term_component,
 747                       space_split, term_dst))
 748             return 0;
 749         yaz_log(log_level_rpn, "Relation <");
 750
 751         *term_tmp++ = '(';
 752         for (i = 0; term_component[i]; )
 753         {
 754             int j = 0;
 755
 756             if (i)
 757                 *term_tmp++ = '|';
 758             while (j < i)
 759                 string_rel_add_char(&term_tmp, term_component, &j);
 760
 761             *term_tmp++ = '[';
 762
 763             *term_tmp++ = '^';
 764             string_rel_add_char(&term_tmp, term_component, &i);
 765             *term_tmp++ = '-';
 766
 767             *term_tmp++ = ']';
 768             *term_tmp++ = '.';
 769             *term_tmp++ = '*';
 770
 771             if ((term_tmp - term_dict) > IT_MAX_WORD)
 772                 break;
 773         }
 774         *term_tmp++ = ')';
 775         *term_tmp = '\0';
 776         break;
 777     case 2:
 778         if (!term_100(zh->reg->zebra_maps, reg_type,
 779                       term_sub, term_component,
 780                       space_split, term_dst))
 781             return 0;
 782         yaz_log(log_level_rpn, "Relation <=");
 783
 784         *term_tmp++ = '(';
 785         for (i = 0; term_component[i]; )
 786         {
 787             int j = 0;
 788
 789             while (j < i)
 790                 string_rel_add_char(&term_tmp, term_component, &j);
 791             *term_tmp++ = '[';
 792
 793             *term_tmp++ = '^';
 794             string_rel_add_char(&term_tmp, term_component, &i);
 795             *term_tmp++ = '-';
 796
 797             *term_tmp++ = ']';
 798             *term_tmp++ = '.';
 799             *term_tmp++ = '*';
 800
 801             *term_tmp++ = '|';
 802
 803             if ((term_tmp - term_dict) > IT_MAX_WORD)
 804                 break;
 805         }
 806         for (i = 0; term_component[i]; )
 807             string_rel_add_char(&term_tmp, term_component, &i);
 808         *term_tmp++ = ')';
 809         *term_tmp = '\0';
 810         break;
 811     case 5:
 812         if (!term_100 (zh->reg->zebra_maps, reg_type,
 813                        term_sub, term_component, space_split, term_dst))
 814             return 0;
 815         yaz_log(log_level_rpn, "Relation >");
 816
 817         *term_tmp++ = '(';
 818         for (i = 0; term_component[i];)
 819         {
 820             int j = 0;
 821
 822             while (j < i)
 823                 string_rel_add_char(&term_tmp, term_component, &j);
 824             *term_tmp++ = '[';
 825
 826             *term_tmp++ = '^';
 827             *term_tmp++ = '-';
 828             string_rel_add_char(&term_tmp, term_component, &i);
 829
 830             *term_tmp++ = ']';
 831             *term_tmp++ = '.';
 832             *term_tmp++ = '*';
 833
 834             *term_tmp++ = '|';
 835
 836             if ((term_tmp - term_dict) > IT_MAX_WORD)
 837                 break;
 838         }
 839         for (i = 0; term_component[i];)
 840             string_rel_add_char(&term_tmp, term_component, &i);
 841         *term_tmp++ = '.';
 842         *term_tmp++ = '+';
 843         *term_tmp++ = ')';
 844         *term_tmp = '\0';
 845         break;
 846     case 4:
 847         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 848                       term_component, space_split, term_dst))
 849             return 0;
 850         yaz_log(log_level_rpn, "Relation >=");
 851
 852         *term_tmp++ = '(';
 853         for (i = 0; term_component[i];)
 854         {
 855             int j = 0;
 856
 857             if (i)
 858                 *term_tmp++ = '|';
 859             while (j < i)
 860                 string_rel_add_char(&term_tmp, term_component, &j);
 861             *term_tmp++ = '[';
 862
 863             if (term_component[i+1])
 864             {
 865                 *term_tmp++ = '^';
 866                 *term_tmp++ = '-';
 867                 string_rel_add_char(&term_tmp, term_component, &i);
 868             }
 869             else
 870             {
 871                 string_rel_add_char(&term_tmp, term_component, &i);
 872                 *term_tmp++ = '-';
 873             }
 874             *term_tmp++ = ']';
 875             *term_tmp++ = '.';
 876             *term_tmp++ = '*';
 877
 878             if ((term_tmp - term_dict) > IT_MAX_WORD)
 879                 break;
 880         }
 881         *term_tmp++ = ')';
 882         *term_tmp = '\0';
 883         break;
 884     case 3:
 885     case 102:
 886     case -1:
 887         if (!**term_sub)
 888             return 1;
 889         yaz_log(log_level_rpn, "Relation =");
 890         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 891                       term_component, space_split, term_dst))
 892             return 0;
 893         strcat(term_tmp, "(");
 894         strcat(term_tmp, term_component);
 895         strcat(term_tmp, ")");
 896         break;
 897     case 103:
 898         yaz_log(log_level_rpn, "Relation always matches");
 899         /* skip to end of term (we don't care what it is) */
 900         while (**term_sub != '\0')
 901             (*term_sub)++;
 902         break;
 903     default:
 904         *error_code = YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
 905         return 0;
 906     }
 907     return 1;
 908 }
 909
 910 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 911                              const char **term_sub,
 912                              oid_value attributeSet, NMEM stream,
 913                              struct grep_info *grep_info,
 914                              int reg_type, int complete_flag,
 915                              int num_bases, char **basenames,
 916                              char *term_dst,
 917                              const char *xpath_use,
 918                              struct ord_list **ol);
 919
 920 static ZEBRA_RES term_limits_APT(ZebraHandle zh,
 921                                  Z_AttributesPlusTerm *zapt,
 922                                  zint *hits_limit_value,
 923                                  const char **term_ref_id_str,
 924                                  NMEM nmem)
 925 {
 926     AttrType term_ref_id_attr;
 927     AttrType hits_limit_attr;
 928     int term_ref_id_int;
 929
 930     attr_init_APT(&hits_limit_attr, zapt, 9);
 931     *hits_limit_value  = attr_find(&hits_limit_attr, NULL);
 932
 933     attr_init_APT(&term_ref_id_attr, zapt, 10);
 934     term_ref_id_int = attr_find_ex(&term_ref_id_attr, NULL, term_ref_id_str);
 935     if (term_ref_id_int >= 0)
 936     {
 937         char *res = nmem_malloc(nmem, 20);
 938         sprintf(res, "%d", term_ref_id_int);
 939         *term_ref_id_str = res;
 940     }
 941
 942     /* no limit given ? */
 943     if (*hits_limit_value == -1)
 944     {
 945         if (*term_ref_id_str)
 946         {
 947             /* use global if term_ref is present */
 948             *hits_limit_value = zh->approx_limit;
 949         }
 950         else
 951         {
 952             /* no counting if term_ref is not present */
 953             *hits_limit_value = 0;
 954         }
 955     }
 956     else if (*hits_limit_value == 0)
 957     {
 958         /* 0 is the same as global limit */
 959         *hits_limit_value = zh->approx_limit;
 960     }
 961     yaz_log(YLOG_DEBUG, "term_limits_APT ref_id=%s limit=" ZINT_FORMAT,
 962             *term_ref_id_str ? *term_ref_id_str : "none",
 963             *hits_limit_value);
 964     return ZEBRA_OK;
 965 }
 966
 967 static ZEBRA_RES term_trunc(ZebraHandle zh,
 968                             Z_AttributesPlusTerm *zapt,
 969                             const char **term_sub,
 970                             oid_value attributeSet, NMEM stream,
 971                             struct grep_info *grep_info,
 972                             int reg_type, int complete_flag,
 973                             int num_bases, char **basenames,
 974                             char *term_dst,
 975                             const char *rank_type,
 976                             const char *xpath_use,
 977                             NMEM rset_nmem,
 978                             RSET *rset,
 979                             struct rset_key_control *kc)
 980 {
 981     ZEBRA_RES res;
 982     struct ord_list *ol;
 983     zint hits_limit_value;
 984     const char *term_ref_id_str = 0;
 985     *rset = 0;
 986
 987     term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str, stream);
 988     grep_info->isam_p_indx = 0;
 989     res = string_term(zh, zapt, term_sub, attributeSet, stream, grep_info,
 990                       reg_type, complete_flag, num_bases, basenames,
 991                       term_dst, xpath_use, &ol);
 992     if (res != ZEBRA_OK)
 993         return res;
 994     if (!*term_sub)  /* no more terms ? */
 995         return res;
 996     yaz_log(log_level_rpn, "term: %s", term_dst);
 997     *rset = rset_trunc(zh, grep_info->isam_p_buf,
 998                        grep_info->isam_p_indx, term_dst,
 999                        strlen(term_dst), rank_type, 1 /* preserve pos */,
1000                        zapt->term->which, rset_nmem,
1001                        kc, kc->scope, ol, reg_type, hits_limit_value,
1002                        term_ref_id_str);
1003     if (!*rset)
1004         return ZEBRA_FAIL;
1005     return ZEBRA_OK;
1006 }
1007
1008 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1009                              const char **term_sub,
1010                              oid_value attributeSet, NMEM stream,
1011                              struct grep_info *grep_info,
1012                              int reg_type, int complete_flag,
1013                              int num_bases, char **basenames,
1014                              char *term_dst,
1015                              const char *xpath_use,
1016                              struct ord_list **ol)
1017 {
1018     char term_dict[2*IT_MAX_WORD+4000];
1019     int j, r, base_no;
1020     AttrType truncation;
1021     int truncation_value;
1022     const char *termp;
1023     struct rpn_char_map_info rcmi;
1024     int space_split = complete_flag ? 0 : 1;
1025
1026     int bases_ok = 0;     /* no of databases with OK attribute */
1027
1028     *ol = ord_list_create(stream);
1029
1030     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1031     attr_init_APT(&truncation, zapt, 5);
1032     truncation_value = attr_find(&truncation, NULL);
1033     yaz_log(log_level_rpn, "truncation value %d", truncation_value);
1034
1035     for (base_no = 0; base_no < num_bases; base_no++)
1036     {
1037         int ord = -1;
1038         int regex_range = 0;
1039         int max_pos, prefix_len = 0;
1040         int relation_error;
1041         char ord_buf[32];
1042         int ord_len, i;
1043
1044         termp = *term_sub; /* start of term for each database */
1045
1046         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1047         {
1048             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1049                            basenames[base_no]);
1050             return ZEBRA_FAIL;
1051         }
1052
1053         if (zebra_apt_get_ord(zh, zapt, reg_type, xpath_use,
1054                               attributeSet, &ord) != ZEBRA_OK)
1055             continue;
1056
1057         bases_ok++;
1058
1059         *ol = ord_list_append(stream, *ol, ord);
1060         ord_len = key_SU_encode (ord, ord_buf);
1061
1062         term_dict[prefix_len++] = '(';
1063         for (i = 0; i<ord_len; i++)
1064         {
1065             term_dict[prefix_len++] = 1;  /* our internal regexp escape char */
1066             term_dict[prefix_len++] = ord_buf[i];
1067         }
1068         term_dict[prefix_len++] = ')';
1069         term_dict[prefix_len] = '\0';
1070         j = prefix_len;
1071         switch (truncation_value)
1072         {
1073         case -1:         /* not specified */
1074         case 100:        /* do not truncate */
1075             if (!string_relation(zh, zapt, &termp, term_dict,
1076                                  attributeSet,
1077                                  reg_type, space_split, term_dst,
1078                                  &relation_error))
1079             {
1080                 if (relation_error)
1081                 {
1082                     zebra_setError(zh, relation_error, 0);
1083                     return ZEBRA_FAIL;
1084                 }
1085                 *term_sub = 0;
1086                 return ZEBRA_OK;
1087             }
1088             break;
1089         case 1:          /* right truncation */
1090             term_dict[j++] = '(';
1091             if (!term_100(zh->reg->zebra_maps, reg_type,
1092                           &termp, term_dict + j, space_split, term_dst))
1093             {
1094                 *term_sub = 0;
1095                 return ZEBRA_OK;
1096             }
1097             strcat(term_dict, ".*)");
1098             break;
1099         case 2:          /* keft truncation */
1100             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1101             if (!term_100(zh->reg->zebra_maps, reg_type,
1102                           &termp, term_dict + j, space_split, term_dst))
1103             {
1104                 *term_sub = 0;
1105                 return ZEBRA_OK;
1106             }
1107             strcat(term_dict, ")");
1108             break;
1109         case 3:          /* left&right truncation */
1110             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1111             if (!term_100(zh->reg->zebra_maps, reg_type,
1112                           &termp, term_dict + j, space_split, term_dst))
1113             {
1114                 *term_sub = 0;
1115                 return ZEBRA_OK;
1116             }
1117             strcat(term_dict, ".*)");
1118             break;
1119         case 101:        /* process # in term */
1120             term_dict[j++] = '(';
1121             if (!term_101(zh->reg->zebra_maps, reg_type,
1122                           &termp, term_dict + j, space_split, term_dst))
1123             {
1124                 *term_sub = 0;
1125                 return ZEBRA_OK;
1126             }
1127             strcat(term_dict, ")");
1128             break;
1129         case 102:        /* Regexp-1 */
1130             term_dict[j++] = '(';
1131             if (!term_102(zh->reg->zebra_maps, reg_type,
1132                           &termp, term_dict + j, space_split, term_dst))
1133             {
1134                 *term_sub = 0;
1135                 return ZEBRA_OK;
1136             }
1137             strcat(term_dict, ")");
1138             break;
1139         case 103:       /* Regexp-2 */
1140             regex_range = 1;
1141             term_dict[j++] = '(';
1142             if (!term_103(zh->reg->zebra_maps, reg_type,
1143                           &termp, term_dict + j, &regex_range,
1144                           space_split, term_dst))
1145             {
1146                 *term_sub = 0;
1147                 return ZEBRA_OK;
1148             }
1149             strcat(term_dict, ")");
1150             break;
1151         case 104:        /* process # and ! in term */
1152             term_dict[j++] = '(';
1153             if (!term_104(zh->reg->zebra_maps, reg_type,
1154                           &termp, term_dict + j, space_split, term_dst))
1155             {
1156                 *term_sub = 0;
1157                 return ZEBRA_OK;
1158             }
1159             strcat(term_dict, ")");
1160             break;
1161         case 105:        /* process * and ! in term */
1162             term_dict[j++] = '(';
1163             if (!term_105(zh->reg->zebra_maps, reg_type,
1164                           &termp, term_dict + j, space_split, term_dst, 1))
1165             {
1166                 *term_sub = 0;
1167                 return ZEBRA_OK;
1168             }
1169             strcat(term_dict, ")");
1170             break;
1171         case 106:        /* process * and ! in term */
1172             term_dict[j++] = '(';
1173             if (!term_105(zh->reg->zebra_maps, reg_type,
1174                           &termp, term_dict + j, space_split, term_dst, 0))
1175             {
1176                 *term_sub = 0;
1177                 return ZEBRA_OK;
1178             }
1179             strcat(term_dict, ")");
1180             break;
1181         default:
1182             zebra_setError_zint(zh,
1183                                 YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE,
1184                                 truncation_value);
1185             return ZEBRA_FAIL;
1186         }
1187         if (1)
1188         {
1189             char buf[80];
1190             const char *input = term_dict + prefix_len;
1191             esc_str(buf, sizeof(buf), input, strlen(input));
1192         }
1193         yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_dict+prefix_len);
1194         r = dict_lookup_grep(zh->reg->dict, term_dict, regex_range,
1195                              grep_info, &max_pos,
1196                              ord_len /* number of "exact" chars */,
1197                              grep_handle);
1198         if (r)
1199             yaz_log(YLOG_WARN, "dict_lookup_grep fail %d", r);
1200     }
1201     if (!bases_ok)
1202         return ZEBRA_FAIL;
1203     *term_sub = termp;
1204     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1205     return ZEBRA_OK;
1206 }
1207
1208
1209 /* convert APT search term to UTF8 */
1210 static ZEBRA_RES zapt_term_to_utf8(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1211                                    char *termz)
1212 {
1213     size_t sizez;
1214     Z_Term *term = zapt->term;
1215
1216     switch (term->which)
1217     {
1218     case Z_Term_general:
1219         if (zh->iconv_to_utf8 != 0)
1220         {
1221             char *inbuf = (char *) term->u.general->buf;
1222             size_t inleft = term->u.general->len;
1223             char *outbuf = termz;
1224             size_t outleft = IT_MAX_WORD-1;
1225             size_t ret;
1226
1227             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1228                         &outbuf, &outleft);
1229             if (ret == (size_t)(-1))
1230             {
1231                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1232                 zebra_setError(
1233                     zh,
1234                     YAZ_BIB1_QUERY_TERM_INCLUDES_CHARS_THAT_DO_NOT_TRANSLATE_INTO_,
1235                     0);
1236                 return ZEBRA_FAIL;
1237             }
1238             *outbuf = 0;
1239         }
1240         else
1241         {
1242             sizez = term->u.general->len;
1243             if (sizez > IT_MAX_WORD-1)
1244                 sizez = IT_MAX_WORD-1;
1245             memcpy (termz, term->u.general->buf, sizez);
1246             termz[sizez] = '\0';
1247         }
1248         break;
1249     case Z_Term_characterString:
1250         sizez = strlen(term->u.characterString);
1251         if (sizez > IT_MAX_WORD-1)
1252             sizez = IT_MAX_WORD-1;
1253         memcpy (termz, term->u.characterString, sizez);
1254         termz[sizez] = '\0';
1255         break;
1256     default:
1257         zebra_setError(zh, YAZ_BIB1_UNSUPP_CODED_VALUE_FOR_TERM, 0);
1258         return ZEBRA_FAIL;
1259     }
1260     return ZEBRA_OK;
1261 }
1262
1263 /* convert APT SCAN term to internal cmap */
1264 static ZEBRA_RES trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1265                                  char *termz, int reg_type)
1266 {
1267     char termz0[IT_MAX_WORD];
1268
1269     if (zapt_term_to_utf8(zh, zapt, termz0) == ZEBRA_FAIL)
1270         return ZEBRA_FAIL;    /* error */
1271     else
1272     {
1273         const char **map;
1274         const char *cp = (const char *) termz0;
1275         const char *cp_end = cp + strlen(cp);
1276         const char *src;
1277         int i = 0;
1278         const char *space_map = NULL;
1279         int len;
1280
1281         while ((len = (cp_end - cp)) > 0)
1282         {
1283             map = zebra_maps_input(zh->reg->zebra_maps, reg_type, &cp, len, 0);
1284             if (**map == *CHR_SPACE)
1285                 space_map = *map;
1286             else
1287             {
1288                 if (i && space_map)
1289                     for (src = space_map; *src; src++)
1290                         termz[i++] = *src;
1291                 space_map = NULL;
1292                 for (src = *map; *src; src++)
1293                     termz[i++] = *src;
1294             }
1295         }
1296         termz[i] = '\0';
1297     }
1298     return ZEBRA_OK;
1299 }
1300
1301 static void grep_info_delete(struct grep_info *grep_info)
1302 {
1303 #ifdef TERM_COUNT
1304     xfree(grep_info->term_no);
1305 #endif
1306     xfree(grep_info->isam_p_buf);
1307 }
1308
1309 static ZEBRA_RES grep_info_prepare(ZebraHandle zh,
1310                                    Z_AttributesPlusTerm *zapt,
1311                                    struct grep_info *grep_info,
1312                                    int reg_type)
1313 {
1314     AttrType termset;
1315     int termset_value_numeric;
1316     const char *termset_value_string;
1317
1318 #ifdef TERM_COUNT
1319     grep_info->term_no = 0;
1320 #endif
1321     grep_info->isam_p_size = 0;
1322     grep_info->isam_p_buf = NULL;
1323     grep_info->zh = zh;
1324     grep_info->reg_type = reg_type;
1325     grep_info->termset = 0;
1326
1327     if (!zapt)
1328         return ZEBRA_OK;
1329     attr_init_APT(&termset, zapt, 8);
1330     termset_value_numeric =
1331         attr_find_ex(&termset, NULL, &termset_value_string);
1332     if (termset_value_numeric != -1)
1333     {
1334         char resname[32];
1335         const char *termset_name = 0;
1336         if (termset_value_numeric != -2)
1337         {
1338
1339             sprintf(resname, "%d", termset_value_numeric);
1340             termset_name = resname;
1341         }
1342         else
1343             termset_name = termset_value_string;
1344         yaz_log(log_level_rpn, "creating termset set %s", termset_name);
1345         grep_info->termset = resultSetAdd(zh, termset_name, 1);
1346         if (!grep_info->termset)
1347         {
1348             zebra_setError(zh, YAZ_BIB1_ILLEGAL_RESULT_SET_NAME, termset_name);
1349             return ZEBRA_FAIL;
1350         }
1351     }
1352     return ZEBRA_OK;
1353 }
1354
1355 /**
1356   \brief Create result set(s) for list of terms
1357   \param zh Zebra Handle
1358   \param termz term as used in query but converted to UTF-8
1359   \param attributeSet default attribute set
1360   \param stream memory for result
1361   \param reg_type register type ('w', 'p',..)
1362   \param complete_flag whether it's phrases or not
1363   \param rank_type term flags for ranking
1364   \param xpath_use use attribute for X-Path (-1 for no X-path)
1365   \param num_bases number of databases
1366   \param basenames array of databases
1367   \param rset_mem memory for result sets
1368   \param result_sets output result set for each term in list (output)
1369   \param number number of output result sets
1370   \param kc rset key control to be used for created result sets
1371 */
1372 static ZEBRA_RES term_list_trunc(ZebraHandle zh,
1373                                  Z_AttributesPlusTerm *zapt,
1374                                  const char *termz,
1375                                  oid_value attributeSet,
1376                                  NMEM stream,
1377                                  int reg_type, int complete_flag,
1378                                  const char *rank_type,
1379                                  const char *xpath_use,
1380                                  int num_bases, char **basenames,
1381                                  NMEM rset_nmem,
1382                                  RSET **result_sets, int *num_result_sets,
1383                                  struct rset_key_control *kc)
1384 {
1385     char term_dst[IT_MAX_WORD+1];
1386     struct grep_info grep_info;
1387     const char *termp = termz;
1388     int alloc_sets = 0;
1389
1390     *num_result_sets = 0;
1391     *term_dst = 0;
1392     if (grep_info_prepare(zh, zapt, &grep_info, reg_type) == ZEBRA_FAIL)
1393         return ZEBRA_FAIL;
1394     while(1)
1395     {
1396         ZEBRA_RES res;
1397
1398         if (alloc_sets == *num_result_sets)
1399         {
1400             int add = 10;
1401             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1402                                               sizeof(*rnew));
1403             if (alloc_sets)
1404                 memcpy(rnew, *result_sets, alloc_sets * sizeof(*rnew));
1405             alloc_sets = alloc_sets + add;
1406             *result_sets = rnew;
1407         }
1408         res = term_trunc(zh, zapt, &termp, attributeSet,
1409                          stream, &grep_info,
1410                          reg_type, complete_flag,
1411                          num_bases, basenames,
1412                          term_dst, rank_type,
1413                          xpath_use, rset_nmem,
1414                          &(*result_sets)[*num_result_sets],
1415                          kc);
1416         if (res != ZEBRA_OK)
1417         {
1418             int i;
1419             for (i = 0; i < *num_result_sets; i++)
1420                 rset_delete((*result_sets)[i]);
1421             grep_info_delete (&grep_info);
1422             return res;
1423         }
1424         if ((*result_sets)[*num_result_sets] == 0)
1425             break;
1426         (*num_result_sets)++;
1427
1428         if (!*termp)
1429             break;
1430     }
1431     grep_info_delete(&grep_info);
1432     return ZEBRA_OK;
1433 }
1434
1435 static ZEBRA_RES rpn_search_APT_phrase(ZebraHandle zh,
1436                                        Z_AttributesPlusTerm *zapt,
1437                                        const char *termz_org,
1438                                        oid_value attributeSet,
1439                                        NMEM stream,
1440                                        int reg_type, int complete_flag,
1441                                        const char *rank_type,
1442                                        const char *xpath_use,
1443                                        int num_bases, char **basenames,
1444                                        NMEM rset_nmem,
1445                                        RSET *rset,
1446                                        struct rset_key_control *kc)
1447 {
1448     RSET *result_sets = 0;
1449     int num_result_sets = 0;
1450     ZEBRA_RES res =
1451         term_list_trunc(zh, zapt, termz_org, attributeSet,
1452                         stream, reg_type, complete_flag,
1453                         rank_type, xpath_use,
1454                         num_bases, basenames,
1455                         rset_nmem,
1456                         &result_sets, &num_result_sets, kc);
1457     if (res != ZEBRA_OK)
1458         return res;
1459     if (num_result_sets == 0)
1460         *rset = rset_create_null(rset_nmem, kc, 0);
1461     else if (num_result_sets == 1)
1462         *rset = result_sets[0];
1463     else
1464         *rset = rset_create_prox(rset_nmem, kc, kc->scope,
1465                                  num_result_sets, result_sets,
1466                                  1 /* ordered */, 0 /* exclusion */,
1467                                  3 /* relation */, 1 /* distance */);
1468     if (!*rset)
1469         return ZEBRA_FAIL;
1470     return ZEBRA_OK;
1471 }
1472
1473 static ZEBRA_RES rpn_search_APT_or_list(ZebraHandle zh,
1474                                         Z_AttributesPlusTerm *zapt,
1475                                         const char *termz_org,
1476                                         oid_value attributeSet,
1477                                         NMEM stream,
1478                                         int reg_type, int complete_flag,
1479                                         const char *rank_type,
1480                                         const char *xpath_use,
1481                                         int num_bases, char **basenames,
1482                                         NMEM rset_nmem,
1483                                         RSET *rset,
1484                                         struct rset_key_control *kc)
1485 {
1486     RSET *result_sets = 0;
1487     int num_result_sets = 0;
1488     ZEBRA_RES res =
1489         term_list_trunc(zh, zapt, termz_org, attributeSet,
1490                         stream, reg_type, complete_flag,
1491                         rank_type, xpath_use,
1492                         num_bases, basenames,
1493                         rset_nmem,
1494                         &result_sets, &num_result_sets, kc);
1495     if (res != ZEBRA_OK)
1496         return res;
1497     if (num_result_sets == 0)
1498         *rset = rset_create_null(rset_nmem, kc, 0);
1499     else if (num_result_sets == 1)
1500         *rset = result_sets[0];
1501     else
1502         *rset = rset_create_or(rset_nmem, kc, kc->scope, 0 /* termid */,
1503                                num_result_sets, result_sets);
1504     if (!*rset)
1505         return ZEBRA_FAIL;
1506     return ZEBRA_OK;
1507 }
1508
1509 static ZEBRA_RES rpn_search_APT_and_list(ZebraHandle zh,
1510                                          Z_AttributesPlusTerm *zapt,
1511                                          const char *termz_org,
1512                                          oid_value attributeSet,
1513                                          NMEM stream,
1514                                          int reg_type, int complete_flag,
1515                                          const char *rank_type,
1516                                          const char *xpath_use,
1517                                          int num_bases, char **basenames,
1518                                          NMEM rset_nmem,
1519                                          RSET *rset,
1520                                          struct rset_key_control *kc)
1521 {
1522     RSET *result_sets = 0;
1523     int num_result_sets = 0;
1524     ZEBRA_RES res =
1525         term_list_trunc(zh, zapt, termz_org, attributeSet,
1526                         stream, reg_type, complete_flag,
1527                         rank_type, xpath_use,
1528                         num_bases, basenames,
1529                         rset_nmem,
1530                         &result_sets, &num_result_sets,
1531                         kc);
1532     if (res != ZEBRA_OK)
1533         return res;
1534     if (num_result_sets == 0)
1535         *rset = rset_create_null(rset_nmem, kc, 0);
1536     else if (num_result_sets == 1)
1537         *rset = result_sets[0];
1538     else
1539         *rset = rset_create_and(rset_nmem, kc, kc->scope,
1540                                 num_result_sets, result_sets);
1541     if (!*rset)
1542         return ZEBRA_FAIL;
1543     return ZEBRA_OK;
1544 }
1545
1546 static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1547                             const char **term_sub,
1548                             char *term_dict,
1549                             oid_value attributeSet,
1550                             struct grep_info *grep_info,
1551                             int *max_pos,
1552                             int reg_type,
1553                             char *term_dst,
1554                             int *error_code)
1555 {
1556     AttrType relation;
1557     int relation_value;
1558     int term_value;
1559     int r;
1560     char *term_tmp = term_dict + strlen(term_dict);
1561
1562     *error_code = 0;
1563     attr_init_APT(&relation, zapt, 2);
1564     relation_value = attr_find(&relation, NULL);
1565
1566     yaz_log(log_level_rpn, "numeric relation value=%d", relation_value);
1567
1568     switch (relation_value)
1569     {
1570     case 1:
1571         yaz_log(log_level_rpn, "Relation <");
1572         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1573                       term_dst))
1574             return 0;
1575         term_value = atoi (term_tmp);
1576         gen_regular_rel(term_tmp, term_value-1, 1);
1577         break;
1578     case 2:
1579         yaz_log(log_level_rpn, "Relation <=");
1580         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1581                       term_dst))
1582             return 0;
1583         term_value = atoi (term_tmp);
1584         gen_regular_rel(term_tmp, term_value, 1);
1585         break;
1586     case 4:
1587         yaz_log(log_level_rpn, "Relation >=");
1588         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1589                       term_dst))
1590             return 0;
1591         term_value = atoi (term_tmp);
1592         gen_regular_rel(term_tmp, term_value, 0);
1593         break;
1594     case 5:
1595         yaz_log(log_level_rpn, "Relation >");
1596         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1597                       term_dst))
1598             return 0;
1599         term_value = atoi (term_tmp);
1600         gen_regular_rel(term_tmp, term_value+1, 0);
1601         break;
1602     case -1:
1603     case 3:
1604         yaz_log(log_level_rpn, "Relation =");
1605         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1606                       term_dst))
1607             return 0;
1608         term_value = atoi (term_tmp);
1609         sprintf(term_tmp, "(0*%d)", term_value);
1610         break;
1611     case 103:
1612         /* term_tmp untouched.. */
1613         while (**term_sub != '\0')
1614             (*term_sub)++;
1615         break;
1616     default:
1617         *error_code = YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
1618         return 0;
1619     }
1620     yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_tmp);
1621     r = dict_lookup_grep(zh->reg->dict, term_dict, 0, grep_info, max_pos,
1622                           0, grep_handle);
1623     if (r)
1624         yaz_log(YLOG_WARN, "dict_lookup_grep fail, rel = gt: %d", r);
1625     yaz_log(log_level_rpn, "%d positions", grep_info->isam_p_indx);
1626     return 1;
1627 }
1628
1629 static ZEBRA_RES numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1630                               const char **term_sub,
1631                               oid_value attributeSet, NMEM stream,
1632                               struct grep_info *grep_info,
1633                               int reg_type, int complete_flag,
1634                               int num_bases, char **basenames,
1635                               char *term_dst,
1636                               const char *xpath_use,
1637                               struct ord_list **ol)
1638 {
1639     char term_dict[2*IT_MAX_WORD+2];
1640     int base_no;
1641     const char *termp;
1642     struct rpn_char_map_info rcmi;
1643
1644     int bases_ok = 0;     /* no of databases with OK attribute */
1645
1646     *ol = ord_list_create(stream);
1647
1648     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1649
1650     for (base_no = 0; base_no < num_bases; base_no++)
1651     {
1652         int max_pos, prefix_len = 0;
1653         int relation_error = 0;
1654         int ord, ord_len, i;
1655         char ord_buf[32];
1656
1657         termp = *term_sub;
1658
1659         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1660         {
1661             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1662                            basenames[base_no]);
1663             return ZEBRA_FAIL;
1664         }
1665
1666         if (zebra_apt_get_ord(zh, zapt, reg_type, xpath_use,
1667                               attributeSet, &ord) != ZEBRA_OK)
1668             continue;
1669         bases_ok++;
1670
1671         *ol = ord_list_append(stream, *ol, ord);
1672
1673         ord_len = key_SU_encode (ord, ord_buf);
1674
1675         term_dict[prefix_len++] = '(';
1676         for (i = 0; i < ord_len; i++)
1677         {
1678             term_dict[prefix_len++] = 1;
1679             term_dict[prefix_len++] = ord_buf[i];
1680         }
1681         term_dict[prefix_len++] = ')';
1682         term_dict[prefix_len] = '\0';
1683
1684         if (!numeric_relation(zh, zapt, &termp, term_dict,
1685                               attributeSet, grep_info, &max_pos, reg_type,
1686                               term_dst, &relation_error))
1687         {
1688             if (relation_error)
1689             {
1690                 zebra_setError(zh, relation_error, 0);
1691                 return ZEBRA_FAIL;
1692             }
1693             *term_sub = 0;
1694             return ZEBRA_OK;
1695         }
1696     }
1697     if (!bases_ok)
1698         return ZEBRA_FAIL;
1699     *term_sub = termp;
1700     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1701     return ZEBRA_OK;
1702 }
1703
1704
1705 static ZEBRA_RES rpn_search_APT_numeric(ZebraHandle zh,
1706                                         Z_AttributesPlusTerm *zapt,
1707                                         const char *termz,
1708                                         oid_value attributeSet,
1709                                         NMEM stream,
1710                                         int reg_type, int complete_flag,
1711                                         const char *rank_type,
1712                                         const char *xpath_use,
1713                                         int num_bases, char **basenames,
1714                                         NMEM rset_nmem,
1715                                         RSET *rset,
1716                                         struct rset_key_control *kc)
1717 {
1718     char term_dst[IT_MAX_WORD+1];
1719     const char *termp = termz;
1720     RSET *result_sets = 0;
1721     int num_result_sets = 0;
1722     ZEBRA_RES res;
1723     struct grep_info grep_info;
1724     int alloc_sets = 0;
1725     zint hits_limit_value;
1726     const char *term_ref_id_str = 0;
1727
1728     term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str, stream);
1729
1730     yaz_log(log_level_rpn, "APT_numeric t='%s'", termz);
1731     if (grep_info_prepare(zh, zapt, &grep_info, reg_type) == ZEBRA_FAIL)
1732         return ZEBRA_FAIL;
1733     while (1)
1734     {
1735         struct ord_list *ol;
1736         if (alloc_sets == num_result_sets)
1737         {
1738             int add = 10;
1739             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1740                                               sizeof(*rnew));
1741             if (alloc_sets)
1742                 memcpy(rnew, result_sets, alloc_sets * sizeof(*rnew));
1743             alloc_sets = alloc_sets + add;
1744             result_sets = rnew;
1745         }
1746         yaz_log(YLOG_DEBUG, "APT_numeric termp=%s", termp);
1747         grep_info.isam_p_indx = 0;
1748         res = numeric_term(zh, zapt, &termp, attributeSet, stream, &grep_info,
1749                            reg_type, complete_flag, num_bases, basenames,
1750                            term_dst, xpath_use, &ol);
1751         if (res == ZEBRA_FAIL || termp == 0)
1752             break;
1753         yaz_log(YLOG_DEBUG, "term: %s", term_dst);
1754         result_sets[num_result_sets] =
1755             rset_trunc(zh, grep_info.isam_p_buf,
1756                        grep_info.isam_p_indx, term_dst,
1757                        strlen(term_dst), rank_type,
1758                        0 /* preserve position */,
1759                        zapt->term->which, rset_nmem,
1760                        kc, kc->scope, ol, reg_type,
1761                        hits_limit_value,
1762                        term_ref_id_str);
1763         if (!result_sets[num_result_sets])
1764             break;
1765         num_result_sets++;
1766         if (!*termp)
1767             break;
1768     }
1769     grep_info_delete(&grep_info);
1770
1771     if (res != ZEBRA_OK)
1772         return res;
1773     if (num_result_sets == 0)
1774         *rset = rset_create_null(rset_nmem, kc, 0);
1775     else if (num_result_sets == 1)
1776         *rset = result_sets[0];
1777     else
1778         *rset = rset_create_and(rset_nmem, kc, kc->scope,
1779                                 num_result_sets, result_sets);
1780     if (!*rset)
1781         return ZEBRA_FAIL;
1782     return ZEBRA_OK;
1783 }
1784
1785 static ZEBRA_RES rpn_search_APT_local(ZebraHandle zh,
1786                                       Z_AttributesPlusTerm *zapt,
1787                                       const char *termz,
1788                                       oid_value attributeSet,
1789                                       NMEM stream,
1790                                       const char *rank_type, NMEM rset_nmem,
1791                                       RSET *rset,
1792                                       struct rset_key_control *kc)
1793 {
1794     RSFD rsfd;
1795     struct it_key key;
1796     int sys;
1797     *rset = rset_create_temp(rset_nmem, kc, kc->scope,
1798                              res_get (zh->res, "setTmpDir"),0 );
1799     rsfd = rset_open(*rset, RSETF_WRITE);
1800
1801     sys = atoi(termz);
1802     if (sys <= 0)
1803         sys = 1;
1804     key.mem[0] = sys;
1805     key.mem[1] = 1;
1806     key.len = 2;
1807     rset_write (rsfd, &key);
1808     rset_close (rsfd);
1809     return ZEBRA_OK;
1810 }
1811
1812 static ZEBRA_RES rpn_sort_spec(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1813                                oid_value attributeSet, NMEM stream,
1814                                Z_SortKeySpecList *sort_sequence,
1815                                const char *rank_type,
1816                                NMEM rset_nmem,
1817                                RSET *rset,
1818                                struct rset_key_control *kc)
1819 {
1820     int i;
1821     int sort_relation_value;
1822     AttrType sort_relation_type;
1823     Z_SortKeySpec *sks;
1824     Z_SortKey *sk;
1825     int oid[OID_SIZE];
1826     oident oe;
1827     char termz[20];
1828
1829     attr_init_APT(&sort_relation_type, zapt, 7);
1830     sort_relation_value = attr_find(&sort_relation_type, &attributeSet);
1831
1832     if (!sort_sequence->specs)
1833     {
1834         sort_sequence->num_specs = 10;
1835         sort_sequence->specs = (Z_SortKeySpec **)
1836             nmem_malloc(stream, sort_sequence->num_specs *
1837                          sizeof(*sort_sequence->specs));
1838         for (i = 0; i<sort_sequence->num_specs; i++)
1839             sort_sequence->specs[i] = 0;
1840     }
1841     if (zapt->term->which != Z_Term_general)
1842         i = 0;
1843     else
1844         i = atoi_n ((char *) zapt->term->u.general->buf,
1845                     zapt->term->u.general->len);
1846     if (i >= sort_sequence->num_specs)
1847         i = 0;
1848     sprintf(termz, "%d", i);
1849
1850     oe.proto = PROTO_Z3950;
1851     oe.oclass = CLASS_ATTSET;
1852     oe.value = attributeSet;
1853     if (!oid_ent_to_oid (&oe, oid))
1854         return ZEBRA_FAIL;
1855
1856     sks = (Z_SortKeySpec *) nmem_malloc(stream, sizeof(*sks));
1857     sks->sortElement = (Z_SortElement *)
1858         nmem_malloc(stream, sizeof(*sks->sortElement));
1859     sks->sortElement->which = Z_SortElement_generic;
1860     sk = sks->sortElement->u.generic = (Z_SortKey *)
1861         nmem_malloc(stream, sizeof(*sk));
1862     sk->which = Z_SortKey_sortAttributes;
1863     sk->u.sortAttributes = (Z_SortAttributes *)
1864         nmem_malloc(stream, sizeof(*sk->u.sortAttributes));
1865
1866     sk->u.sortAttributes->id = oid;
1867     sk->u.sortAttributes->list = zapt->attributes;
1868
1869     sks->sortRelation = (int *)
1870         nmem_malloc(stream, sizeof(*sks->sortRelation));
1871     if (sort_relation_value == 1)
1872         *sks->sortRelation = Z_SortKeySpec_ascending;
1873     else if (sort_relation_value == 2)
1874         *sks->sortRelation = Z_SortKeySpec_descending;
1875     else
1876         *sks->sortRelation = Z_SortKeySpec_ascending;
1877
1878     sks->caseSensitivity = (int *)
1879         nmem_malloc(stream, sizeof(*sks->caseSensitivity));
1880     *sks->caseSensitivity = 0;
1881
1882     sks->which = Z_SortKeySpec_null;
1883     sks->u.null = odr_nullval ();
1884     sort_sequence->specs[i] = sks;
1885     *rset = rset_create_null(rset_nmem, kc, 0);
1886     return ZEBRA_OK;
1887 }
1888
1889
1890 static int rpn_check_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1891                            oid_value attributeSet,
1892                            struct xpath_location_step *xpath, int max,
1893                            NMEM mem)
1894 {
1895     oid_value curAttributeSet = attributeSet;
1896     AttrType use;
1897     const char *use_string = 0;
1898
1899     attr_init_APT(&use, zapt, 1);
1900     attr_find_ex(&use, &curAttributeSet, &use_string);
1901
1902     if (!use_string || *use_string != '/')
1903         return -1;
1904
1905     return zebra_parse_xpath_str(use_string, xpath, max, mem);
1906 }
1907
1908
1909
1910 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
1911                         int reg_type, const char *term,
1912                         const char *xpath_use,
1913                         NMEM rset_nmem,
1914                         struct rset_key_control *kc)
1915 {
1916     RSET rset;
1917     struct grep_info grep_info;
1918     char term_dict[2048];
1919     char ord_buf[32];
1920     int prefix_len = 0;
1921     int ord = zebraExplain_lookup_attr_str(zh->reg->zei,
1922                                            zinfo_index_category_index,
1923                                            reg_type,
1924                                            xpath_use);
1925     int ord_len, i, r, max_pos;
1926     int term_type = Z_Term_characterString;
1927     const char *flags = "void";
1928
1929     if (grep_info_prepare(zh, 0 /* zapt */, &grep_info, '0') == ZEBRA_FAIL)
1930         return rset_create_null(rset_nmem, kc, 0);
1931
1932     if (ord < 0)
1933         return rset_create_null(rset_nmem, kc, 0);
1934     if (prefix_len)
1935         term_dict[prefix_len++] = '|';
1936     else
1937         term_dict[prefix_len++] = '(';
1938
1939     ord_len = key_SU_encode (ord, ord_buf);
1940     for (i = 0; i<ord_len; i++)
1941     {
1942         term_dict[prefix_len++] = 1;
1943         term_dict[prefix_len++] = ord_buf[i];
1944     }
1945     term_dict[prefix_len++] = ')';
1946     strcpy(term_dict+prefix_len, term);
1947
1948     grep_info.isam_p_indx = 0;
1949     r = dict_lookup_grep(zh->reg->dict, term_dict, 0,
1950                           &grep_info, &max_pos, 0, grep_handle);
1951     yaz_log(YLOG_DEBUG, "%s %d positions", term,
1952              grep_info.isam_p_indx);
1953     rset = rset_trunc(zh, grep_info.isam_p_buf,
1954                       grep_info.isam_p_indx, term, strlen(term),
1955                       flags, 1, term_type,rset_nmem,
1956                       kc, kc->scope, 0, reg_type, 0 /* hits_limit */,
1957                       0 /* term_ref_id_str */);
1958     grep_info_delete(&grep_info);
1959     return rset;
1960 }
1961
1962 static
1963 ZEBRA_RES rpn_search_xpath(ZebraHandle zh,
1964                            int num_bases, char **basenames,
1965                            NMEM stream, const char *rank_type, RSET rset,
1966                            int xpath_len, struct xpath_location_step *xpath,
1967                            NMEM rset_nmem,
1968                            RSET *rset_out,
1969                            struct rset_key_control *kc)
1970 {
1971     int base_no;
1972     int i;
1973     int always_matches = rset ? 0 : 1;
1974
1975     if (xpath_len < 0)
1976     {
1977         *rset_out = rset;
1978         return ZEBRA_OK;
1979     }
1980
1981     yaz_log(YLOG_DEBUG, "xpath len=%d", xpath_len);
1982     for (i = 0; i<xpath_len; i++)
1983     {
1984         yaz_log(log_level_rpn, "XPATH %d %s", i, xpath[i].part);
1985
1986     }
1987
1988     /*
1989       //a    ->    a/.*
1990       //a/b  ->    b/a/.*
1991       /a     ->    a/
1992       /a/b   ->    b/a/
1993
1994       /      ->    none
1995
1996    a[@attr = value]/b[@other = othervalue]
1997
1998  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
1999  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
2000  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
2001  /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y)
2002  /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y)
2003  /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x)
2004
2005     */
2006
2007     dict_grep_cmap (zh->reg->dict, 0, 0);
2008
2009     for (base_no = 0; base_no < num_bases; base_no++)
2010     {
2011         int level = xpath_len;
2012         int first_path = 1;
2013
2014         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2015         {
2016             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
2017                            basenames[base_no]);
2018             *rset_out = rset;
2019             return ZEBRA_FAIL;
2020         }
2021         while (--level >= 0)
2022         {
2023             WRBUF xpath_rev = wrbuf_alloc();
2024             int i;
2025             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
2026
2027             for (i = level; i >= 1; --i)
2028             {
2029                 const char *cp = xpath[i].part;
2030                 if (*cp)
2031                 {
2032                     for (; *cp; cp++)
2033                     {
2034                         if (*cp == '*')
2035                             wrbuf_puts(xpath_rev, "[^/]*");
2036                         else if (*cp == ' ')
2037                             wrbuf_puts(xpath_rev, "\001 ");
2038                         else
2039                             wrbuf_putc(xpath_rev, *cp);
2040
2041                         /* wrbuf_putc does not null-terminate , but
2042                            wrbuf_puts below ensures it does.. so xpath_rev
2043                            is OK iff length is > 0 */
2044                     }
2045                     wrbuf_puts(xpath_rev, "/");
2046                 }
2047                 else if (i == 1)  /* // case */
2048                     wrbuf_puts(xpath_rev, ".*");
2049             }
2050             if (xpath[level].predicate &&
2051                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
2052                 xpath[level].predicate->u.relation.name[0])
2053             {
2054                 WRBUF wbuf = wrbuf_alloc();
2055                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
2056                 if (xpath[level].predicate->u.relation.value)
2057                 {
2058                     const char *cp = xpath[level].predicate->u.relation.value;
2059                     wrbuf_putc(wbuf, '=');
2060
2061                     while (*cp)
2062                     {
2063                         if (strchr(REGEX_CHARS, *cp))
2064                             wrbuf_putc(wbuf, '\\');
2065                         wrbuf_putc(wbuf, *cp);
2066                         cp++;
2067                     }
2068                 }
2069                 wrbuf_puts(wbuf, "");
2070                 rset_attr = xpath_trunc(
2071                     zh, stream, '0', wrbuf_buf(wbuf), ZEBRA_XPATH_ATTR_NAME,
2072                     rset_nmem, kc);
2073                 wrbuf_free(wbuf, 1);
2074             }
2075             else
2076             {
2077                 if (!first_path)
2078                 {
2079                     wrbuf_free(xpath_rev, 1);
2080                     continue;
2081                 }
2082             }
2083             yaz_log(log_level_rpn, "xpath_rev (%d) = %.*s", level,
2084                     wrbuf_len(xpath_rev), wrbuf_buf(xpath_rev));
2085             if (wrbuf_len(xpath_rev))
2086             {
2087                 rset_start_tag = xpath_trunc(zh, stream, '0',
2088                                              wrbuf_buf(xpath_rev),
2089                                              ZEBRA_XPATH_ELM_BEGIN,
2090                                              rset_nmem, kc);
2091                 if (always_matches)
2092                     rset = rset_start_tag;
2093                 else
2094                 {
2095                     rset_end_tag = xpath_trunc(zh, stream, '0',
2096                                                wrbuf_buf(xpath_rev),
2097                                                ZEBRA_XPATH_ELM_END,
2098                                                rset_nmem, kc);
2099
2100                     rset = rset_create_between(rset_nmem, kc, kc->scope,
2101                                                rset_start_tag, rset,
2102                                                rset_end_tag, rset_attr);
2103                 }
2104             }
2105             wrbuf_free(xpath_rev, 1);
2106             first_path = 0;
2107         }
2108     }
2109     *rset_out = rset;
2110     return ZEBRA_OK;
2111 }
2112
2113 #define MAX_XPATH_STEPS 10
2114
2115 static ZEBRA_RES rpn_search_APT(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2116                                 oid_value attributeSet, NMEM stream,
2117                                 Z_SortKeySpecList *sort_sequence,
2118                                 int num_bases, char **basenames,
2119                                 NMEM rset_nmem,
2120                                 RSET *rset,
2121                                 struct rset_key_control *kc)
2122 {
2123     ZEBRA_RES res = ZEBRA_OK;
2124     unsigned reg_id;
2125     char *search_type = NULL;
2126     char rank_type[128];
2127     int complete_flag;
2128     int sort_flag;
2129     char termz[IT_MAX_WORD+1];
2130     int xpath_len;
2131     const char *xpath_use = 0;
2132     struct xpath_location_step xpath[MAX_XPATH_STEPS];
2133
2134     if (!log_level_set)
2135     {
2136         log_level_rpn = yaz_log_module_level("rpn");
2137         log_level_set = 1;
2138     }
2139     zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2140                     rank_type, &complete_flag, &sort_flag);
2141
2142     yaz_log(YLOG_DEBUG, "reg_id=%c", reg_id);
2143     yaz_log(YLOG_DEBUG, "complete_flag=%d", complete_flag);
2144     yaz_log(YLOG_DEBUG, "search_type=%s", search_type);
2145     yaz_log(YLOG_DEBUG, "rank_type=%s", rank_type);
2146
2147     if (zapt_term_to_utf8(zh, zapt, termz) == ZEBRA_FAIL)
2148         return ZEBRA_FAIL;
2149
2150     if (sort_flag)
2151         return rpn_sort_spec(zh, zapt, attributeSet, stream, sort_sequence,
2152                              rank_type, rset_nmem, rset, kc);
2153     /* consider if an X-Path query is used */
2154     xpath_len = rpn_check_xpath(zh, zapt, attributeSet,
2155                                 xpath, MAX_XPATH_STEPS, stream);
2156     if (xpath_len >= 0)
2157     {
2158         if (xpath[xpath_len-1].part[0] == '@')
2159             xpath_use = ZEBRA_XPATH_ATTR_CDATA;  /* last step is attribute  */
2160         else
2161             xpath_use = ZEBRA_XPATH_CDATA;  /* searching for cdata */
2162
2163         if (1)
2164         {
2165             AttrType relation;
2166             int relation_value;
2167
2168             attr_init_APT(&relation, zapt, 2);
2169             relation_value = attr_find(&relation, NULL);
2170
2171             if (relation_value == 103) /* alwaysmatches */
2172             {
2173                 *rset = 0; /* signal no "term" set */
2174                 return rpn_search_xpath(zh, num_bases, basenames,
2175                                         stream, rank_type, *rset,
2176                                         xpath_len, xpath, rset_nmem, rset, kc);
2177             }
2178         }
2179     }
2180
2181     /* search using one of the various search type strategies
2182        termz is our UTF-8 search term
2183        attributeSet is top-level default attribute set
2184        stream is ODR for search
2185        reg_id is the register type
2186        complete_flag is 1 for complete subfield, 0 for incomplete
2187        xpath_use is use-attribute to be used for X-Path search, 0 for none
2188     */
2189     if (!strcmp(search_type, "phrase"))
2190     {
2191         res = rpn_search_APT_phrase(zh, zapt, termz, attributeSet, stream,
2192                                     reg_id, complete_flag, rank_type,
2193                                     xpath_use,
2194                                     num_bases, basenames, rset_nmem,
2195                                     rset, kc);
2196     }
2197     else if (!strcmp(search_type, "and-list"))
2198     {
2199         res = rpn_search_APT_and_list(zh, zapt, termz, attributeSet, stream,
2200                                       reg_id, complete_flag, rank_type,
2201                                       xpath_use,
2202                                       num_bases, basenames, rset_nmem,
2203                                       rset, kc);
2204     }
2205     else if (!strcmp(search_type, "or-list"))
2206     {
2207         res = rpn_search_APT_or_list(zh, zapt, termz, attributeSet, stream,
2208                                      reg_id, complete_flag, rank_type,
2209                                      xpath_use,
2210                                      num_bases, basenames, rset_nmem,
2211                                      rset, kc);
2212     }
2213     else if (!strcmp(search_type, "local"))
2214     {
2215         res = rpn_search_APT_local(zh, zapt, termz, attributeSet, stream,
2216                                    rank_type, rset_nmem, rset, kc);
2217     }
2218     else if (!strcmp(search_type, "numeric"))
2219     {
2220         res = rpn_search_APT_numeric(zh, zapt, termz, attributeSet, stream,
2221                                      reg_id, complete_flag, rank_type,
2222                                      xpath_use,
2223                                      num_bases, basenames, rset_nmem,
2224                                      rset, kc);
2225     }
2226     else
2227     {
2228         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2229         res = ZEBRA_FAIL;
2230     }
2231     if (res != ZEBRA_OK)
2232         return res;
2233     if (!*rset)
2234         return ZEBRA_FAIL;
2235     return rpn_search_xpath(zh, num_bases, basenames,
2236                             stream, rank_type, *rset,
2237                             xpath_len, xpath, rset_nmem, rset, kc);
2238 }
2239
2240 static ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2241                                       oid_value attributeSet,
2242                                       NMEM stream, NMEM rset_nmem,
2243                                       Z_SortKeySpecList *sort_sequence,
2244                                       int num_bases, char **basenames,
2245                                       RSET **result_sets, int *num_result_sets,
2246                                       Z_Operator *parent_op,
2247                                       struct rset_key_control *kc);
2248
2249 ZEBRA_RES rpn_search_top(ZebraHandle zh, Z_RPNStructure *zs,
2250                          oid_value attributeSet,
2251                          NMEM stream, NMEM rset_nmem,
2252                          Z_SortKeySpecList *sort_sequence,
2253                          int num_bases, char **basenames,
2254                          RSET *result_set)
2255 {
2256     RSET *result_sets = 0;
2257     int num_result_sets = 0;
2258     ZEBRA_RES res;
2259     struct rset_key_control *kc = zebra_key_control_create(zh);
2260
2261     res = rpn_search_structure(zh, zs, attributeSet,
2262                                stream, rset_nmem,
2263                                sort_sequence,
2264                                num_bases, basenames,
2265                                &result_sets, &num_result_sets,
2266                                0 /* no parent op */,
2267                                kc);
2268     if (res != ZEBRA_OK)
2269     {
2270         int i;
2271         for (i = 0; i<num_result_sets; i++)
2272             rset_delete(result_sets[i]);
2273         *result_set = 0;
2274     }
2275     else
2276     {
2277         assert(num_result_sets == 1);
2278         assert(result_sets);
2279         assert(*result_sets);
2280         *result_set = *result_sets;
2281     }
2282     (*kc->dec)(kc);
2283     return res;
2284 }
2285
2286 ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2287                                oid_value attributeSet,
2288                                NMEM stream, NMEM rset_nmem,
2289                                Z_SortKeySpecList *sort_sequence,
2290                                int num_bases, char **basenames,
2291                                RSET **result_sets, int *num_result_sets,
2292                                Z_Operator *parent_op,
2293                                struct rset_key_control *kc)
2294 {
2295     *num_result_sets = 0;
2296     if (zs->which == Z_RPNStructure_complex)
2297     {
2298         ZEBRA_RES res;
2299         Z_Operator *zop = zs->u.complex->roperator;
2300         RSET *result_sets_l = 0;
2301         int num_result_sets_l = 0;
2302         RSET *result_sets_r = 0;
2303         int num_result_sets_r = 0;
2304
2305         res = rpn_search_structure(zh, zs->u.complex->s1,
2306                                    attributeSet, stream, rset_nmem,
2307                                    sort_sequence,
2308                                    num_bases, basenames,
2309                                    &result_sets_l, &num_result_sets_l,
2310                                    zop, kc);
2311         if (res != ZEBRA_OK)
2312         {
2313             int i;
2314             for (i = 0; i<num_result_sets_l; i++)
2315                 rset_delete(result_sets_l[i]);
2316             return res;
2317         }
2318         res = rpn_search_structure(zh, zs->u.complex->s2,
2319                                    attributeSet, stream, rset_nmem,
2320                                    sort_sequence,
2321                                    num_bases, basenames,
2322                                    &result_sets_r, &num_result_sets_r,
2323                                    zop, kc);
2324         if (res != ZEBRA_OK)
2325         {
2326             int i;
2327             for (i = 0; i<num_result_sets_l; i++)
2328                 rset_delete(result_sets_l[i]);
2329             for (i = 0; i<num_result_sets_r; i++)
2330                 rset_delete(result_sets_r[i]);
2331             return res;
2332         }
2333
2334         /* make a new list of result for all children */
2335         *num_result_sets = num_result_sets_l + num_result_sets_r;
2336         *result_sets = nmem_malloc(stream, *num_result_sets *
2337                                    sizeof(**result_sets));
2338         memcpy(*result_sets, result_sets_l,
2339                num_result_sets_l * sizeof(**result_sets));
2340         memcpy(*result_sets + num_result_sets_l, result_sets_r,
2341                num_result_sets_r * sizeof(**result_sets));
2342
2343         if (!parent_op || parent_op->which != zop->which
2344             || (zop->which != Z_Operator_and &&
2345                 zop->which != Z_Operator_or))
2346         {
2347             /* parent node different from this one (or non-present) */
2348             /* we must combine result sets now */
2349             RSET rset;
2350             switch (zop->which)
2351             {
2352             case Z_Operator_and:
2353                 rset = rset_create_and(rset_nmem, kc,
2354                                        kc->scope,
2355                                        *num_result_sets, *result_sets);
2356                 break;
2357             case Z_Operator_or:
2358                 rset = rset_create_or(rset_nmem, kc,
2359                                       kc->scope, 0, /* termid */
2360                                       *num_result_sets, *result_sets);
2361                 break;
2362             case Z_Operator_and_not:
2363                 rset = rset_create_not(rset_nmem, kc,
2364                                        kc->scope,
2365                                        (*result_sets)[0],
2366                                        (*result_sets)[1]);
2367                 break;
2368             case Z_Operator_prox:
2369                 if (zop->u.prox->which != Z_ProximityOperator_known)
2370                 {
2371                     zebra_setError(zh,
2372                                    YAZ_BIB1_UNSUPP_PROX_UNIT_CODE,
2373                                    0);
2374                     return ZEBRA_FAIL;
2375                 }
2376                 if (*zop->u.prox->u.known != Z_ProxUnit_word)
2377                 {
2378                     zebra_setError_zint(zh,
2379                                         YAZ_BIB1_UNSUPP_PROX_UNIT_CODE,
2380                                         *zop->u.prox->u.known);
2381                     return ZEBRA_FAIL;
2382                 }
2383                 else
2384                 {
2385                     rset = rset_create_prox(rset_nmem, kc,
2386                                             kc->scope,
2387                                             *num_result_sets, *result_sets,
2388                                             *zop->u.prox->ordered,
2389                                             (!zop->u.prox->exclusion ?
2390                                              0 : *zop->u.prox->exclusion),
2391                                             *zop->u.prox->relationType,
2392                                             *zop->u.prox->distance );
2393                 }
2394                 break;
2395             default:
2396                 zebra_setError(zh, YAZ_BIB1_OPERATOR_UNSUPP, 0);
2397                 return ZEBRA_FAIL;
2398             }
2399             *num_result_sets = 1;
2400             *result_sets = nmem_malloc(stream, *num_result_sets *
2401                                        sizeof(**result_sets));
2402             (*result_sets)[0] = rset;
2403         }
2404     }
2405     else if (zs->which == Z_RPNStructure_simple)
2406     {
2407         RSET rset;
2408         ZEBRA_RES res;
2409
2410         if (zs->u.simple->which == Z_Operand_APT)
2411         {
2412             yaz_log(YLOG_DEBUG, "rpn_search_APT");
2413             res = rpn_search_APT(zh, zs->u.simple->u.attributesPlusTerm,
2414                                  attributeSet, stream, sort_sequence,
2415                                  num_bases, basenames, rset_nmem, &rset,
2416                                  kc);
2417             if (res != ZEBRA_OK)
2418                 return res;
2419         }
2420         else if (zs->u.simple->which == Z_Operand_resultSetId)
2421         {
2422             yaz_log(YLOG_DEBUG, "rpn_search_ref");
2423             rset = resultSetRef(zh, zs->u.simple->u.resultSetId);
2424             if (!rset)
2425             {
2426                 zebra_setError(zh,
2427                                YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
2428                                zs->u.simple->u.resultSetId);
2429                 return ZEBRA_FAIL;
2430             }
2431             rset_dup(rset);
2432         }
2433         else
2434         {
2435             zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2436             return ZEBRA_FAIL;
2437         }
2438         *num_result_sets = 1;
2439         *result_sets = nmem_malloc(stream, *num_result_sets *
2440                                    sizeof(**result_sets));
2441         (*result_sets)[0] = rset;
2442     }
2443     else
2444     {
2445         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2446         return ZEBRA_FAIL;
2447     }
2448     return ZEBRA_OK;
2449 }
2450
2451 struct scan_info_entry {
2452     char *term;
2453     ISAM_P isam_p;
2454 };
2455
2456 struct scan_info {
2457     struct scan_info_entry *list;
2458     ODR odr;
2459     int before, after;
2460     char prefix[20];
2461 };
2462
2463 static int scan_handle (char *name, const char *info, int pos, void *client)
2464 {
2465     int len_prefix, idx;
2466     struct scan_info *scan_info = (struct scan_info *) client;
2467
2468     len_prefix = strlen(scan_info->prefix);
2469     if (memcmp (name, scan_info->prefix, len_prefix))
2470         return 1;
2471     if (pos > 0)
2472         idx = scan_info->after - pos + scan_info->before;
2473     else
2474         idx = - pos - 1;
2475
2476     if (idx < 0)
2477         return 0;
2478     scan_info->list[idx].term = (char *)
2479         odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
2480     strcpy(scan_info->list[idx].term, name + len_prefix);
2481     assert (*info == sizeof(ISAM_P));
2482     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
2483     return 0;
2484 }
2485
2486 void zebra_term_untrans_iconv(ZebraHandle zh, NMEM stream, int reg_type,
2487                               char **dst, const char *src)
2488 {
2489     char term_src[IT_MAX_WORD];
2490     char term_dst[IT_MAX_WORD];
2491
2492     zebra_term_untrans (zh, reg_type, term_src, src);
2493
2494     if (zh->iconv_from_utf8 != 0)
2495     {
2496         int len;
2497         char *inbuf = term_src;
2498         size_t inleft = strlen(term_src);
2499         char *outbuf = term_dst;
2500         size_t outleft = sizeof(term_dst)-1;
2501         size_t ret;
2502
2503         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2504                          &outbuf, &outleft);
2505         if (ret == (size_t)(-1))
2506             len = 0;
2507         else
2508             len = outbuf - term_dst;
2509         *dst = nmem_malloc(stream, len + 1);
2510         if (len > 0)
2511             memcpy (*dst, term_dst, len);
2512         (*dst)[len] = '\0';
2513     }
2514     else
2515         *dst = nmem_strdup(stream, term_src);
2516 }
2517
2518 static void count_set(ZebraHandle zh, RSET rset, zint *count)
2519 {
2520     zint psysno = 0;
2521     struct it_key key;
2522     RSFD rfd;
2523
2524     yaz_log(YLOG_DEBUG, "count_set");
2525
2526     rset->hits_limit = zh->approx_limit;
2527
2528     *count = 0;
2529     rfd = rset_open(rset, RSETF_READ);
2530     while (rset_read(rfd, &key,0 /* never mind terms */))
2531     {
2532         if (key.mem[0] != psysno)
2533         {
2534             psysno = key.mem[0];
2535             if (rfd->counted_items >= rset->hits_limit)
2536                 break;
2537         }
2538     }
2539     rset_close (rfd);
2540     *count = rset->hits_count;
2541 }
2542
2543 #define RPN_MAX_ORDS 32
2544
2545 ZEBRA_RES rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2546                    oid_value attributeset,
2547                    int num_bases, char **basenames,
2548                    int *position, int *num_entries, ZebraScanEntry **list,
2549                    int *is_partial, RSET limit_set, int return_zero)
2550 {
2551     int i;
2552     int pos = *position;
2553     int num = *num_entries;
2554     int before;
2555     int after;
2556     int base_no;
2557     char termz[IT_MAX_WORD+20];
2558     struct scan_info *scan_info_array;
2559     ZebraScanEntry *glist;
2560     int ords[RPN_MAX_ORDS], ord_no = 0;
2561     int ptr[RPN_MAX_ORDS];
2562
2563     unsigned index_type;
2564     char *search_type = NULL;
2565     char rank_type[128];
2566     int complete_flag;
2567     int sort_flag;
2568     NMEM rset_nmem = NULL;
2569     struct rset_key_control *kc = 0;
2570
2571     *list = 0;
2572     *is_partial = 0;
2573
2574     if (attributeset == VAL_NONE)
2575         attributeset = VAL_BIB1;
2576
2577     if (!limit_set)
2578     {
2579         AttrType termset;
2580         int termset_value_numeric;
2581         const char *termset_value_string;
2582         attr_init_APT(&termset, zapt, 8);
2583         termset_value_numeric =
2584             attr_find_ex(&termset, NULL, &termset_value_string);
2585         if (termset_value_numeric != -1)
2586         {
2587             char resname[32];
2588             const char *termset_name = 0;
2589
2590             if (termset_value_numeric != -2)
2591             {
2592
2593                 sprintf(resname, "%d", termset_value_numeric);
2594                 termset_name = resname;
2595             }
2596             else
2597                 termset_name = termset_value_string;
2598
2599             limit_set = resultSetRef (zh, termset_name);
2600         }
2601     }
2602
2603     yaz_log(YLOG_DEBUG, "position = %d, num = %d set=%d",
2604             pos, num, attributeset);
2605
2606     if (zebra_maps_attr(zh->reg->zebra_maps, zapt, &index_type, &search_type,
2607                         rank_type, &complete_flag, &sort_flag))
2608     {
2609         *num_entries = 0;
2610         zebra_setError(zh, YAZ_BIB1_UNSUPP_ATTRIBUTE_TYPE, 0);
2611         return ZEBRA_FAIL;
2612     }
2613     for (base_no = 0; base_no < num_bases && ord_no < RPN_MAX_ORDS; base_no++)
2614     {
2615         int ord;
2616
2617         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2618         {
2619             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
2620                            basenames[base_no]);
2621             *num_entries = 0;
2622             return ZEBRA_FAIL;
2623         }
2624         if (zebra_apt_get_ord(zh, zapt, index_type, 0, attributeset, &ord)
2625             != ZEBRA_OK)
2626             continue;
2627         ords[ord_no++] = ord;
2628     }
2629     if (ord_no == 0)
2630     {
2631         *num_entries = 0;
2632         return ZEBRA_OK;
2633     }
2634     /* prepare dictionary scanning */
2635     if (num < 1)
2636     {
2637         *num_entries = 0;
2638         return ZEBRA_OK;
2639     }
2640     before = pos-1;
2641     if (before < 0)
2642         before = 0;
2643     after = 1+num-pos;
2644     if (after < 0)
2645         after = 0;
2646     yaz_log(YLOG_DEBUG, "rpn_scan pos=%d num=%d before=%d "
2647             "after=%d before+after=%d",
2648             pos, num, before, after, before+after);
2649     scan_info_array = (struct scan_info *)
2650         odr_malloc(stream, ord_no * sizeof(*scan_info_array));
2651     for (i = 0; i < ord_no; i++)
2652     {
2653         int j, prefix_len = 0;
2654         int before_tmp = before, after_tmp = after;
2655         struct scan_info *scan_info = scan_info_array + i;
2656         struct rpn_char_map_info rcmi;
2657
2658         rpn_char_map_prepare (zh->reg, index_type, &rcmi);
2659
2660         scan_info->before = before;
2661         scan_info->after = after;
2662         scan_info->odr = stream;
2663
2664         scan_info->list = (struct scan_info_entry *)
2665             odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
2666         for (j = 0; j<before+after; j++)
2667             scan_info->list[j].term = NULL;
2668
2669         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2670         termz[prefix_len] = 0;
2671         strcpy(scan_info->prefix, termz);
2672
2673         if (trans_scan_term(zh, zapt, termz+prefix_len, index_type) ==
2674             ZEBRA_FAIL)
2675             return ZEBRA_FAIL;
2676
2677         dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
2678                   scan_info, scan_handle);
2679     }
2680     glist = (ZebraScanEntry *)
2681         odr_malloc(stream, (before+after)*sizeof(*glist));
2682
2683     rset_nmem = nmem_create();
2684     kc = zebra_key_control_create(zh);
2685
2686     /* consider terms after main term */
2687     for (i = 0; i < ord_no; i++)
2688         ptr[i] = before;
2689
2690     *is_partial = 0;
2691     for (i = 0; i<after; i++)
2692     {
2693         int j, j0 = -1;
2694         const char *mterm = NULL;
2695         const char *tst;
2696         RSET rset = 0;
2697         int lo = i + pos-1; /* offset in result list */
2698
2699         /* find: j0 is the first of the minimal values */
2700         for (j = 0; j < ord_no; j++)
2701         {
2702             if (ptr[j] < before+after && ptr[j] >= 0 &&
2703                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2704                 (!mterm || strcmp (tst, mterm) < 0))
2705             {
2706                 j0 = j;
2707                 mterm = tst;
2708             }
2709         }
2710         if (j0 == -1)
2711             break;  /* no value found, stop */
2712
2713         /* get result set for first one , but only if it's within bounds */
2714         if (lo >= 0)
2715         {
2716             /* get result set for first term */
2717             zebra_term_untrans_iconv(zh, stream->mem, index_type,
2718                                      &glist[lo].term, mterm);
2719             rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2720                               glist[lo].term, strlen(glist[lo].term),
2721                               NULL, 0, zapt->term->which, rset_nmem,
2722                               kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2723                               0 /* term_ref_id_str */);
2724         }
2725         ptr[j0]++; /* move index for this set .. */
2726         /* get result set for remaining scan terms */
2727         for (j = j0+1; j<ord_no; j++)
2728         {
2729             if (ptr[j] < before+after && ptr[j] >= 0 &&
2730                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2731                 !strcmp (tst, mterm))
2732             {
2733                 if (lo >= 0)
2734                 {
2735                     RSET rsets[2];
2736
2737                     rsets[0] = rset;
2738                     rsets[1] =
2739                         rset_trunc(
2740                             zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
2741                             glist[lo].term,
2742                             strlen(glist[lo].term), NULL, 0,
2743                             zapt->term->which,rset_nmem,
2744                             kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2745                             0 /* term_ref_id_str */ );
2746                     rset = rset_create_or(rset_nmem, kc,
2747                                           kc->scope, 0 /* termid */,
2748                                           2, rsets);
2749                 }
2750                 ptr[j]++;
2751             }
2752         }
2753         if (lo >= 0)
2754         {
2755             zint count;
2756             /* merge with limit_set if given */
2757             if (limit_set)
2758             {
2759                 RSET rsets[2];
2760                 rsets[0] = rset;
2761                 rsets[1] = rset_dup(limit_set);
2762
2763                 rset = rset_create_and(rset_nmem, kc, kc->scope, 2, rsets);
2764             }
2765             /* count it */
2766             count_set(zh, rset, &count);
2767             glist[lo].occurrences = count;
2768             rset_delete(rset);
2769         }
2770     }
2771     if (i < after)
2772     {
2773         *num_entries -= (after-i);
2774         *is_partial = 1;
2775         if (*num_entries < 0)
2776         {
2777             (*kc->dec)(kc);
2778             nmem_destroy(rset_nmem);
2779             *num_entries = 0;
2780             return ZEBRA_OK;
2781         }
2782     }
2783     /* consider terms before main term */
2784     for (i = 0; i<ord_no; i++)
2785         ptr[i] = 0;
2786
2787     for (i = 0; i<before; i++)
2788     {
2789         int j, j0 = -1;
2790         const char *mterm = NULL;
2791         const char *tst;
2792         RSET rset;
2793         int lo = before-1-i; /* offset in result list */
2794         zint count;
2795
2796         for (j = 0; j <ord_no; j++)
2797         {
2798             if (ptr[j] < before && ptr[j] >= 0 &&
2799                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2800                 (!mterm || strcmp (tst, mterm) > 0))
2801             {
2802                 j0 = j;
2803                     mterm = tst;
2804             }
2805         }
2806         if (j0 == -1)
2807             break;
2808
2809         zebra_term_untrans_iconv(zh, stream->mem, index_type,
2810                                  &glist[lo].term, mterm);
2811
2812         rset = rset_trunc
2813             (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
2814              glist[lo].term, strlen(glist[lo].term),
2815              NULL, 0, zapt->term->which, rset_nmem,
2816              kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2817              0 /* term_ref_id_str */);
2818
2819         ptr[j0]++;
2820
2821         for (j = j0+1; j<ord_no; j++)
2822         {
2823             if (ptr[j] < before && ptr[j] >= 0 &&
2824                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2825                 !strcmp (tst, mterm))
2826             {
2827                 RSET rsets[2];
2828
2829                 rsets[0] = rset;
2830                 rsets[1] = rset_trunc(
2831                     zh,
2832                     &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
2833                     glist[lo].term,
2834                     strlen(glist[lo].term), NULL, 0,
2835                     zapt->term->which, rset_nmem,
2836                     kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2837                     0 /* term_ref_id_str */);
2838                 rset = rset_create_or(rset_nmem, kc,
2839                                       kc->scope, 0 /* termid */, 2, rsets);
2840
2841                 ptr[j]++;
2842             }
2843         }
2844         if (limit_set)
2845         {
2846             RSET rsets[2];
2847             rsets[0] = rset;
2848             rsets[1] = rset_dup(limit_set);
2849
2850             rset = rset_create_and(rset_nmem, kc, kc->scope, 2, rsets);
2851         }
2852         count_set(zh, rset, &count);
2853         glist[lo].occurrences = count;
2854         rset_delete (rset);
2855     }
2856     (*kc->dec)(kc);
2857     nmem_destroy(rset_nmem);
2858     i = before-i;
2859     if (i)
2860     {
2861         *is_partial = 1;
2862         *position -= i;
2863         *num_entries -= i;
2864         if (*num_entries <= 0)
2865         {
2866             *num_entries = 0;
2867             return ZEBRA_OK;
2868         }
2869     }
2870
2871     *list = glist + i;               /* list is set to first 'real' entry */
2872
2873     yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
2874             *position, *num_entries);
2875     return ZEBRA_OK;
2876 }
2877
2878 /*
2879  * Local variables:
2880  * c-basic-offset: 4
2881  * indent-tabs-mode: nil
2882  * End:
2883  * vim: shiftwidth=4 tabstop=8 expandtab
2884  */
2885