index/zrpn.c

   1 /* $Id: zrpn.c,v 1.228 2006-09-08 14:40:53 adam Exp $
   2    Copyright (C) 1995-2006
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with this program; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #ifdef WIN32
  26 #include <io.h>
  27 #endif
  28 #if HAVE_UNISTD_H
  29 #include <unistd.h>
  30 #endif
  31 #include <ctype.h>
  32
  33 #include <yaz/diagbib1.h>
  34 #include "index.h"
  35 #include <zebra_xpath.h>
  36 #include <attrfind.h>
  37 #include <charmap.h>
  38 #include <rset.h>
  39
  40 struct rpn_char_map_info
  41 {
  42     ZebraMaps zm;
  43     int reg_type;
  44 };
  45
  46 static int log_level_set = 0;
  47 static int log_level_rpn = 0;
  48
  49 #define TERMSET_DISABLE 1
  50
  51 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  52 {
  53     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  54     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  55 #if 0
  56     if (out && *out)
  57     {
  58         const char *outp = *out;
  59         yaz_log(YLOG_LOG, "---");
  60         while (*outp)
  61         {
  62             yaz_log(YLOG_LOG, "%02X", *outp);
  63             outp++;
  64         }
  65     }
  66 #endif
  67     return out;
  68 }
  69
  70 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  71                                  struct rpn_char_map_info *map_info)
  72 {
  73     map_info->zm = reg->zebra_maps;
  74     map_info->reg_type = reg_type;
  75     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  76 }
  77
  78 #define TERM_COUNT
  79
  80 struct grep_info {
  81 #ifdef TERM_COUNT
  82     int *term_no;
  83 #endif
  84     ISAM_P *isam_p_buf;
  85     int isam_p_size;
  86     int isam_p_indx;
  87     ZebraHandle zh;
  88     int reg_type;
  89     ZebraSet termset;
  90 };
  91
  92 void zebra_term_untrans(ZebraHandle zh, int reg_type,
  93                         char *dst, const char *src)
  94 {
  95     int len = 0;
  96     while (*src)
  97     {
  98         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
  99                                            reg_type, &src);
 100         if (!cp)
 101         {
 102             if (len < IT_MAX_WORD-1)
 103                 dst[len++] = *src;
 104             src++;
 105         }
 106         else
 107             while (*cp && len < IT_MAX_WORD-1)
 108                 dst[len++] = *cp++;
 109     }
 110     dst[len] = '\0';
 111 }
 112
 113 static void add_isam_p(const char *name, const char *info,
 114                        struct grep_info *p)
 115 {
 116     if (!log_level_set)
 117     {
 118         log_level_rpn = yaz_log_module_level("rpn");
 119         log_level_set = 1;
 120     }
 121     if (p->isam_p_indx == p->isam_p_size)
 122     {
 123         ISAM_P *new_isam_p_buf;
 124 #ifdef TERM_COUNT
 125         int *new_term_no;
 126 #endif
 127         p->isam_p_size = 2*p->isam_p_size + 100;
 128         new_isam_p_buf = (ISAM_P *) xmalloc(sizeof(*new_isam_p_buf) *
 129                                             p->isam_p_size);
 130         if (p->isam_p_buf)
 131         {
 132             memcpy(new_isam_p_buf, p->isam_p_buf,
 133                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 134             xfree(p->isam_p_buf);
 135         }
 136         p->isam_p_buf = new_isam_p_buf;
 137
 138 #ifdef TERM_COUNT
 139         new_term_no = (int *) xmalloc(sizeof(*new_term_no) * p->isam_p_size);
 140         if (p->term_no)
 141         {
 142             memcpy(new_term_no, p->isam_p_buf,
 143                     p->isam_p_indx * sizeof(*p->term_no));
 144             xfree(p->term_no);
 145         }
 146         p->term_no = new_term_no;
 147 #endif
 148     }
 149     assert(*info == sizeof(*p->isam_p_buf));
 150     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 151
 152     if (p->termset)
 153     {
 154         const char *db;
 155         char term_tmp[IT_MAX_WORD];
 156         int ord = 0;
 157         const char *index_name;
 158         int len = key_SU_decode (&ord, (const unsigned char *) name);
 159
 160         zebra_term_untrans  (p->zh, p->reg_type, term_tmp, name+len);
 161         yaz_log(log_level_rpn, "grep: %d %c %s", ord, name[len], term_tmp);
 162         zebraExplain_lookup_ord(p->zh->reg->zei,
 163                                 ord, 0 /* index_type */, &db, &index_name);
 164         yaz_log(log_level_rpn, "grep:  db=%s index=%s", db, index_name);
 165
 166         resultSetAddTerm(p->zh, p->termset, name[len], db,
 167                          index_name, term_tmp);
 168     }
 169     (p->isam_p_indx)++;
 170 }
 171
 172 static int grep_handle(char *name, const char *info, void *p)
 173 {
 174     add_isam_p(name, info, (struct grep_info *) p);
 175     return 0;
 176 }
 177
 178 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 179                     const char *ct1, const char *ct2, int first)
 180 {
 181     const char *s1, *s0 = *src;
 182     const char **map;
 183
 184     /* skip white space */
 185     while (*s0)
 186     {
 187         if (ct1 && strchr(ct1, *s0))
 188             break;
 189         if (ct2 && strchr(ct2, *s0))
 190             break;
 191         s1 = s0;
 192         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 193         if (**map != *CHR_SPACE)
 194             break;
 195         s0 = s1;
 196     }
 197     *src = s0;
 198     return *s0;
 199 }
 200
 201
 202 static void esc_str(char *out_buf, size_t out_size,
 203                     const char *in_buf, int in_size)
 204 {
 205     int k;
 206
 207     assert(out_buf);
 208     assert(in_buf);
 209     assert(out_size > 20);
 210     *out_buf = '\0';
 211     for (k = 0; k<in_size; k++)
 212     {
 213         int c = in_buf[k] & 0xff;
 214         int pc;
 215         if (c < 32 || c > 126)
 216             pc = '?';
 217         else
 218             pc = c;
 219         sprintf(out_buf +strlen(out_buf), "%02X:%c  ", c, pc);
 220         if (strlen(out_buf) > out_size-20)
 221         {
 222             strcat(out_buf, "..");
 223             break;
 224         }
 225     }
 226 }
 227
 228 #define REGEX_CHARS " []()|.*+?!"
 229
 230 /* term_100: handle term, where trunc = none(no operators at all) */
 231 static int term_100(ZebraMaps zebra_maps, int reg_type,
 232                     const char **src, char *dst, int space_split,
 233                     char *dst_term)
 234 {
 235     const char *s0;
 236     const char **map;
 237     int i = 0;
 238     int j = 0;
 239
 240     const char *space_start = 0;
 241     const char *space_end = 0;
 242
 243     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 244         return 0;
 245     s0 = *src;
 246     while (*s0)
 247     {
 248         const char *s1 = s0;
 249         int q_map_match = 0;
 250         map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 251                                 &q_map_match);
 252         if (space_split)
 253         {
 254             if (**map == *CHR_SPACE)
 255                 break;
 256         }
 257         else  /* complete subfield only. */
 258         {
 259             if (**map == *CHR_SPACE)
 260             {   /* save space mapping for later  .. */
 261                 space_start = s1;
 262                 space_end = s0;
 263                 continue;
 264             }
 265             else if (space_start)
 266             {   /* reload last space */
 267                 while (space_start < space_end)
 268                 {
 269                     if (strchr(REGEX_CHARS, *space_start))
 270                         dst[i++] = '\\';
 271                     dst_term[j++] = *space_start;
 272                     dst[i++] = *space_start++;
 273                 }
 274                 /* and reset */
 275                 space_start = space_end = 0;
 276             }
 277         }
 278         /* add non-space char */
 279         memcpy(dst_term+j, s1, s0 - s1);
 280         j += (s0 - s1);
 281         if (!q_map_match)
 282         {
 283             while (s1 < s0)
 284             {
 285                 if (strchr(REGEX_CHARS, *s1))
 286                     dst[i++] = '\\';
 287                 dst[i++] = *s1++;
 288             }
 289         }
 290         else
 291         {
 292             char tmpbuf[80];
 293             esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 294
 295             strcpy(dst + i, map[0]);
 296             i += strlen(map[0]);
 297         }
 298     }
 299     dst[i] = '\0';
 300     dst_term[j] = '\0';
 301     *src = s0;
 302     return i;
 303 }
 304
 305 /* term_101: handle term, where trunc = Process # */
 306 static int term_101(ZebraMaps zebra_maps, int reg_type,
 307                     const char **src, char *dst, int space_split,
 308                     char *dst_term)
 309 {
 310     const char *s0;
 311     const char **map;
 312     int i = 0;
 313     int j = 0;
 314
 315     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 316         return 0;
 317     s0 = *src;
 318     while (*s0)
 319     {
 320         if (*s0 == '#')
 321         {
 322             dst[i++] = '.';
 323             dst[i++] = '*';
 324             dst_term[j++] = *s0++;
 325         }
 326         else
 327         {
 328             const char *s1 = s0;
 329             int q_map_match = 0;
 330             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 331                                     &q_map_match);
 332             if (space_split && **map == *CHR_SPACE)
 333                 break;
 334
 335             /* add non-space char */
 336             memcpy(dst_term+j, s1, s0 - s1);
 337             j += (s0 - s1);
 338             if (!q_map_match)
 339             {
 340                 while (s1 < s0)
 341                 {
 342                     if (strchr(REGEX_CHARS, *s1))
 343                         dst[i++] = '\\';
 344                     dst[i++] = *s1++;
 345                 }
 346             }
 347             else
 348             {
 349                 char tmpbuf[80];
 350                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 351
 352                 strcpy(dst + i, map[0]);
 353                 i += strlen(map[0]);
 354             }
 355         }
 356     }
 357     dst[i] = '\0';
 358     dst_term[j++] = '\0';
 359     *src = s0;
 360     return i;
 361 }
 362
 363 /* term_103: handle term, where trunc = re-2 (regular expressions) */
 364 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 365                     char *dst, int *errors, int space_split,
 366                     char *dst_term)
 367 {
 368     int i = 0;
 369     int j = 0;
 370     const char *s0;
 371     const char **map;
 372
 373     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 374         return 0;
 375     s0 = *src;
 376     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 377         isdigit(((const unsigned char *)s0)[1]))
 378     {
 379         *errors = s0[1] - '0';
 380         s0 += 3;
 381         if (*errors > 3)
 382             *errors = 3;
 383     }
 384     while (*s0)
 385     {
 386         if (strchr("^\\()[].*+?|-", *s0))
 387         {
 388             dst_term[j++] = *s0;
 389             dst[i++] = *s0++;
 390         }
 391         else
 392         {
 393             const char *s1 = s0;
 394             int q_map_match = 0;
 395             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 396                                     &q_map_match);
 397             if (space_split && **map == *CHR_SPACE)
 398                 break;
 399
 400             /* add non-space char */
 401             memcpy(dst_term+j, s1, s0 - s1);
 402             j += (s0 - s1);
 403             if (!q_map_match)
 404             {
 405                 while (s1 < s0)
 406                 {
 407                     if (strchr(REGEX_CHARS, *s1))
 408                         dst[i++] = '\\';
 409                     dst[i++] = *s1++;
 410                 }
 411             }
 412             else
 413             {
 414                 char tmpbuf[80];
 415                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 416
 417                 strcpy(dst + i, map[0]);
 418                 i += strlen(map[0]);
 419             }
 420         }
 421     }
 422     dst[i] = '\0';
 423     dst_term[j] = '\0';
 424     *src = s0;
 425
 426     return i;
 427 }
 428
 429 /* term_103: handle term, where trunc = re-1 (regular expressions) */
 430 static int term_102(ZebraMaps zebra_maps, int reg_type, const char **src,
 431                     char *dst, int space_split, char *dst_term)
 432 {
 433     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 434                     dst_term);
 435 }
 436
 437
 438 /* term_104: handle term, where trunc = Process # and ! */
 439 static int term_104(ZebraMaps zebra_maps, int reg_type,
 440                     const char **src, char *dst, int space_split,
 441                     char *dst_term)
 442 {
 443     const char *s0;
 444     const char **map;
 445     int i = 0;
 446     int j = 0;
 447
 448     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 449         return 0;
 450     s0 = *src;
 451     while (*s0)
 452     {
 453         if (*s0 == '?')
 454         {
 455             dst_term[j++] = *s0++;
 456             if (*s0 >= '0' && *s0 <= '9')
 457             {
 458                 int limit = 0;
 459                 while (*s0 >= '0' && *s0 <= '9')
 460                 {
 461                     limit = limit * 10 + (*s0 - '0');
 462                     dst_term[j++] = *s0++;
 463                 }
 464                 if (limit > 20)
 465                     limit = 20;
 466                 while (--limit >= 0)
 467                 {
 468                     dst[i++] = '.';
 469                     dst[i++] = '?';
 470                 }
 471             }
 472             else
 473             {
 474                 dst[i++] = '.';
 475                 dst[i++] = '*';
 476             }
 477         }
 478         else if (*s0 == '*')
 479         {
 480             dst[i++] = '.';
 481             dst[i++] = '*';
 482             dst_term[j++] = *s0++;
 483         }
 484         else if (*s0 == '#')
 485         {
 486             dst[i++] = '.';
 487             dst_term[j++] = *s0++;
 488         }
 489         else
 490         {
 491             const char *s1 = s0;
 492             int q_map_match = 0;
 493             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 494                                     &q_map_match);
 495             if (space_split && **map == *CHR_SPACE)
 496                 break;
 497
 498             /* add non-space char */
 499             memcpy(dst_term+j, s1, s0 - s1);
 500             j += (s0 - s1);
 501             if (!q_map_match)
 502             {
 503                 while (s1 < s0)
 504                 {
 505                     if (strchr(REGEX_CHARS, *s1))
 506                         dst[i++] = '\\';
 507                     dst[i++] = *s1++;
 508                 }
 509             }
 510             else
 511             {
 512                 char tmpbuf[80];
 513                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 514
 515                 strcpy(dst + i, map[0]);
 516                 i += strlen(map[0]);
 517             }
 518         }
 519     }
 520     dst[i] = '\0';
 521     dst_term[j++] = '\0';
 522     *src = s0;
 523     return i;
 524 }
 525
 526 /* term_105/106: handle term, where trunc = Process * and ! and right trunc */
 527 static int term_105(ZebraMaps zebra_maps, int reg_type,
 528                     const char **src, char *dst, int space_split,
 529                     char *dst_term, int right_truncate)
 530 {
 531     const char *s0;
 532     const char **map;
 533     int i = 0;
 534     int j = 0;
 535
 536     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 537         return 0;
 538     s0 = *src;
 539     while (*s0)
 540     {
 541         if (*s0 == '*')
 542         {
 543             dst[i++] = '.';
 544             dst[i++] = '*';
 545             dst_term[j++] = *s0++;
 546         }
 547         else if (*s0 == '!')
 548         {
 549             dst[i++] = '.';
 550             dst_term[j++] = *s0++;
 551         }
 552         else
 553         {
 554             const char *s1 = s0;
 555             int q_map_match = 0;
 556             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 557                                     &q_map_match);
 558             if (space_split && **map == *CHR_SPACE)
 559                 break;
 560
 561             /* add non-space char */
 562             memcpy(dst_term+j, s1, s0 - s1);
 563             j += (s0 - s1);
 564             if (!q_map_match)
 565             {
 566                 while (s1 < s0)
 567                 {
 568                     if (strchr(REGEX_CHARS, *s1))
 569                         dst[i++] = '\\';
 570                     dst[i++] = *s1++;
 571                 }
 572             }
 573             else
 574             {
 575                 char tmpbuf[80];
 576                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 577
 578                 strcpy(dst + i, map[0]);
 579                 i += strlen(map[0]);
 580             }
 581         }
 582     }
 583     if (right_truncate)
 584     {
 585         dst[i++] = '.';
 586         dst[i++] = '*';
 587     }
 588     dst[i] = '\0';
 589
 590     dst_term[j++] = '\0';
 591     *src = s0;
 592     return i;
 593 }
 594
 595
 596 /* gen_regular_rel - generate regular expression from relation
 597  *  val:     border value (inclusive)
 598  *  islt:    1 if <=; 0 if >=.
 599  */
 600 static void gen_regular_rel(char *dst, int val, int islt)
 601 {
 602     int dst_p;
 603     int w, d, i;
 604     int pos = 0;
 605     char numstr[20];
 606
 607     yaz_log(YLOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 608     if (val >= 0)
 609     {
 610         if (islt)
 611             strcpy(dst, "(-[0-9]+|(");
 612         else
 613             strcpy(dst, "((");
 614     }
 615     else
 616     {
 617         if (!islt)
 618         {
 619             strcpy(dst, "([0-9]+|-(");
 620             dst_p = strlen(dst);
 621             islt = 1;
 622         }
 623         else
 624         {
 625             strcpy(dst, "(-(");
 626             islt = 0;
 627         }
 628         val = -val;
 629     }
 630     dst_p = strlen(dst);
 631     sprintf(numstr, "%d", val);
 632     for (w = strlen(numstr); --w >= 0; pos++)
 633     {
 634         d = numstr[w];
 635         if (pos > 0)
 636         {
 637             if (islt)
 638             {
 639                 if (d == '0')
 640                     continue;
 641                 d--;
 642             }
 643             else
 644             {
 645                 if (d == '9')
 646                     continue;
 647                 d++;
 648             }
 649         }
 650
 651         strcpy(dst + dst_p, numstr);
 652         dst_p = strlen(dst) - pos - 1;
 653
 654         if (islt)
 655         {
 656             if (d != '0')
 657             {
 658                 dst[dst_p++] = '[';
 659                 dst[dst_p++] = '0';
 660                 dst[dst_p++] = '-';
 661                 dst[dst_p++] = d;
 662                 dst[dst_p++] = ']';
 663             }
 664             else
 665                 dst[dst_p++] = d;
 666         }
 667         else
 668         {
 669             if (d != '9')
 670             {
 671                 dst[dst_p++] = '[';
 672                 dst[dst_p++] = d;
 673                 dst[dst_p++] = '-';
 674                 dst[dst_p++] = '9';
 675                 dst[dst_p++] = ']';
 676             }
 677             else
 678                 dst[dst_p++] = d;
 679         }
 680         for (i = 0; i<pos; i++)
 681         {
 682             dst[dst_p++] = '[';
 683             dst[dst_p++] = '0';
 684             dst[dst_p++] = '-';
 685             dst[dst_p++] = '9';
 686             dst[dst_p++] = ']';
 687         }
 688         dst[dst_p++] = '|';
 689     }
 690     dst[dst_p] = '\0';
 691     if (islt)
 692     {
 693         /* match everything less than 10^(pos-1) */
 694         strcat(dst, "0*");
 695         for (i = 1; i<pos; i++)
 696             strcat(dst, "[0-9]?");
 697     }
 698     else
 699     {
 700         /* match everything greater than 10^pos */
 701         for (i = 0; i <= pos; i++)
 702             strcat(dst, "[0-9]");
 703         strcat(dst, "[0-9]*");
 704     }
 705     strcat(dst, "))");
 706 }
 707
 708 void string_rel_add_char(char **term_p, const char *src, int *indx)
 709 {
 710     if (src[*indx] == '\\')
 711         *(*term_p)++ = src[(*indx)++];
 712     *(*term_p)++ = src[(*indx)++];
 713 }
 714
 715 /*
 716  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 717  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 718  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 719  *              ([^-a].*|a[^-b].*|ab[c-].*)
 720  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 721  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 722  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 723  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 724  */
 725 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 726                            const char **term_sub, char *term_dict,
 727                            oid_value attributeSet,
 728                            int reg_type, int space_split, char *term_dst,
 729                            int *error_code)
 730 {
 731     AttrType relation;
 732     int relation_value;
 733     int i;
 734     char *term_tmp = term_dict + strlen(term_dict);
 735     char term_component[2*IT_MAX_WORD+20];
 736
 737     attr_init_APT(&relation, zapt, 2);
 738     relation_value = attr_find(&relation, NULL);
 739
 740     *error_code = 0;
 741     yaz_log(YLOG_DEBUG, "string relation value=%d", relation_value);
 742     switch (relation_value)
 743     {
 744     case 1:
 745         if (!term_100(zh->reg->zebra_maps, reg_type,
 746                       term_sub, term_component,
 747                       space_split, term_dst))
 748             return 0;
 749         yaz_log(log_level_rpn, "Relation <");
 750
 751         *term_tmp++ = '(';
 752         for (i = 0; term_component[i]; )
 753         {
 754             int j = 0;
 755
 756             if (i)
 757                 *term_tmp++ = '|';
 758             while (j < i)
 759                 string_rel_add_char(&term_tmp, term_component, &j);
 760
 761             *term_tmp++ = '[';
 762
 763             *term_tmp++ = '^';
 764             string_rel_add_char(&term_tmp, term_component, &i);
 765             *term_tmp++ = '-';
 766
 767             *term_tmp++ = ']';
 768             *term_tmp++ = '.';
 769             *term_tmp++ = '*';
 770
 771             if ((term_tmp - term_dict) > IT_MAX_WORD)
 772                 break;
 773         }
 774         *term_tmp++ = ')';
 775         *term_tmp = '\0';
 776         break;
 777     case 2:
 778         if (!term_100(zh->reg->zebra_maps, reg_type,
 779                       term_sub, term_component,
 780                       space_split, term_dst))
 781             return 0;
 782         yaz_log(log_level_rpn, "Relation <=");
 783
 784         *term_tmp++ = '(';
 785         for (i = 0; term_component[i]; )
 786         {
 787             int j = 0;
 788
 789             while (j < i)
 790                 string_rel_add_char(&term_tmp, term_component, &j);
 791             *term_tmp++ = '[';
 792
 793             *term_tmp++ = '^';
 794             string_rel_add_char(&term_tmp, term_component, &i);
 795             *term_tmp++ = '-';
 796
 797             *term_tmp++ = ']';
 798             *term_tmp++ = '.';
 799             *term_tmp++ = '*';
 800
 801             *term_tmp++ = '|';
 802
 803             if ((term_tmp - term_dict) > IT_MAX_WORD)
 804                 break;
 805         }
 806         for (i = 0; term_component[i]; )
 807             string_rel_add_char(&term_tmp, term_component, &i);
 808         *term_tmp++ = ')';
 809         *term_tmp = '\0';
 810         break;
 811     case 5:
 812         if (!term_100 (zh->reg->zebra_maps, reg_type,
 813                        term_sub, term_component, space_split, term_dst))
 814             return 0;
 815         yaz_log(log_level_rpn, "Relation >");
 816
 817         *term_tmp++ = '(';
 818         for (i = 0; term_component[i];)
 819         {
 820             int j = 0;
 821
 822             while (j < i)
 823                 string_rel_add_char(&term_tmp, term_component, &j);
 824             *term_tmp++ = '[';
 825
 826             *term_tmp++ = '^';
 827             *term_tmp++ = '-';
 828             string_rel_add_char(&term_tmp, term_component, &i);
 829
 830             *term_tmp++ = ']';
 831             *term_tmp++ = '.';
 832             *term_tmp++ = '*';
 833
 834             *term_tmp++ = '|';
 835
 836             if ((term_tmp - term_dict) > IT_MAX_WORD)
 837                 break;
 838         }
 839         for (i = 0; term_component[i];)
 840             string_rel_add_char(&term_tmp, term_component, &i);
 841         *term_tmp++ = '.';
 842         *term_tmp++ = '+';
 843         *term_tmp++ = ')';
 844         *term_tmp = '\0';
 845         break;
 846     case 4:
 847         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 848                       term_component, space_split, term_dst))
 849             return 0;
 850         yaz_log(log_level_rpn, "Relation >=");
 851
 852         *term_tmp++ = '(';
 853         for (i = 0; term_component[i];)
 854         {
 855             int j = 0;
 856
 857             if (i)
 858                 *term_tmp++ = '|';
 859             while (j < i)
 860                 string_rel_add_char(&term_tmp, term_component, &j);
 861             *term_tmp++ = '[';
 862
 863             if (term_component[i+1])
 864             {
 865                 *term_tmp++ = '^';
 866                 *term_tmp++ = '-';
 867                 string_rel_add_char(&term_tmp, term_component, &i);
 868             }
 869             else
 870             {
 871                 string_rel_add_char(&term_tmp, term_component, &i);
 872                 *term_tmp++ = '-';
 873             }
 874             *term_tmp++ = ']';
 875             *term_tmp++ = '.';
 876             *term_tmp++ = '*';
 877
 878             if ((term_tmp - term_dict) > IT_MAX_WORD)
 879                 break;
 880         }
 881         *term_tmp++ = ')';
 882         *term_tmp = '\0';
 883         break;
 884     case 3:
 885     case 102:
 886     case -1:
 887         if (!**term_sub)
 888             return 1;
 889         yaz_log(log_level_rpn, "Relation =");
 890         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 891                       term_component, space_split, term_dst))
 892             return 0;
 893         strcat(term_tmp, "(");
 894         strcat(term_tmp, term_component);
 895         strcat(term_tmp, ")");
 896         break;
 897     case 103:
 898         yaz_log(log_level_rpn, "Relation always matches");
 899         /* skip to end of term (we don't care what it is) */
 900         while (**term_sub != '\0')
 901             (*term_sub)++;
 902         break;
 903     default:
 904         *error_code = YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
 905         return 0;
 906     }
 907     return 1;
 908 }
 909
 910 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 911                              const char **term_sub,
 912                              oid_value attributeSet, NMEM stream,
 913                              struct grep_info *grep_info,
 914                              int reg_type, int complete_flag,
 915                              int num_bases, char **basenames,
 916                              char *term_dst,
 917                              const char *xpath_use,
 918                              struct ord_list **ol);
 919
 920 static ZEBRA_RES term_limits_APT(ZebraHandle zh,
 921                                  Z_AttributesPlusTerm *zapt,
 922                                  zint *hits_limit_value,
 923                                  const char **term_ref_id_str,
 924                                  NMEM nmem)
 925 {
 926     AttrType term_ref_id_attr;
 927     AttrType hits_limit_attr;
 928     int term_ref_id_int;
 929
 930     attr_init_APT(&hits_limit_attr, zapt, 11);
 931     *hits_limit_value  = attr_find(&hits_limit_attr, NULL);
 932
 933     attr_init_APT(&term_ref_id_attr, zapt, 10);
 934     term_ref_id_int = attr_find_ex(&term_ref_id_attr, NULL, term_ref_id_str);
 935     if (term_ref_id_int >= 0)
 936     {
 937         char *res = nmem_malloc(nmem, 20);
 938         sprintf(res, "%d", term_ref_id_int);
 939         *term_ref_id_str = res;
 940     }
 941
 942     /* no limit given ? */
 943     if (*hits_limit_value == -1)
 944     {
 945         if (*term_ref_id_str)
 946         {
 947             /* use global if term_ref is present */
 948             *hits_limit_value = zh->approx_limit;
 949         }
 950         else
 951         {
 952             /* no counting if term_ref is not present */
 953             *hits_limit_value = 0;
 954         }
 955     }
 956     else if (*hits_limit_value == 0)
 957     {
 958         /* 0 is the same as global limit */
 959         *hits_limit_value = zh->approx_limit;
 960     }
 961     yaz_log(YLOG_DEBUG, "term_limits_APT ref_id=%s limit=" ZINT_FORMAT,
 962             *term_ref_id_str ? *term_ref_id_str : "none",
 963             *hits_limit_value);
 964     return ZEBRA_OK;
 965 }
 966
 967 static ZEBRA_RES term_trunc(ZebraHandle zh,
 968                             Z_AttributesPlusTerm *zapt,
 969                             const char **term_sub,
 970                             oid_value attributeSet, NMEM stream,
 971                             struct grep_info *grep_info,
 972                             int reg_type, int complete_flag,
 973                             int num_bases, char **basenames,
 974                             char *term_dst,
 975                             const char *rank_type,
 976                             const char *xpath_use,
 977                             NMEM rset_nmem,
 978                             RSET *rset,
 979                             struct rset_key_control *kc)
 980 {
 981     ZEBRA_RES res;
 982     struct ord_list *ol;
 983     zint hits_limit_value;
 984     const char *term_ref_id_str = 0;
 985     *rset = 0;
 986
 987     term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str, stream);
 988     grep_info->isam_p_indx = 0;
 989     res = string_term(zh, zapt, term_sub, attributeSet, stream, grep_info,
 990                       reg_type, complete_flag, num_bases, basenames,
 991                       term_dst, xpath_use, &ol);
 992     if (res != ZEBRA_OK)
 993         return res;
 994     if (!*term_sub)  /* no more terms ? */
 995         return res;
 996     yaz_log(log_level_rpn, "term: %s", term_dst);
 997     *rset = rset_trunc(zh, grep_info->isam_p_buf,
 998                        grep_info->isam_p_indx, term_dst,
 999                        strlen(term_dst), rank_type, 1 /* preserve pos */,
1000                        zapt->term->which, rset_nmem,
1001                        kc, kc->scope, ol, reg_type, hits_limit_value,
1002                        term_ref_id_str);
1003     if (!*rset)
1004         return ZEBRA_FAIL;
1005     return ZEBRA_OK;
1006 }
1007
1008 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1009                              const char **term_sub,
1010                              oid_value attributeSet, NMEM stream,
1011                              struct grep_info *grep_info,
1012                              int reg_type, int complete_flag,
1013                              int num_bases, char **basenames,
1014                              char *term_dst,
1015                              const char *xpath_use,
1016                              struct ord_list **ol)
1017 {
1018     char term_dict[2*IT_MAX_WORD+4000];
1019     int j, r, base_no;
1020     AttrType truncation;
1021     int truncation_value;
1022     const char *termp;
1023     struct rpn_char_map_info rcmi;
1024     int space_split = complete_flag ? 0 : 1;
1025
1026     int bases_ok = 0;     /* no of databases with OK attribute */
1027
1028     *ol = ord_list_create(stream);
1029
1030     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1031     attr_init_APT(&truncation, zapt, 5);
1032     truncation_value = attr_find(&truncation, NULL);
1033     yaz_log(log_level_rpn, "truncation value %d", truncation_value);
1034
1035     for (base_no = 0; base_no < num_bases; base_no++)
1036     {
1037         int ord = -1;
1038         int regex_range = 0;
1039         int max_pos, prefix_len = 0;
1040         int relation_error;
1041         char ord_buf[32];
1042         int ord_len, i;
1043
1044         termp = *term_sub; /* start of term for each database */
1045
1046         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1047         {
1048             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1049                            basenames[base_no]);
1050             return ZEBRA_FAIL;
1051         }
1052
1053         if (zebra_apt_get_ord(zh, zapt, reg_type, xpath_use,
1054                               attributeSet, &ord) != ZEBRA_OK)
1055             continue;
1056
1057         bases_ok++;
1058
1059         *ol = ord_list_append(stream, *ol, ord);
1060         ord_len = key_SU_encode (ord, ord_buf);
1061
1062         term_dict[prefix_len++] = '(';
1063         for (i = 0; i<ord_len; i++)
1064         {
1065             term_dict[prefix_len++] = 1;  /* our internal regexp escape char */
1066             term_dict[prefix_len++] = ord_buf[i];
1067         }
1068         term_dict[prefix_len++] = ')';
1069         term_dict[prefix_len] = '\0';
1070         j = prefix_len;
1071         switch (truncation_value)
1072         {
1073         case -1:         /* not specified */
1074         case 100:        /* do not truncate */
1075             if (!string_relation(zh, zapt, &termp, term_dict,
1076                                  attributeSet,
1077                                  reg_type, space_split, term_dst,
1078                                  &relation_error))
1079             {
1080                 if (relation_error)
1081                 {
1082                     zebra_setError(zh, relation_error, 0);
1083                     return ZEBRA_FAIL;
1084                 }
1085                 *term_sub = 0;
1086                 return ZEBRA_OK;
1087             }
1088             break;
1089         case 1:          /* right truncation */
1090             term_dict[j++] = '(';
1091             if (!term_100(zh->reg->zebra_maps, reg_type,
1092                           &termp, term_dict + j, space_split, term_dst))
1093             {
1094                 *term_sub = 0;
1095                 return ZEBRA_OK;
1096             }
1097             strcat(term_dict, ".*)");
1098             break;
1099         case 2:          /* keft truncation */
1100             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1101             if (!term_100(zh->reg->zebra_maps, reg_type,
1102                           &termp, term_dict + j, space_split, term_dst))
1103             {
1104                 *term_sub = 0;
1105                 return ZEBRA_OK;
1106             }
1107             strcat(term_dict, ")");
1108             break;
1109         case 3:          /* left&right truncation */
1110             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1111             if (!term_100(zh->reg->zebra_maps, reg_type,
1112                           &termp, term_dict + j, space_split, term_dst))
1113             {
1114                 *term_sub = 0;
1115                 return ZEBRA_OK;
1116             }
1117             strcat(term_dict, ".*)");
1118             break;
1119         case 101:        /* process # in term */
1120             term_dict[j++] = '(';
1121             if (!term_101(zh->reg->zebra_maps, reg_type,
1122                           &termp, term_dict + j, space_split, term_dst))
1123             {
1124                 *term_sub = 0;
1125                 return ZEBRA_OK;
1126             }
1127             strcat(term_dict, ")");
1128             break;
1129         case 102:        /* Regexp-1 */
1130             term_dict[j++] = '(';
1131             if (!term_102(zh->reg->zebra_maps, reg_type,
1132                           &termp, term_dict + j, space_split, term_dst))
1133             {
1134                 *term_sub = 0;
1135                 return ZEBRA_OK;
1136             }
1137             strcat(term_dict, ")");
1138             break;
1139         case 103:       /* Regexp-2 */
1140             regex_range = 1;
1141             term_dict[j++] = '(';
1142             if (!term_103(zh->reg->zebra_maps, reg_type,
1143                           &termp, term_dict + j, &regex_range,
1144                           space_split, term_dst))
1145             {
1146                 *term_sub = 0;
1147                 return ZEBRA_OK;
1148             }
1149             strcat(term_dict, ")");
1150             break;
1151         case 104:        /* process # and ! in term */
1152             term_dict[j++] = '(';
1153             if (!term_104(zh->reg->zebra_maps, reg_type,
1154                           &termp, term_dict + j, space_split, term_dst))
1155             {
1156                 *term_sub = 0;
1157                 return ZEBRA_OK;
1158             }
1159             strcat(term_dict, ")");
1160             break;
1161         case 105:        /* process * and ! in term */
1162             term_dict[j++] = '(';
1163             if (!term_105(zh->reg->zebra_maps, reg_type,
1164                           &termp, term_dict + j, space_split, term_dst, 1))
1165             {
1166                 *term_sub = 0;
1167                 return ZEBRA_OK;
1168             }
1169             strcat(term_dict, ")");
1170             break;
1171         case 106:        /* process * and ! in term */
1172             term_dict[j++] = '(';
1173             if (!term_105(zh->reg->zebra_maps, reg_type,
1174                           &termp, term_dict + j, space_split, term_dst, 0))
1175             {
1176                 *term_sub = 0;
1177                 return ZEBRA_OK;
1178             }
1179             strcat(term_dict, ")");
1180             break;
1181         default:
1182             zebra_setError_zint(zh,
1183                                 YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE,
1184                                 truncation_value);
1185             return ZEBRA_FAIL;
1186         }
1187         if (1)
1188         {
1189             char buf[80];
1190             const char *input = term_dict + prefix_len;
1191             esc_str(buf, sizeof(buf), input, strlen(input));
1192         }
1193         yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_dict+prefix_len);
1194         r = dict_lookup_grep(zh->reg->dict, term_dict, regex_range,
1195                              grep_info, &max_pos,
1196                              ord_len /* number of "exact" chars */,
1197                              grep_handle);
1198         if (r)
1199             yaz_log(YLOG_WARN, "dict_lookup_grep fail %d", r);
1200     }
1201     if (!bases_ok)
1202         return ZEBRA_FAIL;
1203     *term_sub = termp;
1204     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1205     return ZEBRA_OK;
1206 }
1207
1208
1209 /* convert APT search term to UTF8 */
1210 static ZEBRA_RES zapt_term_to_utf8(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1211                                    char *termz)
1212 {
1213     size_t sizez;
1214     Z_Term *term = zapt->term;
1215
1216     switch (term->which)
1217     {
1218     case Z_Term_general:
1219         if (zh->iconv_to_utf8 != 0)
1220         {
1221             char *inbuf = (char *) term->u.general->buf;
1222             size_t inleft = term->u.general->len;
1223             char *outbuf = termz;
1224             size_t outleft = IT_MAX_WORD-1;
1225             size_t ret;
1226
1227             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1228                         &outbuf, &outleft);
1229             if (ret == (size_t)(-1))
1230             {
1231                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1232                 zebra_setError(
1233                     zh,
1234                     YAZ_BIB1_QUERY_TERM_INCLUDES_CHARS_THAT_DO_NOT_TRANSLATE_INTO_,
1235                     0);
1236                 return ZEBRA_FAIL;
1237             }
1238             *outbuf = 0;
1239         }
1240         else
1241         {
1242             sizez = term->u.general->len;
1243             if (sizez > IT_MAX_WORD-1)
1244                 sizez = IT_MAX_WORD-1;
1245             memcpy (termz, term->u.general->buf, sizez);
1246             termz[sizez] = '\0';
1247         }
1248         break;
1249     case Z_Term_characterString:
1250         sizez = strlen(term->u.characterString);
1251         if (sizez > IT_MAX_WORD-1)
1252             sizez = IT_MAX_WORD-1;
1253         memcpy (termz, term->u.characterString, sizez);
1254         termz[sizez] = '\0';
1255         break;
1256     default:
1257         zebra_setError(zh, YAZ_BIB1_UNSUPP_CODED_VALUE_FOR_TERM, 0);
1258         return ZEBRA_FAIL;
1259     }
1260     return ZEBRA_OK;
1261 }
1262
1263 /* convert APT SCAN term to internal cmap */
1264 static ZEBRA_RES trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1265                                  char *termz, int reg_type)
1266 {
1267     char termz0[IT_MAX_WORD];
1268
1269     if (zapt_term_to_utf8(zh, zapt, termz0) == ZEBRA_FAIL)
1270         return ZEBRA_FAIL;    /* error */
1271     else
1272     {
1273         const char **map;
1274         const char *cp = (const char *) termz0;
1275         const char *cp_end = cp + strlen(cp);
1276         const char *src;
1277         int i = 0;
1278         const char *space_map = NULL;
1279         int len;
1280
1281         while ((len = (cp_end - cp)) > 0)
1282         {
1283             map = zebra_maps_input(zh->reg->zebra_maps, reg_type, &cp, len, 0);
1284             if (**map == *CHR_SPACE)
1285                 space_map = *map;
1286             else
1287             {
1288                 if (i && space_map)
1289                     for (src = space_map; *src; src++)
1290                         termz[i++] = *src;
1291                 space_map = NULL;
1292                 for (src = *map; *src; src++)
1293                     termz[i++] = *src;
1294             }
1295         }
1296         termz[i] = '\0';
1297     }
1298     return ZEBRA_OK;
1299 }
1300
1301 static void grep_info_delete(struct grep_info *grep_info)
1302 {
1303 #ifdef TERM_COUNT
1304     xfree(grep_info->term_no);
1305 #endif
1306     xfree(grep_info->isam_p_buf);
1307 }
1308
1309 static ZEBRA_RES grep_info_prepare(ZebraHandle zh,
1310                                    Z_AttributesPlusTerm *zapt,
1311                                    struct grep_info *grep_info,
1312                                    int reg_type)
1313 {
1314     AttrType termset;
1315     int termset_value_numeric;
1316     const char *termset_value_string;
1317
1318 #ifdef TERM_COUNT
1319     grep_info->term_no = 0;
1320 #endif
1321     grep_info->isam_p_size = 0;
1322     grep_info->isam_p_buf = NULL;
1323     grep_info->zh = zh;
1324     grep_info->reg_type = reg_type;
1325     grep_info->termset = 0;
1326     if (!zapt)
1327         return ZEBRA_OK;
1328     attr_init_APT(&termset, zapt, 8);
1329     termset_value_numeric =
1330         attr_find_ex(&termset, NULL, &termset_value_string);
1331     if (termset_value_numeric != -1)
1332     {
1333 #if TERMSET_DISABLE
1334         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, "termset");
1335         return ZEBRA_FAIL;
1336 #else
1337         char resname[32];
1338         const char *termset_name = 0;
1339         if (termset_value_numeric != -2)
1340         {
1341
1342             sprintf(resname, "%d", termset_value_numeric);
1343             termset_name = resname;
1344         }
1345         else
1346             termset_name = termset_value_string;
1347         yaz_log(log_level_rpn, "creating termset set %s", termset_name);
1348         grep_info->termset = resultSetAdd(zh, termset_name, 1);
1349         if (!grep_info->termset)
1350         {
1351             zebra_setError(zh, YAZ_BIB1_ILLEGAL_RESULT_SET_NAME, termset_name);
1352             return ZEBRA_FAIL;
1353         }
1354 #endif
1355     }
1356     return ZEBRA_OK;
1357 }
1358
1359 /**
1360   \brief Create result set(s) for list of terms
1361   \param zh Zebra Handle
1362   \param zapt Attributes Plust Term (RPN leaf)
1363   \param termz term as used in query but converted to UTF-8
1364   \param attributeSet default attribute set
1365   \param stream memory for result
1366   \param reg_type register type ('w', 'p',..)
1367   \param complete_flag whether it's phrases or not
1368   \param rank_type term flags for ranking
1369   \param xpath_use use attribute for X-Path (-1 for no X-path)
1370   \param num_bases number of databases
1371   \param basenames array of databases
1372   \param rset_nmem memory for result sets
1373   \param result_sets output result set for each term in list (output)
1374   \param num_result_sets number of output result sets
1375   \param kc rset key control to be used for created result sets
1376 */
1377 static ZEBRA_RES term_list_trunc(ZebraHandle zh,
1378                                  Z_AttributesPlusTerm *zapt,
1379                                  const char *termz,
1380                                  oid_value attributeSet,
1381                                  NMEM stream,
1382                                  int reg_type, int complete_flag,
1383                                  const char *rank_type,
1384                                  const char *xpath_use,
1385                                  int num_bases, char **basenames,
1386                                  NMEM rset_nmem,
1387                                  RSET **result_sets, int *num_result_sets,
1388                                  struct rset_key_control *kc)
1389 {
1390     char term_dst[IT_MAX_WORD+1];
1391     struct grep_info grep_info;
1392     const char *termp = termz;
1393     int alloc_sets = 0;
1394
1395     *num_result_sets = 0;
1396     *term_dst = 0;
1397     if (grep_info_prepare(zh, zapt, &grep_info, reg_type) == ZEBRA_FAIL)
1398         return ZEBRA_FAIL;
1399     while(1)
1400     {
1401         ZEBRA_RES res;
1402
1403         if (alloc_sets == *num_result_sets)
1404         {
1405             int add = 10;
1406             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1407                                               sizeof(*rnew));
1408             if (alloc_sets)
1409                 memcpy(rnew, *result_sets, alloc_sets * sizeof(*rnew));
1410             alloc_sets = alloc_sets + add;
1411             *result_sets = rnew;
1412         }
1413         res = term_trunc(zh, zapt, &termp, attributeSet,
1414                          stream, &grep_info,
1415                          reg_type, complete_flag,
1416                          num_bases, basenames,
1417                          term_dst, rank_type,
1418                          xpath_use, rset_nmem,
1419                          &(*result_sets)[*num_result_sets],
1420                          kc);
1421         if (res != ZEBRA_OK)
1422         {
1423             int i;
1424             for (i = 0; i < *num_result_sets; i++)
1425                 rset_delete((*result_sets)[i]);
1426             grep_info_delete (&grep_info);
1427             return res;
1428         }
1429         if ((*result_sets)[*num_result_sets] == 0)
1430             break;
1431         (*num_result_sets)++;
1432
1433         if (!*termp)
1434             break;
1435     }
1436     grep_info_delete(&grep_info);
1437     return ZEBRA_OK;
1438 }
1439
1440 static ZEBRA_RES rpn_search_APT_position(ZebraHandle zh,
1441                                          Z_AttributesPlusTerm *zapt,
1442                                          oid_value attributeSet,
1443                                          int reg_type,
1444                                          int num_bases, char **basenames,
1445                                          NMEM rset_nmem,
1446                                          RSET *rset,
1447                                          struct rset_key_control *kc)
1448 {
1449     RSET *f_set;
1450     int base_no;
1451     int position_value;
1452     int num_sets = 0;
1453     AttrType position;
1454
1455     attr_init_APT(&position, zapt, 3);
1456     position_value = attr_find(&position, NULL);
1457     switch(position_value)
1458     {
1459     case 3:
1460     case -1:
1461         return ZEBRA_OK;
1462     case 1:
1463     case 2:
1464         break;
1465     default:
1466         zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_POSITION_ATTRIBUTE,
1467                             position_value);
1468         return ZEBRA_FAIL;
1469     }
1470
1471     if (!zebra_maps_is_first_in_field(zh->reg->zebra_maps, reg_type))
1472     {
1473         zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_POSITION_ATTRIBUTE,
1474                             position_value);
1475         return ZEBRA_FAIL;
1476     }
1477
1478     if (!zh->reg->isamb)
1479     {
1480         zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_POSITION_ATTRIBUTE,
1481                             position_value);
1482         return ZEBRA_FAIL;
1483     }
1484     f_set = xmalloc(sizeof(RSET) * num_bases);
1485     for (base_no = 0; base_no < num_bases; base_no++)
1486     {
1487         int ord = -1;
1488         char ord_buf[32];
1489         char term_dict[100];
1490         int ord_len;
1491         char *val;
1492         ISAM_P isam_p;
1493
1494         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1495         {
1496             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1497                            basenames[base_no]);
1498             return ZEBRA_FAIL;
1499         }
1500
1501         if (zebra_apt_get_ord(zh, zapt, reg_type, 0,
1502                               attributeSet, &ord) != ZEBRA_OK)
1503             continue;
1504
1505         ord_len = key_SU_encode (ord, ord_buf);
1506         memcpy(term_dict, ord_buf, ord_len);
1507         strcpy(term_dict+ord_len, FIRST_IN_FIELD_STR);
1508         val = dict_lookup(zh->reg->dict, term_dict);
1509         if (!val)
1510             continue;
1511         assert(*val == sizeof(ISAM_P));
1512         memcpy(&isam_p, val+1, sizeof(isam_p));
1513
1514         f_set[num_sets++] = rsisamb_create(rset_nmem, kc, kc->scope,
1515                                            zh->reg->isamb, isam_p, 0);
1516
1517     }
1518     if (num_sets)
1519     {
1520         *rset = rset_create_or(rset_nmem, kc, kc->scope,
1521                                0 /* termid */, num_sets, f_set);
1522     }
1523     xfree(f_set);
1524     return ZEBRA_OK;
1525 }
1526
1527 static ZEBRA_RES rpn_search_APT_phrase(ZebraHandle zh,
1528                                        Z_AttributesPlusTerm *zapt,
1529                                        const char *termz_org,
1530                                        oid_value attributeSet,
1531                                        NMEM stream,
1532                                        int reg_type, int complete_flag,
1533                                        const char *rank_type,
1534                                        const char *xpath_use,
1535                                        int num_bases, char **basenames,
1536                                        NMEM rset_nmem,
1537                                        RSET *rset,
1538                                        struct rset_key_control *kc)
1539 {
1540     RSET *result_sets = 0;
1541     int num_result_sets = 0;
1542     ZEBRA_RES res =
1543         term_list_trunc(zh, zapt, termz_org, attributeSet,
1544                         stream, reg_type, complete_flag,
1545                         rank_type, xpath_use,
1546                         num_bases, basenames,
1547                         rset_nmem,
1548                         &result_sets, &num_result_sets, kc);
1549
1550     if (res != ZEBRA_OK)
1551         return res;
1552
1553     if (num_result_sets > 0)
1554     {
1555         RSET first_set = 0;
1556         res = rpn_search_APT_position(zh, zapt, attributeSet,
1557                                       reg_type,
1558                                       num_bases, basenames,
1559                                       rset_nmem, &first_set,
1560                                       kc);
1561         if (res != ZEBRA_OK)
1562             return res;
1563         if (first_set)
1564         {
1565             RSET *nsets = nmem_malloc(stream,
1566                                       sizeof(RSET) * (num_result_sets+1));
1567             nsets[0] = first_set;
1568             memcpy(nsets+1, result_sets, sizeof(RSET) * num_result_sets);
1569             result_sets = nsets;
1570             num_result_sets++;
1571         }
1572     }
1573     if (num_result_sets == 0)
1574         *rset = rset_create_null(rset_nmem, kc, 0);
1575     else if (num_result_sets == 1)
1576         *rset = result_sets[0];
1577     else
1578         *rset = rset_create_prox(rset_nmem, kc, kc->scope,
1579                                  num_result_sets, result_sets,
1580                                  1 /* ordered */, 0 /* exclusion */,
1581                                  3 /* relation */, 1 /* distance */);
1582     if (!*rset)
1583         return ZEBRA_FAIL;
1584     return ZEBRA_OK;
1585 }
1586
1587 static ZEBRA_RES rpn_search_APT_or_list(ZebraHandle zh,
1588                                         Z_AttributesPlusTerm *zapt,
1589                                         const char *termz_org,
1590                                         oid_value attributeSet,
1591                                         NMEM stream,
1592                                         int reg_type, int complete_flag,
1593                                         const char *rank_type,
1594                                         const char *xpath_use,
1595                                         int num_bases, char **basenames,
1596                                         NMEM rset_nmem,
1597                                         RSET *rset,
1598                                         struct rset_key_control *kc)
1599 {
1600     RSET *result_sets = 0;
1601     int num_result_sets = 0;
1602     ZEBRA_RES res =
1603         term_list_trunc(zh, zapt, termz_org, attributeSet,
1604                         stream, reg_type, complete_flag,
1605                         rank_type, xpath_use,
1606                         num_bases, basenames,
1607                         rset_nmem,
1608                         &result_sets, &num_result_sets, kc);
1609     if (res != ZEBRA_OK)
1610         return res;
1611     if (num_result_sets == 0)
1612         *rset = rset_create_null(rset_nmem, kc, 0);
1613     else if (num_result_sets == 1)
1614         *rset = result_sets[0];
1615     else
1616         *rset = rset_create_or(rset_nmem, kc, kc->scope, 0 /* termid */,
1617                                num_result_sets, result_sets);
1618     if (!*rset)
1619         return ZEBRA_FAIL;
1620     return ZEBRA_OK;
1621 }
1622
1623 static ZEBRA_RES rpn_search_APT_and_list(ZebraHandle zh,
1624                                          Z_AttributesPlusTerm *zapt,
1625                                          const char *termz_org,
1626                                          oid_value attributeSet,
1627                                          NMEM stream,
1628                                          int reg_type, int complete_flag,
1629                                          const char *rank_type,
1630                                          const char *xpath_use,
1631                                          int num_bases, char **basenames,
1632                                          NMEM rset_nmem,
1633                                          RSET *rset,
1634                                          struct rset_key_control *kc)
1635 {
1636     RSET *result_sets = 0;
1637     int num_result_sets = 0;
1638     ZEBRA_RES res =
1639         term_list_trunc(zh, zapt, termz_org, attributeSet,
1640                         stream, reg_type, complete_flag,
1641                         rank_type, xpath_use,
1642                         num_bases, basenames,
1643                         rset_nmem,
1644                         &result_sets, &num_result_sets,
1645                         kc);
1646     if (res != ZEBRA_OK)
1647         return res;
1648     if (num_result_sets == 0)
1649         *rset = rset_create_null(rset_nmem, kc, 0);
1650     else if (num_result_sets == 1)
1651         *rset = result_sets[0];
1652     else
1653         *rset = rset_create_and(rset_nmem, kc, kc->scope,
1654                                 num_result_sets, result_sets);
1655     if (!*rset)
1656         return ZEBRA_FAIL;
1657     return ZEBRA_OK;
1658 }
1659
1660 static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1661                             const char **term_sub,
1662                             char *term_dict,
1663                             oid_value attributeSet,
1664                             struct grep_info *grep_info,
1665                             int *max_pos,
1666                             int reg_type,
1667                             char *term_dst,
1668                             int *error_code)
1669 {
1670     AttrType relation;
1671     int relation_value;
1672     int term_value;
1673     int r;
1674     char *term_tmp = term_dict + strlen(term_dict);
1675
1676     *error_code = 0;
1677     attr_init_APT(&relation, zapt, 2);
1678     relation_value = attr_find(&relation, NULL);
1679
1680     yaz_log(log_level_rpn, "numeric relation value=%d", relation_value);
1681
1682     switch (relation_value)
1683     {
1684     case 1:
1685         yaz_log(log_level_rpn, "Relation <");
1686         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1687                       term_dst))
1688             return 0;
1689         term_value = atoi (term_tmp);
1690         gen_regular_rel(term_tmp, term_value-1, 1);
1691         break;
1692     case 2:
1693         yaz_log(log_level_rpn, "Relation <=");
1694         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1695                       term_dst))
1696             return 0;
1697         term_value = atoi (term_tmp);
1698         gen_regular_rel(term_tmp, term_value, 1);
1699         break;
1700     case 4:
1701         yaz_log(log_level_rpn, "Relation >=");
1702         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1703                       term_dst))
1704             return 0;
1705         term_value = atoi (term_tmp);
1706         gen_regular_rel(term_tmp, term_value, 0);
1707         break;
1708     case 5:
1709         yaz_log(log_level_rpn, "Relation >");
1710         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1711                       term_dst))
1712             return 0;
1713         term_value = atoi (term_tmp);
1714         gen_regular_rel(term_tmp, term_value+1, 0);
1715         break;
1716     case -1:
1717     case 3:
1718         yaz_log(log_level_rpn, "Relation =");
1719         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1720                       term_dst))
1721             return 0;
1722         term_value = atoi (term_tmp);
1723         sprintf(term_tmp, "(0*%d)", term_value);
1724         break;
1725     case 103:
1726         /* term_tmp untouched.. */
1727         while (**term_sub != '\0')
1728             (*term_sub)++;
1729         break;
1730     default:
1731         *error_code = YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
1732         return 0;
1733     }
1734     yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_tmp);
1735     r = dict_lookup_grep(zh->reg->dict, term_dict, 0, grep_info, max_pos,
1736                           0, grep_handle);
1737     if (r)
1738         yaz_log(YLOG_WARN, "dict_lookup_grep fail, rel = gt: %d", r);
1739     yaz_log(log_level_rpn, "%d positions", grep_info->isam_p_indx);
1740     return 1;
1741 }
1742
1743 static ZEBRA_RES numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1744                               const char **term_sub,
1745                               oid_value attributeSet, NMEM stream,
1746                               struct grep_info *grep_info,
1747                               int reg_type, int complete_flag,
1748                               int num_bases, char **basenames,
1749                               char *term_dst,
1750                               const char *xpath_use,
1751                               struct ord_list **ol)
1752 {
1753     char term_dict[2*IT_MAX_WORD+2];
1754     int base_no;
1755     const char *termp;
1756     struct rpn_char_map_info rcmi;
1757
1758     int bases_ok = 0;     /* no of databases with OK attribute */
1759
1760     *ol = ord_list_create(stream);
1761
1762     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1763
1764     for (base_no = 0; base_no < num_bases; base_no++)
1765     {
1766         int max_pos, prefix_len = 0;
1767         int relation_error = 0;
1768         int ord, ord_len, i;
1769         char ord_buf[32];
1770
1771         termp = *term_sub;
1772
1773         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1774         {
1775             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1776                            basenames[base_no]);
1777             return ZEBRA_FAIL;
1778         }
1779
1780         if (zebra_apt_get_ord(zh, zapt, reg_type, xpath_use,
1781                               attributeSet, &ord) != ZEBRA_OK)
1782             continue;
1783         bases_ok++;
1784
1785         *ol = ord_list_append(stream, *ol, ord);
1786
1787         ord_len = key_SU_encode (ord, ord_buf);
1788
1789         term_dict[prefix_len++] = '(';
1790         for (i = 0; i < ord_len; i++)
1791         {
1792             term_dict[prefix_len++] = 1;
1793             term_dict[prefix_len++] = ord_buf[i];
1794         }
1795         term_dict[prefix_len++] = ')';
1796         term_dict[prefix_len] = '\0';
1797
1798         if (!numeric_relation(zh, zapt, &termp, term_dict,
1799                               attributeSet, grep_info, &max_pos, reg_type,
1800                               term_dst, &relation_error))
1801         {
1802             if (relation_error)
1803             {
1804                 zebra_setError(zh, relation_error, 0);
1805                 return ZEBRA_FAIL;
1806             }
1807             *term_sub = 0;
1808             return ZEBRA_OK;
1809         }
1810     }
1811     if (!bases_ok)
1812         return ZEBRA_FAIL;
1813     *term_sub = termp;
1814     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1815     return ZEBRA_OK;
1816 }
1817
1818
1819 static ZEBRA_RES rpn_search_APT_numeric(ZebraHandle zh,
1820                                         Z_AttributesPlusTerm *zapt,
1821                                         const char *termz,
1822                                         oid_value attributeSet,
1823                                         NMEM stream,
1824                                         int reg_type, int complete_flag,
1825                                         const char *rank_type,
1826                                         const char *xpath_use,
1827                                         int num_bases, char **basenames,
1828                                         NMEM rset_nmem,
1829                                         RSET *rset,
1830                                         struct rset_key_control *kc)
1831 {
1832     char term_dst[IT_MAX_WORD+1];
1833     const char *termp = termz;
1834     RSET *result_sets = 0;
1835     int num_result_sets = 0;
1836     ZEBRA_RES res;
1837     struct grep_info grep_info;
1838     int alloc_sets = 0;
1839     zint hits_limit_value;
1840     const char *term_ref_id_str = 0;
1841
1842     term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str, stream);
1843
1844     yaz_log(log_level_rpn, "APT_numeric t='%s'", termz);
1845     if (grep_info_prepare(zh, zapt, &grep_info, reg_type) == ZEBRA_FAIL)
1846         return ZEBRA_FAIL;
1847     while (1)
1848     {
1849         struct ord_list *ol;
1850         if (alloc_sets == num_result_sets)
1851         {
1852             int add = 10;
1853             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1854                                               sizeof(*rnew));
1855             if (alloc_sets)
1856                 memcpy(rnew, result_sets, alloc_sets * sizeof(*rnew));
1857             alloc_sets = alloc_sets + add;
1858             result_sets = rnew;
1859         }
1860         yaz_log(YLOG_DEBUG, "APT_numeric termp=%s", termp);
1861         grep_info.isam_p_indx = 0;
1862         res = numeric_term(zh, zapt, &termp, attributeSet, stream, &grep_info,
1863                            reg_type, complete_flag, num_bases, basenames,
1864                            term_dst, xpath_use, &ol);
1865         if (res == ZEBRA_FAIL || termp == 0)
1866             break;
1867         yaz_log(YLOG_DEBUG, "term: %s", term_dst);
1868         result_sets[num_result_sets] =
1869             rset_trunc(zh, grep_info.isam_p_buf,
1870                        grep_info.isam_p_indx, term_dst,
1871                        strlen(term_dst), rank_type,
1872                        0 /* preserve position */,
1873                        zapt->term->which, rset_nmem,
1874                        kc, kc->scope, ol, reg_type,
1875                        hits_limit_value,
1876                        term_ref_id_str);
1877         if (!result_sets[num_result_sets])
1878             break;
1879         num_result_sets++;
1880         if (!*termp)
1881             break;
1882     }
1883     grep_info_delete(&grep_info);
1884
1885     if (res != ZEBRA_OK)
1886         return res;
1887     if (num_result_sets == 0)
1888         *rset = rset_create_null(rset_nmem, kc, 0);
1889     else if (num_result_sets == 1)
1890         *rset = result_sets[0];
1891     else
1892         *rset = rset_create_and(rset_nmem, kc, kc->scope,
1893                                 num_result_sets, result_sets);
1894     if (!*rset)
1895         return ZEBRA_FAIL;
1896     return ZEBRA_OK;
1897 }
1898
1899 static ZEBRA_RES rpn_search_APT_local(ZebraHandle zh,
1900                                       Z_AttributesPlusTerm *zapt,
1901                                       const char *termz,
1902                                       oid_value attributeSet,
1903                                       NMEM stream,
1904                                       const char *rank_type, NMEM rset_nmem,
1905                                       RSET *rset,
1906                                       struct rset_key_control *kc)
1907 {
1908     RSFD rsfd;
1909     struct it_key key;
1910     int sys;
1911     *rset = rset_create_temp(rset_nmem, kc, kc->scope,
1912                              res_get (zh->res, "setTmpDir"),0 );
1913     rsfd = rset_open(*rset, RSETF_WRITE);
1914
1915     sys = atoi(termz);
1916     if (sys <= 0)
1917         sys = 1;
1918     key.mem[0] = sys;
1919     key.mem[1] = 1;
1920     key.len = 2;
1921     rset_write (rsfd, &key);
1922     rset_close (rsfd);
1923     return ZEBRA_OK;
1924 }
1925
1926 static ZEBRA_RES rpn_sort_spec(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1927                                oid_value attributeSet, NMEM stream,
1928                                Z_SortKeySpecList *sort_sequence,
1929                                const char *rank_type,
1930                                NMEM rset_nmem,
1931                                RSET *rset,
1932                                struct rset_key_control *kc)
1933 {
1934     int i;
1935     int sort_relation_value;
1936     AttrType sort_relation_type;
1937     Z_SortKeySpec *sks;
1938     Z_SortKey *sk;
1939     int oid[OID_SIZE];
1940     oident oe;
1941     char termz[20];
1942
1943     attr_init_APT(&sort_relation_type, zapt, 7);
1944     sort_relation_value = attr_find(&sort_relation_type, &attributeSet);
1945
1946     if (!sort_sequence->specs)
1947     {
1948         sort_sequence->num_specs = 10;
1949         sort_sequence->specs = (Z_SortKeySpec **)
1950             nmem_malloc(stream, sort_sequence->num_specs *
1951                          sizeof(*sort_sequence->specs));
1952         for (i = 0; i<sort_sequence->num_specs; i++)
1953             sort_sequence->specs[i] = 0;
1954     }
1955     if (zapt->term->which != Z_Term_general)
1956         i = 0;
1957     else
1958         i = atoi_n ((char *) zapt->term->u.general->buf,
1959                     zapt->term->u.general->len);
1960     if (i >= sort_sequence->num_specs)
1961         i = 0;
1962     sprintf(termz, "%d", i);
1963
1964     oe.proto = PROTO_Z3950;
1965     oe.oclass = CLASS_ATTSET;
1966     oe.value = attributeSet;
1967     if (!oid_ent_to_oid (&oe, oid))
1968         return ZEBRA_FAIL;
1969
1970     sks = (Z_SortKeySpec *) nmem_malloc(stream, sizeof(*sks));
1971     sks->sortElement = (Z_SortElement *)
1972         nmem_malloc(stream, sizeof(*sks->sortElement));
1973     sks->sortElement->which = Z_SortElement_generic;
1974     sk = sks->sortElement->u.generic = (Z_SortKey *)
1975         nmem_malloc(stream, sizeof(*sk));
1976     sk->which = Z_SortKey_sortAttributes;
1977     sk->u.sortAttributes = (Z_SortAttributes *)
1978         nmem_malloc(stream, sizeof(*sk->u.sortAttributes));
1979
1980     sk->u.sortAttributes->id = oid;
1981     sk->u.sortAttributes->list = zapt->attributes;
1982
1983     sks->sortRelation = (int *)
1984         nmem_malloc(stream, sizeof(*sks->sortRelation));
1985     if (sort_relation_value == 1)
1986         *sks->sortRelation = Z_SortKeySpec_ascending;
1987     else if (sort_relation_value == 2)
1988         *sks->sortRelation = Z_SortKeySpec_descending;
1989     else
1990         *sks->sortRelation = Z_SortKeySpec_ascending;
1991
1992     sks->caseSensitivity = (int *)
1993         nmem_malloc(stream, sizeof(*sks->caseSensitivity));
1994     *sks->caseSensitivity = 0;
1995
1996     sks->which = Z_SortKeySpec_null;
1997     sks->u.null = odr_nullval ();
1998     sort_sequence->specs[i] = sks;
1999     *rset = rset_create_null(rset_nmem, kc, 0);
2000     return ZEBRA_OK;
2001 }
2002
2003
2004 static int rpn_check_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2005                            oid_value attributeSet,
2006                            struct xpath_location_step *xpath, int max,
2007                            NMEM mem)
2008 {
2009     oid_value curAttributeSet = attributeSet;
2010     AttrType use;
2011     const char *use_string = 0;
2012
2013     attr_init_APT(&use, zapt, 1);
2014     attr_find_ex(&use, &curAttributeSet, &use_string);
2015
2016     if (!use_string || *use_string != '/')
2017         return -1;
2018
2019     return zebra_parse_xpath_str(use_string, xpath, max, mem);
2020 }
2021
2022
2023
2024 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
2025                         int reg_type, const char *term,
2026                         const char *xpath_use,
2027                         NMEM rset_nmem,
2028                         struct rset_key_control *kc)
2029 {
2030     RSET rset;
2031     struct grep_info grep_info;
2032     char term_dict[2048];
2033     char ord_buf[32];
2034     int prefix_len = 0;
2035     int ord = zebraExplain_lookup_attr_str(zh->reg->zei,
2036                                            zinfo_index_category_index,
2037                                            reg_type,
2038                                            xpath_use);
2039     int ord_len, i, r, max_pos;
2040     int term_type = Z_Term_characterString;
2041     const char *flags = "void";
2042
2043     if (grep_info_prepare(zh, 0 /* zapt */, &grep_info, '0') == ZEBRA_FAIL)
2044         return rset_create_null(rset_nmem, kc, 0);
2045
2046     if (ord < 0)
2047         return rset_create_null(rset_nmem, kc, 0);
2048     if (prefix_len)
2049         term_dict[prefix_len++] = '|';
2050     else
2051         term_dict[prefix_len++] = '(';
2052
2053     ord_len = key_SU_encode (ord, ord_buf);
2054     for (i = 0; i<ord_len; i++)
2055     {
2056         term_dict[prefix_len++] = 1;
2057         term_dict[prefix_len++] = ord_buf[i];
2058     }
2059     term_dict[prefix_len++] = ')';
2060     strcpy(term_dict+prefix_len, term);
2061
2062     grep_info.isam_p_indx = 0;
2063     r = dict_lookup_grep(zh->reg->dict, term_dict, 0,
2064                           &grep_info, &max_pos, 0, grep_handle);
2065     yaz_log(YLOG_DEBUG, "%s %d positions", term,
2066              grep_info.isam_p_indx);
2067     rset = rset_trunc(zh, grep_info.isam_p_buf,
2068                       grep_info.isam_p_indx, term, strlen(term),
2069                       flags, 1, term_type,rset_nmem,
2070                       kc, kc->scope, 0, reg_type, 0 /* hits_limit */,
2071                       0 /* term_ref_id_str */);
2072     grep_info_delete(&grep_info);
2073     return rset;
2074 }
2075
2076 static
2077 ZEBRA_RES rpn_search_xpath(ZebraHandle zh,
2078                            int num_bases, char **basenames,
2079                            NMEM stream, const char *rank_type, RSET rset,
2080                            int xpath_len, struct xpath_location_step *xpath,
2081                            NMEM rset_nmem,
2082                            RSET *rset_out,
2083                            struct rset_key_control *kc)
2084 {
2085     int base_no;
2086     int i;
2087     int always_matches = rset ? 0 : 1;
2088
2089     if (xpath_len < 0)
2090     {
2091         *rset_out = rset;
2092         return ZEBRA_OK;
2093     }
2094
2095     yaz_log(YLOG_DEBUG, "xpath len=%d", xpath_len);
2096     for (i = 0; i<xpath_len; i++)
2097     {
2098         yaz_log(log_level_rpn, "XPATH %d %s", i, xpath[i].part);
2099
2100     }
2101
2102     /*
2103       //a    ->    a/.*
2104       //a/b  ->    b/a/.*
2105       /a     ->    a/
2106       /a/b   ->    b/a/
2107
2108       /      ->    none
2109
2110    a[@attr = value]/b[@other = othervalue]
2111
2112  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
2113  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
2114  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
2115  /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y)
2116  /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y)
2117  /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x)
2118
2119     */
2120
2121     dict_grep_cmap (zh->reg->dict, 0, 0);
2122
2123     for (base_no = 0; base_no < num_bases; base_no++)
2124     {
2125         int level = xpath_len;
2126         int first_path = 1;
2127
2128         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2129         {
2130             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
2131                            basenames[base_no]);
2132             *rset_out = rset;
2133             return ZEBRA_FAIL;
2134         }
2135         while (--level >= 0)
2136         {
2137             WRBUF xpath_rev = wrbuf_alloc();
2138             int i;
2139             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
2140
2141             for (i = level; i >= 1; --i)
2142             {
2143                 const char *cp = xpath[i].part;
2144                 if (*cp)
2145                 {
2146                     for (; *cp; cp++)
2147                     {
2148                         if (*cp == '*')
2149                             wrbuf_puts(xpath_rev, "[^/]*");
2150                         else if (*cp == ' ')
2151                             wrbuf_puts(xpath_rev, "\001 ");
2152                         else
2153                             wrbuf_putc(xpath_rev, *cp);
2154
2155                         /* wrbuf_putc does not null-terminate , but
2156                            wrbuf_puts below ensures it does.. so xpath_rev
2157                            is OK iff length is > 0 */
2158                     }
2159                     wrbuf_puts(xpath_rev, "/");
2160                 }
2161                 else if (i == 1)  /* // case */
2162                     wrbuf_puts(xpath_rev, ".*");
2163             }
2164             if (xpath[level].predicate &&
2165                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
2166                 xpath[level].predicate->u.relation.name[0])
2167             {
2168                 WRBUF wbuf = wrbuf_alloc();
2169                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
2170                 if (xpath[level].predicate->u.relation.value)
2171                 {
2172                     const char *cp = xpath[level].predicate->u.relation.value;
2173                     wrbuf_putc(wbuf, '=');
2174
2175                     while (*cp)
2176                     {
2177                         if (strchr(REGEX_CHARS, *cp))
2178                             wrbuf_putc(wbuf, '\\');
2179                         wrbuf_putc(wbuf, *cp);
2180                         cp++;
2181                     }
2182                 }
2183                 wrbuf_puts(wbuf, "");
2184                 rset_attr = xpath_trunc(
2185                     zh, stream, '0', wrbuf_buf(wbuf), ZEBRA_XPATH_ATTR_NAME,
2186                     rset_nmem, kc);
2187                 wrbuf_free(wbuf, 1);
2188             }
2189             else
2190             {
2191                 if (!first_path)
2192                 {
2193                     wrbuf_free(xpath_rev, 1);
2194                     continue;
2195                 }
2196             }
2197             yaz_log(log_level_rpn, "xpath_rev (%d) = %.*s", level,
2198                     wrbuf_len(xpath_rev), wrbuf_buf(xpath_rev));
2199             if (wrbuf_len(xpath_rev))
2200             {
2201                 rset_start_tag = xpath_trunc(zh, stream, '0',
2202                                              wrbuf_buf(xpath_rev),
2203                                              ZEBRA_XPATH_ELM_BEGIN,
2204                                              rset_nmem, kc);
2205                 if (always_matches)
2206                     rset = rset_start_tag;
2207                 else
2208                 {
2209                     rset_end_tag = xpath_trunc(zh, stream, '0',
2210                                                wrbuf_buf(xpath_rev),
2211                                                ZEBRA_XPATH_ELM_END,
2212                                                rset_nmem, kc);
2213
2214                     rset = rset_create_between(rset_nmem, kc, kc->scope,
2215                                                rset_start_tag, rset,
2216                                                rset_end_tag, rset_attr);
2217                 }
2218             }
2219             wrbuf_free(xpath_rev, 1);
2220             first_path = 0;
2221         }
2222     }
2223     *rset_out = rset;
2224     return ZEBRA_OK;
2225 }
2226
2227 #define MAX_XPATH_STEPS 10
2228
2229 static ZEBRA_RES rpn_search_APT(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2230                                 oid_value attributeSet, NMEM stream,
2231                                 Z_SortKeySpecList *sort_sequence,
2232                                 int num_bases, char **basenames,
2233                                 NMEM rset_nmem,
2234                                 RSET *rset,
2235                                 struct rset_key_control *kc)
2236 {
2237     ZEBRA_RES res = ZEBRA_OK;
2238     unsigned reg_id;
2239     char *search_type = NULL;
2240     char rank_type[128];
2241     int complete_flag;
2242     int sort_flag;
2243     char termz[IT_MAX_WORD+1];
2244     int xpath_len;
2245     const char *xpath_use = 0;
2246     struct xpath_location_step xpath[MAX_XPATH_STEPS];
2247
2248     if (!log_level_set)
2249     {
2250         log_level_rpn = yaz_log_module_level("rpn");
2251         log_level_set = 1;
2252     }
2253     zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2254                     rank_type, &complete_flag, &sort_flag);
2255
2256     yaz_log(YLOG_DEBUG, "reg_id=%c", reg_id);
2257     yaz_log(YLOG_DEBUG, "complete_flag=%d", complete_flag);
2258     yaz_log(YLOG_DEBUG, "search_type=%s", search_type);
2259     yaz_log(YLOG_DEBUG, "rank_type=%s", rank_type);
2260
2261     if (zapt_term_to_utf8(zh, zapt, termz) == ZEBRA_FAIL)
2262         return ZEBRA_FAIL;
2263
2264     if (sort_flag)
2265         return rpn_sort_spec(zh, zapt, attributeSet, stream, sort_sequence,
2266                              rank_type, rset_nmem, rset, kc);
2267     /* consider if an X-Path query is used */
2268     xpath_len = rpn_check_xpath(zh, zapt, attributeSet,
2269                                 xpath, MAX_XPATH_STEPS, stream);
2270     if (xpath_len >= 0)
2271     {
2272         if (xpath[xpath_len-1].part[0] == '@')
2273             xpath_use = ZEBRA_XPATH_ATTR_CDATA;  /* last step is attribute  */
2274         else
2275             xpath_use = ZEBRA_XPATH_CDATA;  /* searching for cdata */
2276
2277         if (1)
2278         {
2279             AttrType relation;
2280             int relation_value;
2281
2282             attr_init_APT(&relation, zapt, 2);
2283             relation_value = attr_find(&relation, NULL);
2284
2285             if (relation_value == 103) /* alwaysmatches */
2286             {
2287                 *rset = 0; /* signal no "term" set */
2288                 return rpn_search_xpath(zh, num_bases, basenames,
2289                                         stream, rank_type, *rset,
2290                                         xpath_len, xpath, rset_nmem, rset, kc);
2291             }
2292         }
2293     }
2294
2295     /* search using one of the various search type strategies
2296        termz is our UTF-8 search term
2297        attributeSet is top-level default attribute set
2298        stream is ODR for search
2299        reg_id is the register type
2300        complete_flag is 1 for complete subfield, 0 for incomplete
2301        xpath_use is use-attribute to be used for X-Path search, 0 for none
2302     */
2303     if (!strcmp(search_type, "phrase"))
2304     {
2305         res = rpn_search_APT_phrase(zh, zapt, termz, attributeSet, stream,
2306                                     reg_id, complete_flag, rank_type,
2307                                     xpath_use,
2308                                     num_bases, basenames, rset_nmem,
2309                                     rset, kc);
2310     }
2311     else if (!strcmp(search_type, "and-list"))
2312     {
2313         res = rpn_search_APT_and_list(zh, zapt, termz, attributeSet, stream,
2314                                       reg_id, complete_flag, rank_type,
2315                                       xpath_use,
2316                                       num_bases, basenames, rset_nmem,
2317                                       rset, kc);
2318     }
2319     else if (!strcmp(search_type, "or-list"))
2320     {
2321         res = rpn_search_APT_or_list(zh, zapt, termz, attributeSet, stream,
2322                                      reg_id, complete_flag, rank_type,
2323                                      xpath_use,
2324                                      num_bases, basenames, rset_nmem,
2325                                      rset, kc);
2326     }
2327     else if (!strcmp(search_type, "local"))
2328     {
2329         res = rpn_search_APT_local(zh, zapt, termz, attributeSet, stream,
2330                                    rank_type, rset_nmem, rset, kc);
2331     }
2332     else if (!strcmp(search_type, "numeric"))
2333     {
2334         res = rpn_search_APT_numeric(zh, zapt, termz, attributeSet, stream,
2335                                      reg_id, complete_flag, rank_type,
2336                                      xpath_use,
2337                                      num_bases, basenames, rset_nmem,
2338                                      rset, kc);
2339     }
2340     else
2341     {
2342         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2343         res = ZEBRA_FAIL;
2344     }
2345     if (res != ZEBRA_OK)
2346         return res;
2347     if (!*rset)
2348         return ZEBRA_FAIL;
2349     return rpn_search_xpath(zh, num_bases, basenames,
2350                             stream, rank_type, *rset,
2351                             xpath_len, xpath, rset_nmem, rset, kc);
2352 }
2353
2354 static ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2355                                       oid_value attributeSet,
2356                                       NMEM stream, NMEM rset_nmem,
2357                                       Z_SortKeySpecList *sort_sequence,
2358                                       int num_bases, char **basenames,
2359                                       RSET **result_sets, int *num_result_sets,
2360                                       Z_Operator *parent_op,
2361                                       struct rset_key_control *kc);
2362
2363 ZEBRA_RES rpn_search_top(ZebraHandle zh, Z_RPNStructure *zs,
2364                          oid_value attributeSet,
2365                          NMEM stream, NMEM rset_nmem,
2366                          Z_SortKeySpecList *sort_sequence,
2367                          int num_bases, char **basenames,
2368                          RSET *result_set)
2369 {
2370     RSET *result_sets = 0;
2371     int num_result_sets = 0;
2372     ZEBRA_RES res;
2373     struct rset_key_control *kc = zebra_key_control_create(zh);
2374
2375     res = rpn_search_structure(zh, zs, attributeSet,
2376                                stream, rset_nmem,
2377                                sort_sequence,
2378                                num_bases, basenames,
2379                                &result_sets, &num_result_sets,
2380                                0 /* no parent op */,
2381                                kc);
2382     if (res != ZEBRA_OK)
2383     {
2384         int i;
2385         for (i = 0; i<num_result_sets; i++)
2386             rset_delete(result_sets[i]);
2387         *result_set = 0;
2388     }
2389     else
2390     {
2391         assert(num_result_sets == 1);
2392         assert(result_sets);
2393         assert(*result_sets);
2394         *result_set = *result_sets;
2395     }
2396     (*kc->dec)(kc);
2397     return res;
2398 }
2399
2400 ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2401                                oid_value attributeSet,
2402                                NMEM stream, NMEM rset_nmem,
2403                                Z_SortKeySpecList *sort_sequence,
2404                                int num_bases, char **basenames,
2405                                RSET **result_sets, int *num_result_sets,
2406                                Z_Operator *parent_op,
2407                                struct rset_key_control *kc)
2408 {
2409     *num_result_sets = 0;
2410     if (zs->which == Z_RPNStructure_complex)
2411     {
2412         ZEBRA_RES res;
2413         Z_Operator *zop = zs->u.complex->roperator;
2414         RSET *result_sets_l = 0;
2415         int num_result_sets_l = 0;
2416         RSET *result_sets_r = 0;
2417         int num_result_sets_r = 0;
2418
2419         res = rpn_search_structure(zh, zs->u.complex->s1,
2420                                    attributeSet, stream, rset_nmem,
2421                                    sort_sequence,
2422                                    num_bases, basenames,
2423                                    &result_sets_l, &num_result_sets_l,
2424                                    zop, kc);
2425         if (res != ZEBRA_OK)
2426         {
2427             int i;
2428             for (i = 0; i<num_result_sets_l; i++)
2429                 rset_delete(result_sets_l[i]);
2430             return res;
2431         }
2432         res = rpn_search_structure(zh, zs->u.complex->s2,
2433                                    attributeSet, stream, rset_nmem,
2434                                    sort_sequence,
2435                                    num_bases, basenames,
2436                                    &result_sets_r, &num_result_sets_r,
2437                                    zop, kc);
2438         if (res != ZEBRA_OK)
2439         {
2440             int i;
2441             for (i = 0; i<num_result_sets_l; i++)
2442                 rset_delete(result_sets_l[i]);
2443             for (i = 0; i<num_result_sets_r; i++)
2444                 rset_delete(result_sets_r[i]);
2445             return res;
2446         }
2447
2448         /* make a new list of result for all children */
2449         *num_result_sets = num_result_sets_l + num_result_sets_r;
2450         *result_sets = nmem_malloc(stream, *num_result_sets *
2451                                    sizeof(**result_sets));
2452         memcpy(*result_sets, result_sets_l,
2453                num_result_sets_l * sizeof(**result_sets));
2454         memcpy(*result_sets + num_result_sets_l, result_sets_r,
2455                num_result_sets_r * sizeof(**result_sets));
2456
2457         if (!parent_op || parent_op->which != zop->which
2458             || (zop->which != Z_Operator_and &&
2459                 zop->which != Z_Operator_or))
2460         {
2461             /* parent node different from this one (or non-present) */
2462             /* we must combine result sets now */
2463             RSET rset;
2464             switch (zop->which)
2465             {
2466             case Z_Operator_and:
2467                 rset = rset_create_and(rset_nmem, kc,
2468                                        kc->scope,
2469                                        *num_result_sets, *result_sets);
2470                 break;
2471             case Z_Operator_or:
2472                 rset = rset_create_or(rset_nmem, kc,
2473                                       kc->scope, 0, /* termid */
2474                                       *num_result_sets, *result_sets);
2475                 break;
2476             case Z_Operator_and_not:
2477                 rset = rset_create_not(rset_nmem, kc,
2478                                        kc->scope,
2479                                        (*result_sets)[0],
2480                                        (*result_sets)[1]);
2481                 break;
2482             case Z_Operator_prox:
2483                 if (zop->u.prox->which != Z_ProximityOperator_known)
2484                 {
2485                     zebra_setError(zh,
2486                                    YAZ_BIB1_UNSUPP_PROX_UNIT_CODE,
2487                                    0);
2488                     return ZEBRA_FAIL;
2489                 }
2490                 if (*zop->u.prox->u.known != Z_ProxUnit_word)
2491                 {
2492                     zebra_setError_zint(zh,
2493                                         YAZ_BIB1_UNSUPP_PROX_UNIT_CODE,
2494                                         *zop->u.prox->u.known);
2495                     return ZEBRA_FAIL;
2496                 }
2497                 else
2498                 {
2499                     rset = rset_create_prox(rset_nmem, kc,
2500                                             kc->scope,
2501                                             *num_result_sets, *result_sets,
2502                                             *zop->u.prox->ordered,
2503                                             (!zop->u.prox->exclusion ?
2504                                              0 : *zop->u.prox->exclusion),
2505                                             *zop->u.prox->relationType,
2506                                             *zop->u.prox->distance );
2507                 }
2508                 break;
2509             default:
2510                 zebra_setError(zh, YAZ_BIB1_OPERATOR_UNSUPP, 0);
2511                 return ZEBRA_FAIL;
2512             }
2513             *num_result_sets = 1;
2514             *result_sets = nmem_malloc(stream, *num_result_sets *
2515                                        sizeof(**result_sets));
2516             (*result_sets)[0] = rset;
2517         }
2518     }
2519     else if (zs->which == Z_RPNStructure_simple)
2520     {
2521         RSET rset;
2522         ZEBRA_RES res;
2523
2524         if (zs->u.simple->which == Z_Operand_APT)
2525         {
2526             yaz_log(YLOG_DEBUG, "rpn_search_APT");
2527             res = rpn_search_APT(zh, zs->u.simple->u.attributesPlusTerm,
2528                                  attributeSet, stream, sort_sequence,
2529                                  num_bases, basenames, rset_nmem, &rset,
2530                                  kc);
2531             if (res != ZEBRA_OK)
2532                 return res;
2533         }
2534         else if (zs->u.simple->which == Z_Operand_resultSetId)
2535         {
2536             yaz_log(YLOG_DEBUG, "rpn_search_ref");
2537             rset = resultSetRef(zh, zs->u.simple->u.resultSetId);
2538             if (!rset)
2539             {
2540                 zebra_setError(zh,
2541                                YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
2542                                zs->u.simple->u.resultSetId);
2543                 return ZEBRA_FAIL;
2544             }
2545             rset_dup(rset);
2546         }
2547         else
2548         {
2549             zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2550             return ZEBRA_FAIL;
2551         }
2552         *num_result_sets = 1;
2553         *result_sets = nmem_malloc(stream, *num_result_sets *
2554                                    sizeof(**result_sets));
2555         (*result_sets)[0] = rset;
2556     }
2557     else
2558     {
2559         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2560         return ZEBRA_FAIL;
2561     }
2562     return ZEBRA_OK;
2563 }
2564
2565 struct scan_info_entry {
2566     char *term;
2567     ISAM_P isam_p;
2568 };
2569
2570 struct scan_info {
2571     struct scan_info_entry *list;
2572     ODR odr;
2573     int before, after;
2574     char prefix[20];
2575 };
2576
2577 static int scan_handle (char *name, const char *info, int pos, void *client)
2578 {
2579     int len_prefix, idx;
2580     struct scan_info *scan_info = (struct scan_info *) client;
2581
2582     len_prefix = strlen(scan_info->prefix);
2583     if (memcmp (name, scan_info->prefix, len_prefix))
2584         return 1;
2585     if (pos > 0)
2586         idx = scan_info->after - pos + scan_info->before;
2587     else
2588         idx = - pos - 1;
2589
2590     /* skip special terms.. of no interest */
2591     if (name[len_prefix] < 4)
2592         return 1;
2593
2594     if (idx < 0)
2595         return 0;
2596     scan_info->list[idx].term = (char *)
2597         odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
2598     strcpy(scan_info->list[idx].term, name + len_prefix);
2599     assert (*info == sizeof(ISAM_P));
2600     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
2601     return 0;
2602 }
2603
2604 void zebra_term_untrans_iconv(ZebraHandle zh, NMEM stream, int reg_type,
2605                               char **dst, const char *src)
2606 {
2607     char term_src[IT_MAX_WORD];
2608     char term_dst[IT_MAX_WORD];
2609
2610     zebra_term_untrans (zh, reg_type, term_src, src);
2611
2612     if (zh->iconv_from_utf8 != 0)
2613     {
2614         int len;
2615         char *inbuf = term_src;
2616         size_t inleft = strlen(term_src);
2617         char *outbuf = term_dst;
2618         size_t outleft = sizeof(term_dst)-1;
2619         size_t ret;
2620
2621         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2622                          &outbuf, &outleft);
2623         if (ret == (size_t)(-1))
2624             len = 0;
2625         else
2626             len = outbuf - term_dst;
2627         *dst = nmem_malloc(stream, len + 1);
2628         if (len > 0)
2629             memcpy (*dst, term_dst, len);
2630         (*dst)[len] = '\0';
2631     }
2632     else
2633         *dst = nmem_strdup(stream, term_src);
2634 }
2635
2636 static void count_set(ZebraHandle zh, RSET rset, zint *count)
2637 {
2638     zint psysno = 0;
2639     struct it_key key;
2640     RSFD rfd;
2641
2642     yaz_log(YLOG_DEBUG, "count_set");
2643
2644     rset->hits_limit = zh->approx_limit;
2645
2646     *count = 0;
2647     rfd = rset_open(rset, RSETF_READ);
2648     while (rset_read(rfd, &key,0 /* never mind terms */))
2649     {
2650         if (key.mem[0] != psysno)
2651         {
2652             psysno = key.mem[0];
2653             if (rfd->counted_items >= rset->hits_limit)
2654                 break;
2655         }
2656     }
2657     rset_close (rfd);
2658     *count = rset->hits_count;
2659 }
2660
2661 #define RPN_MAX_ORDS 32
2662
2663 ZEBRA_RES rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2664                    oid_value attributeset,
2665                    int num_bases, char **basenames,
2666                    int *position, int *num_entries, ZebraScanEntry **list,
2667                    int *is_partial, RSET limit_set, int return_zero)
2668 {
2669     int i;
2670     int pos = *position;
2671     int num = *num_entries;
2672     int before;
2673     int after;
2674     int base_no;
2675     char termz[IT_MAX_WORD+20];
2676     struct scan_info *scan_info_array;
2677     ZebraScanEntry *glist;
2678     int ords[RPN_MAX_ORDS], ord_no = 0;
2679     int ptr[RPN_MAX_ORDS];
2680
2681     unsigned index_type;
2682     char *search_type = NULL;
2683     char rank_type[128];
2684     int complete_flag;
2685     int sort_flag;
2686     NMEM rset_nmem = NULL;
2687     struct rset_key_control *kc = 0;
2688
2689     *list = 0;
2690     *is_partial = 0;
2691
2692     if (attributeset == VAL_NONE)
2693         attributeset = VAL_BIB1;
2694
2695     if (!limit_set)
2696     {
2697         AttrType termset;
2698         int termset_value_numeric;
2699         const char *termset_value_string;
2700         attr_init_APT(&termset, zapt, 8);
2701         termset_value_numeric =
2702             attr_find_ex(&termset, NULL, &termset_value_string);
2703         if (termset_value_numeric != -1)
2704         {
2705             char resname[32];
2706             const char *termset_name = 0;
2707
2708             if (termset_value_numeric != -2)
2709             {
2710
2711                 sprintf(resname, "%d", termset_value_numeric);
2712                 termset_name = resname;
2713             }
2714             else
2715                 termset_name = termset_value_string;
2716
2717             limit_set = resultSetRef (zh, termset_name);
2718         }
2719     }
2720
2721     yaz_log(YLOG_DEBUG, "position = %d, num = %d set=%d",
2722             pos, num, attributeset);
2723
2724     if (zebra_maps_attr(zh->reg->zebra_maps, zapt, &index_type, &search_type,
2725                         rank_type, &complete_flag, &sort_flag))
2726     {
2727         *num_entries = 0;
2728         zebra_setError(zh, YAZ_BIB1_UNSUPP_ATTRIBUTE_TYPE, 0);
2729         return ZEBRA_FAIL;
2730     }
2731     for (base_no = 0; base_no < num_bases && ord_no < RPN_MAX_ORDS; base_no++)
2732     {
2733         int ord;
2734
2735         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2736         {
2737             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
2738                            basenames[base_no]);
2739             *num_entries = 0;
2740             return ZEBRA_FAIL;
2741         }
2742         if (zebra_apt_get_ord(zh, zapt, index_type, 0, attributeset, &ord)
2743             != ZEBRA_OK)
2744             continue;
2745         ords[ord_no++] = ord;
2746     }
2747     if (ord_no == 0)
2748     {
2749         *num_entries = 0;
2750         return ZEBRA_FAIL;
2751     }
2752     /* prepare dictionary scanning */
2753     if (num < 1)
2754     {
2755         *num_entries = 0;
2756         return ZEBRA_OK;
2757     }
2758     before = pos-1;
2759     if (before < 0)
2760         before = 0;
2761     after = 1+num-pos;
2762     if (after < 0)
2763         after = 0;
2764     yaz_log(YLOG_DEBUG, "rpn_scan pos=%d num=%d before=%d "
2765             "after=%d before+after=%d",
2766             pos, num, before, after, before+after);
2767     scan_info_array = (struct scan_info *)
2768         odr_malloc(stream, ord_no * sizeof(*scan_info_array));
2769     for (i = 0; i < ord_no; i++)
2770     {
2771         int j, prefix_len = 0;
2772         int before_tmp = before, after_tmp = after;
2773         struct scan_info *scan_info = scan_info_array + i;
2774         struct rpn_char_map_info rcmi;
2775
2776         rpn_char_map_prepare (zh->reg, index_type, &rcmi);
2777
2778         scan_info->before = before;
2779         scan_info->after = after;
2780         scan_info->odr = stream;
2781
2782         scan_info->list = (struct scan_info_entry *)
2783             odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
2784         for (j = 0; j<before+after; j++)
2785             scan_info->list[j].term = NULL;
2786
2787         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2788         termz[prefix_len] = 0;
2789         strcpy(scan_info->prefix, termz);
2790
2791         if (trans_scan_term(zh, zapt, termz+prefix_len, index_type) ==
2792             ZEBRA_FAIL)
2793             return ZEBRA_FAIL;
2794
2795         dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
2796                   scan_info, scan_handle);
2797     }
2798     glist = (ZebraScanEntry *)
2799         odr_malloc(stream, (before+after)*sizeof(*glist));
2800
2801     rset_nmem = nmem_create();
2802     kc = zebra_key_control_create(zh);
2803
2804     /* consider terms after main term */
2805     for (i = 0; i < ord_no; i++)
2806         ptr[i] = before;
2807
2808     *is_partial = 0;
2809     for (i = 0; i<after; i++)
2810     {
2811         int j, j0 = -1;
2812         const char *mterm = NULL;
2813         const char *tst;
2814         RSET rset = 0;
2815         int lo = i + pos-1; /* offset in result list */
2816
2817         /* find: j0 is the first of the minimal values */
2818         for (j = 0; j < ord_no; j++)
2819         {
2820             if (ptr[j] < before+after && ptr[j] >= 0 &&
2821                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2822                 (!mterm || strcmp (tst, mterm) < 0))
2823             {
2824                 j0 = j;
2825                 mterm = tst;
2826             }
2827         }
2828         if (j0 == -1)
2829             break;  /* no value found, stop */
2830
2831         /* get result set for first one , but only if it's within bounds */
2832         if (lo >= 0)
2833         {
2834             /* get result set for first term */
2835             zebra_term_untrans_iconv(zh, stream->mem, index_type,
2836                                      &glist[lo].term, mterm);
2837             rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2838                               glist[lo].term, strlen(glist[lo].term),
2839                               NULL, 0, zapt->term->which, rset_nmem,
2840                               kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2841                               0 /* term_ref_id_str */);
2842         }
2843         ptr[j0]++; /* move index for this set .. */
2844         /* get result set for remaining scan terms */
2845         for (j = j0+1; j<ord_no; j++)
2846         {
2847             if (ptr[j] < before+after && ptr[j] >= 0 &&
2848                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2849                 !strcmp (tst, mterm))
2850             {
2851                 if (lo >= 0)
2852                 {
2853                     RSET rsets[2];
2854
2855                     rsets[0] = rset;
2856                     rsets[1] =
2857                         rset_trunc(
2858                             zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
2859                             glist[lo].term,
2860                             strlen(glist[lo].term), NULL, 0,
2861                             zapt->term->which,rset_nmem,
2862                             kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2863                             0 /* term_ref_id_str */ );
2864                     rset = rset_create_or(rset_nmem, kc,
2865                                           kc->scope, 0 /* termid */,
2866                                           2, rsets);
2867                 }
2868                 ptr[j]++;
2869             }
2870         }
2871         if (lo >= 0)
2872         {
2873             zint count;
2874             /* merge with limit_set if given */
2875             if (limit_set)
2876             {
2877                 RSET rsets[2];
2878                 rsets[0] = rset;
2879                 rsets[1] = rset_dup(limit_set);
2880
2881                 rset = rset_create_and(rset_nmem, kc, kc->scope, 2, rsets);
2882             }
2883             /* count it */
2884             count_set(zh, rset, &count);
2885             glist[lo].occurrences = count;
2886             rset_delete(rset);
2887         }
2888     }
2889     if (i < after)
2890     {
2891         *num_entries -= (after-i);
2892         *is_partial = 1;
2893         if (*num_entries < 0)
2894         {
2895             (*kc->dec)(kc);
2896             nmem_destroy(rset_nmem);
2897             *num_entries = 0;
2898             return ZEBRA_OK;
2899         }
2900     }
2901     /* consider terms before main term */
2902     for (i = 0; i<ord_no; i++)
2903         ptr[i] = 0;
2904
2905     for (i = 0; i<before; i++)
2906     {
2907         int j, j0 = -1;
2908         const char *mterm = NULL;
2909         const char *tst;
2910         RSET rset;
2911         int lo = before-1-i; /* offset in result list */
2912         zint count;
2913
2914         for (j = 0; j <ord_no; j++)
2915         {
2916             if (ptr[j] < before && ptr[j] >= 0 &&
2917                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2918                 (!mterm || strcmp (tst, mterm) > 0))
2919             {
2920                 j0 = j;
2921                     mterm = tst;
2922             }
2923         }
2924         if (j0 == -1)
2925             break;
2926
2927         zebra_term_untrans_iconv(zh, stream->mem, index_type,
2928                                  &glist[lo].term, mterm);
2929
2930         rset = rset_trunc
2931             (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
2932              glist[lo].term, strlen(glist[lo].term),
2933              NULL, 0, zapt->term->which, rset_nmem,
2934              kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2935              0 /* term_ref_id_str */);
2936
2937         ptr[j0]++;
2938
2939         for (j = j0+1; j<ord_no; j++)
2940         {
2941             if (ptr[j] < before && ptr[j] >= 0 &&
2942                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2943                 !strcmp (tst, mterm))
2944             {
2945                 RSET rsets[2];
2946
2947                 rsets[0] = rset;
2948                 rsets[1] = rset_trunc(
2949                     zh,
2950                     &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
2951                     glist[lo].term,
2952                     strlen(glist[lo].term), NULL, 0,
2953                     zapt->term->which, rset_nmem,
2954                     kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2955                     0 /* term_ref_id_str */);
2956                 rset = rset_create_or(rset_nmem, kc,
2957                                       kc->scope, 0 /* termid */, 2, rsets);
2958
2959                 ptr[j]++;
2960             }
2961         }
2962         if (limit_set)
2963         {
2964             RSET rsets[2];
2965             rsets[0] = rset;
2966             rsets[1] = rset_dup(limit_set);
2967
2968             rset = rset_create_and(rset_nmem, kc, kc->scope, 2, rsets);
2969         }
2970         count_set(zh, rset, &count);
2971         glist[lo].occurrences = count;
2972         rset_delete (rset);
2973     }
2974     (*kc->dec)(kc);
2975     nmem_destroy(rset_nmem);
2976     i = before-i;
2977     if (i)
2978     {
2979         *is_partial = 1;
2980         *position -= i;
2981         *num_entries -= i;
2982         if (*num_entries <= 0)
2983         {
2984             *num_entries = 0;
2985             return ZEBRA_OK;
2986         }
2987     }
2988
2989     *list = glist + i;               /* list is set to first 'real' entry */
2990
2991     yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
2992             *position, *num_entries);
2993     return ZEBRA_OK;
2994 }
2995
2996 /*
2997  * Local variables:
2998  * c-basic-offset: 4
2999  * indent-tabs-mode: nil
3000  * End:
3001  * vim: shiftwidth=4 tabstop=8 expandtab
3002  */
3003