index/zrpn.c

   1 /* $Id: zrpn.c,v 1.229 2006-09-08 18:24:53 adam Exp $
   2    Copyright (C) 1995-2006
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with this program; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #ifdef WIN32
  26 #include <io.h>
  27 #endif
  28 #if HAVE_UNISTD_H
  29 #include <unistd.h>
  30 #endif
  31 #include <ctype.h>
  32
  33 #include <yaz/diagbib1.h>
  34 #include "index.h"
  35 #include <zebra_xpath.h>
  36 #include <attrfind.h>
  37 #include <charmap.h>
  38 #include <rset.h>
  39
  40 struct rpn_char_map_info
  41 {
  42     ZebraMaps zm;
  43     int reg_type;
  44 };
  45
  46 static int log_level_set = 0;
  47 static int log_level_rpn = 0;
  48
  49 #define TERMSET_DISABLE 1
  50
  51 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  52 {
  53     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  54     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  55 #if 0
  56     if (out && *out)
  57     {
  58         const char *outp = *out;
  59         yaz_log(YLOG_LOG, "---");
  60         while (*outp)
  61         {
  62             yaz_log(YLOG_LOG, "%02X", *outp);
  63             outp++;
  64         }
  65     }
  66 #endif
  67     return out;
  68 }
  69
  70 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  71                                  struct rpn_char_map_info *map_info)
  72 {
  73     map_info->zm = reg->zebra_maps;
  74     map_info->reg_type = reg_type;
  75     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  76 }
  77
  78 #define TERM_COUNT
  79
  80 struct grep_info {
  81 #ifdef TERM_COUNT
  82     int *term_no;
  83 #endif
  84     ISAM_P *isam_p_buf;
  85     int isam_p_size;
  86     int isam_p_indx;
  87     ZebraHandle zh;
  88     int reg_type;
  89     ZebraSet termset;
  90 };
  91
  92 void zebra_term_untrans(ZebraHandle zh, int reg_type,
  93                         char *dst, const char *src)
  94 {
  95     int len = 0;
  96     while (*src)
  97     {
  98         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
  99                                            reg_type, &src);
 100         if (!cp)
 101         {
 102             if (len < IT_MAX_WORD-1)
 103                 dst[len++] = *src;
 104             src++;
 105         }
 106         else
 107             while (*cp && len < IT_MAX_WORD-1)
 108                 dst[len++] = *cp++;
 109     }
 110     dst[len] = '\0';
 111 }
 112
 113 static void add_isam_p(const char *name, const char *info,
 114                        struct grep_info *p)
 115 {
 116     if (!log_level_set)
 117     {
 118         log_level_rpn = yaz_log_module_level("rpn");
 119         log_level_set = 1;
 120     }
 121     if (p->isam_p_indx == p->isam_p_size)
 122     {
 123         ISAM_P *new_isam_p_buf;
 124 #ifdef TERM_COUNT
 125         int *new_term_no;
 126 #endif
 127         p->isam_p_size = 2*p->isam_p_size + 100;
 128         new_isam_p_buf = (ISAM_P *) xmalloc(sizeof(*new_isam_p_buf) *
 129                                             p->isam_p_size);
 130         if (p->isam_p_buf)
 131         {
 132             memcpy(new_isam_p_buf, p->isam_p_buf,
 133                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 134             xfree(p->isam_p_buf);
 135         }
 136         p->isam_p_buf = new_isam_p_buf;
 137
 138 #ifdef TERM_COUNT
 139         new_term_no = (int *) xmalloc(sizeof(*new_term_no) * p->isam_p_size);
 140         if (p->term_no)
 141         {
 142             memcpy(new_term_no, p->isam_p_buf,
 143                     p->isam_p_indx * sizeof(*p->term_no));
 144             xfree(p->term_no);
 145         }
 146         p->term_no = new_term_no;
 147 #endif
 148     }
 149     assert(*info == sizeof(*p->isam_p_buf));
 150     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 151
 152     if (p->termset)
 153     {
 154         const char *db;
 155         char term_tmp[IT_MAX_WORD];
 156         int ord = 0;
 157         const char *index_name;
 158         int len = key_SU_decode (&ord, (const unsigned char *) name);
 159
 160         zebra_term_untrans  (p->zh, p->reg_type, term_tmp, name+len);
 161         yaz_log(log_level_rpn, "grep: %d %c %s", ord, name[len], term_tmp);
 162         zebraExplain_lookup_ord(p->zh->reg->zei,
 163                                 ord, 0 /* index_type */, &db, &index_name);
 164         yaz_log(log_level_rpn, "grep:  db=%s index=%s", db, index_name);
 165
 166         resultSetAddTerm(p->zh, p->termset, name[len], db,
 167                          index_name, term_tmp);
 168     }
 169     (p->isam_p_indx)++;
 170 }
 171
 172 static int grep_handle(char *name, const char *info, void *p)
 173 {
 174     add_isam_p(name, info, (struct grep_info *) p);
 175     return 0;
 176 }
 177
 178 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 179                     const char *ct1, const char *ct2, int first)
 180 {
 181     const char *s1, *s0 = *src;
 182     const char **map;
 183
 184     /* skip white space */
 185     while (*s0)
 186     {
 187         if (ct1 && strchr(ct1, *s0))
 188             break;
 189         if (ct2 && strchr(ct2, *s0))
 190             break;
 191         s1 = s0;
 192         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 193         if (**map != *CHR_SPACE)
 194             break;
 195         s0 = s1;
 196     }
 197     *src = s0;
 198     return *s0;
 199 }
 200
 201
 202 static void esc_str(char *out_buf, size_t out_size,
 203                     const char *in_buf, int in_size)
 204 {
 205     int k;
 206
 207     assert(out_buf);
 208     assert(in_buf);
 209     assert(out_size > 20);
 210     *out_buf = '\0';
 211     for (k = 0; k<in_size; k++)
 212     {
 213         int c = in_buf[k] & 0xff;
 214         int pc;
 215         if (c < 32 || c > 126)
 216             pc = '?';
 217         else
 218             pc = c;
 219         sprintf(out_buf +strlen(out_buf), "%02X:%c  ", c, pc);
 220         if (strlen(out_buf) > out_size-20)
 221         {
 222             strcat(out_buf, "..");
 223             break;
 224         }
 225     }
 226 }
 227
 228 #define REGEX_CHARS " []()|.*+?!"
 229
 230 /* term_100: handle term, where trunc = none(no operators at all) */
 231 static int term_100(ZebraMaps zebra_maps, int reg_type,
 232                     const char **src, char *dst, int space_split,
 233                     char *dst_term)
 234 {
 235     const char *s0;
 236     const char **map;
 237     int i = 0;
 238     int j = 0;
 239
 240     const char *space_start = 0;
 241     const char *space_end = 0;
 242
 243     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 244         return 0;
 245     s0 = *src;
 246     while (*s0)
 247     {
 248         const char *s1 = s0;
 249         int q_map_match = 0;
 250         map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 251                                 &q_map_match);
 252         if (space_split)
 253         {
 254             if (**map == *CHR_SPACE)
 255                 break;
 256         }
 257         else  /* complete subfield only. */
 258         {
 259             if (**map == *CHR_SPACE)
 260             {   /* save space mapping for later  .. */
 261                 space_start = s1;
 262                 space_end = s0;
 263                 continue;
 264             }
 265             else if (space_start)
 266             {   /* reload last space */
 267                 while (space_start < space_end)
 268                 {
 269                     if (strchr(REGEX_CHARS, *space_start))
 270                         dst[i++] = '\\';
 271                     dst_term[j++] = *space_start;
 272                     dst[i++] = *space_start++;
 273                 }
 274                 /* and reset */
 275                 space_start = space_end = 0;
 276             }
 277         }
 278         /* add non-space char */
 279         memcpy(dst_term+j, s1, s0 - s1);
 280         j += (s0 - s1);
 281         if (!q_map_match)
 282         {
 283             while (s1 < s0)
 284             {
 285                 if (strchr(REGEX_CHARS, *s1))
 286                     dst[i++] = '\\';
 287                 dst[i++] = *s1++;
 288             }
 289         }
 290         else
 291         {
 292             char tmpbuf[80];
 293             esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 294
 295             strcpy(dst + i, map[0]);
 296             i += strlen(map[0]);
 297         }
 298     }
 299     dst[i] = '\0';
 300     dst_term[j] = '\0';
 301     *src = s0;
 302     return i;
 303 }
 304
 305 /* term_101: handle term, where trunc = Process # */
 306 static int term_101(ZebraMaps zebra_maps, int reg_type,
 307                     const char **src, char *dst, int space_split,
 308                     char *dst_term)
 309 {
 310     const char *s0;
 311     const char **map;
 312     int i = 0;
 313     int j = 0;
 314
 315     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 316         return 0;
 317     s0 = *src;
 318     while (*s0)
 319     {
 320         if (*s0 == '#')
 321         {
 322             dst[i++] = '.';
 323             dst[i++] = '*';
 324             dst_term[j++] = *s0++;
 325         }
 326         else
 327         {
 328             const char *s1 = s0;
 329             int q_map_match = 0;
 330             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 331                                     &q_map_match);
 332             if (space_split && **map == *CHR_SPACE)
 333                 break;
 334
 335             /* add non-space char */
 336             memcpy(dst_term+j, s1, s0 - s1);
 337             j += (s0 - s1);
 338             if (!q_map_match)
 339             {
 340                 while (s1 < s0)
 341                 {
 342                     if (strchr(REGEX_CHARS, *s1))
 343                         dst[i++] = '\\';
 344                     dst[i++] = *s1++;
 345                 }
 346             }
 347             else
 348             {
 349                 char tmpbuf[80];
 350                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 351
 352                 strcpy(dst + i, map[0]);
 353                 i += strlen(map[0]);
 354             }
 355         }
 356     }
 357     dst[i] = '\0';
 358     dst_term[j++] = '\0';
 359     *src = s0;
 360     return i;
 361 }
 362
 363 /* term_103: handle term, where trunc = re-2 (regular expressions) */
 364 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 365                     char *dst, int *errors, int space_split,
 366                     char *dst_term)
 367 {
 368     int i = 0;
 369     int j = 0;
 370     const char *s0;
 371     const char **map;
 372
 373     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 374         return 0;
 375     s0 = *src;
 376     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 377         isdigit(((const unsigned char *)s0)[1]))
 378     {
 379         *errors = s0[1] - '0';
 380         s0 += 3;
 381         if (*errors > 3)
 382             *errors = 3;
 383     }
 384     while (*s0)
 385     {
 386         if (strchr("^\\()[].*+?|-", *s0))
 387         {
 388             dst_term[j++] = *s0;
 389             dst[i++] = *s0++;
 390         }
 391         else
 392         {
 393             const char *s1 = s0;
 394             int q_map_match = 0;
 395             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 396                                     &q_map_match);
 397             if (space_split && **map == *CHR_SPACE)
 398                 break;
 399
 400             /* add non-space char */
 401             memcpy(dst_term+j, s1, s0 - s1);
 402             j += (s0 - s1);
 403             if (!q_map_match)
 404             {
 405                 while (s1 < s0)
 406                 {
 407                     if (strchr(REGEX_CHARS, *s1))
 408                         dst[i++] = '\\';
 409                     dst[i++] = *s1++;
 410                 }
 411             }
 412             else
 413             {
 414                 char tmpbuf[80];
 415                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 416
 417                 strcpy(dst + i, map[0]);
 418                 i += strlen(map[0]);
 419             }
 420         }
 421     }
 422     dst[i] = '\0';
 423     dst_term[j] = '\0';
 424     *src = s0;
 425
 426     return i;
 427 }
 428
 429 /* term_103: handle term, where trunc = re-1 (regular expressions) */
 430 static int term_102(ZebraMaps zebra_maps, int reg_type, const char **src,
 431                     char *dst, int space_split, char *dst_term)
 432 {
 433     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 434                     dst_term);
 435 }
 436
 437
 438 /* term_104: handle term, where trunc = Process # and ! */
 439 static int term_104(ZebraMaps zebra_maps, int reg_type,
 440                     const char **src, char *dst, int space_split,
 441                     char *dst_term)
 442 {
 443     const char *s0;
 444     const char **map;
 445     int i = 0;
 446     int j = 0;
 447
 448     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 449         return 0;
 450     s0 = *src;
 451     while (*s0)
 452     {
 453         if (*s0 == '?')
 454         {
 455             dst_term[j++] = *s0++;
 456             if (*s0 >= '0' && *s0 <= '9')
 457             {
 458                 int limit = 0;
 459                 while (*s0 >= '0' && *s0 <= '9')
 460                 {
 461                     limit = limit * 10 + (*s0 - '0');
 462                     dst_term[j++] = *s0++;
 463                 }
 464                 if (limit > 20)
 465                     limit = 20;
 466                 while (--limit >= 0)
 467                 {
 468                     dst[i++] = '.';
 469                     dst[i++] = '?';
 470                 }
 471             }
 472             else
 473             {
 474                 dst[i++] = '.';
 475                 dst[i++] = '*';
 476             }
 477         }
 478         else if (*s0 == '*')
 479         {
 480             dst[i++] = '.';
 481             dst[i++] = '*';
 482             dst_term[j++] = *s0++;
 483         }
 484         else if (*s0 == '#')
 485         {
 486             dst[i++] = '.';
 487             dst_term[j++] = *s0++;
 488         }
 489         else
 490         {
 491             const char *s1 = s0;
 492             int q_map_match = 0;
 493             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 494                                     &q_map_match);
 495             if (space_split && **map == *CHR_SPACE)
 496                 break;
 497
 498             /* add non-space char */
 499             memcpy(dst_term+j, s1, s0 - s1);
 500             j += (s0 - s1);
 501             if (!q_map_match)
 502             {
 503                 while (s1 < s0)
 504                 {
 505                     if (strchr(REGEX_CHARS, *s1))
 506                         dst[i++] = '\\';
 507                     dst[i++] = *s1++;
 508                 }
 509             }
 510             else
 511             {
 512                 char tmpbuf[80];
 513                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 514
 515                 strcpy(dst + i, map[0]);
 516                 i += strlen(map[0]);
 517             }
 518         }
 519     }
 520     dst[i] = '\0';
 521     dst_term[j++] = '\0';
 522     *src = s0;
 523     return i;
 524 }
 525
 526 /* term_105/106: handle term, where trunc = Process * and ! and right trunc */
 527 static int term_105(ZebraMaps zebra_maps, int reg_type,
 528                     const char **src, char *dst, int space_split,
 529                     char *dst_term, int right_truncate)
 530 {
 531     const char *s0;
 532     const char **map;
 533     int i = 0;
 534     int j = 0;
 535
 536     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 537         return 0;
 538     s0 = *src;
 539     while (*s0)
 540     {
 541         if (*s0 == '*')
 542         {
 543             dst[i++] = '.';
 544             dst[i++] = '*';
 545             dst_term[j++] = *s0++;
 546         }
 547         else if (*s0 == '!')
 548         {
 549             dst[i++] = '.';
 550             dst_term[j++] = *s0++;
 551         }
 552         else
 553         {
 554             const char *s1 = s0;
 555             int q_map_match = 0;
 556             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 557                                     &q_map_match);
 558             if (space_split && **map == *CHR_SPACE)
 559                 break;
 560
 561             /* add non-space char */
 562             memcpy(dst_term+j, s1, s0 - s1);
 563             j += (s0 - s1);
 564             if (!q_map_match)
 565             {
 566                 while (s1 < s0)
 567                 {
 568                     if (strchr(REGEX_CHARS, *s1))
 569                         dst[i++] = '\\';
 570                     dst[i++] = *s1++;
 571                 }
 572             }
 573             else
 574             {
 575                 char tmpbuf[80];
 576                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 577
 578                 strcpy(dst + i, map[0]);
 579                 i += strlen(map[0]);
 580             }
 581         }
 582     }
 583     if (right_truncate)
 584     {
 585         dst[i++] = '.';
 586         dst[i++] = '*';
 587     }
 588     dst[i] = '\0';
 589
 590     dst_term[j++] = '\0';
 591     *src = s0;
 592     return i;
 593 }
 594
 595
 596 /* gen_regular_rel - generate regular expression from relation
 597  *  val:     border value (inclusive)
 598  *  islt:    1 if <=; 0 if >=.
 599  */
 600 static void gen_regular_rel(char *dst, int val, int islt)
 601 {
 602     int dst_p;
 603     int w, d, i;
 604     int pos = 0;
 605     char numstr[20];
 606
 607     yaz_log(YLOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 608     if (val >= 0)
 609     {
 610         if (islt)
 611             strcpy(dst, "(-[0-9]+|(");
 612         else
 613             strcpy(dst, "((");
 614     }
 615     else
 616     {
 617         if (!islt)
 618         {
 619             strcpy(dst, "([0-9]+|-(");
 620             dst_p = strlen(dst);
 621             islt = 1;
 622         }
 623         else
 624         {
 625             strcpy(dst, "(-(");
 626             islt = 0;
 627         }
 628         val = -val;
 629     }
 630     dst_p = strlen(dst);
 631     sprintf(numstr, "%d", val);
 632     for (w = strlen(numstr); --w >= 0; pos++)
 633     {
 634         d = numstr[w];
 635         if (pos > 0)
 636         {
 637             if (islt)
 638             {
 639                 if (d == '0')
 640                     continue;
 641                 d--;
 642             }
 643             else
 644             {
 645                 if (d == '9')
 646                     continue;
 647                 d++;
 648             }
 649         }
 650
 651         strcpy(dst + dst_p, numstr);
 652         dst_p = strlen(dst) - pos - 1;
 653
 654         if (islt)
 655         {
 656             if (d != '0')
 657             {
 658                 dst[dst_p++] = '[';
 659                 dst[dst_p++] = '0';
 660                 dst[dst_p++] = '-';
 661                 dst[dst_p++] = d;
 662                 dst[dst_p++] = ']';
 663             }
 664             else
 665                 dst[dst_p++] = d;
 666         }
 667         else
 668         {
 669             if (d != '9')
 670             {
 671                 dst[dst_p++] = '[';
 672                 dst[dst_p++] = d;
 673                 dst[dst_p++] = '-';
 674                 dst[dst_p++] = '9';
 675                 dst[dst_p++] = ']';
 676             }
 677             else
 678                 dst[dst_p++] = d;
 679         }
 680         for (i = 0; i<pos; i++)
 681         {
 682             dst[dst_p++] = '[';
 683             dst[dst_p++] = '0';
 684             dst[dst_p++] = '-';
 685             dst[dst_p++] = '9';
 686             dst[dst_p++] = ']';
 687         }
 688         dst[dst_p++] = '|';
 689     }
 690     dst[dst_p] = '\0';
 691     if (islt)
 692     {
 693         /* match everything less than 10^(pos-1) */
 694         strcat(dst, "0*");
 695         for (i = 1; i<pos; i++)
 696             strcat(dst, "[0-9]?");
 697     }
 698     else
 699     {
 700         /* match everything greater than 10^pos */
 701         for (i = 0; i <= pos; i++)
 702             strcat(dst, "[0-9]");
 703         strcat(dst, "[0-9]*");
 704     }
 705     strcat(dst, "))");
 706 }
 707
 708 void string_rel_add_char(char **term_p, const char *src, int *indx)
 709 {
 710     if (src[*indx] == '\\')
 711         *(*term_p)++ = src[(*indx)++];
 712     *(*term_p)++ = src[(*indx)++];
 713 }
 714
 715 /*
 716  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 717  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 718  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 719  *              ([^-a].*|a[^-b].*|ab[c-].*)
 720  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 721  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 722  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 723  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 724  */
 725 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 726                            const char **term_sub, char *term_dict,
 727                            oid_value attributeSet,
 728                            int reg_type, int space_split, char *term_dst,
 729                            int *error_code)
 730 {
 731     AttrType relation;
 732     int relation_value;
 733     int i;
 734     char *term_tmp = term_dict + strlen(term_dict);
 735     char term_component[2*IT_MAX_WORD+20];
 736
 737     attr_init_APT(&relation, zapt, 2);
 738     relation_value = attr_find(&relation, NULL);
 739
 740     *error_code = 0;
 741     yaz_log(YLOG_DEBUG, "string relation value=%d", relation_value);
 742     switch (relation_value)
 743     {
 744     case 1:
 745         if (!term_100(zh->reg->zebra_maps, reg_type,
 746                       term_sub, term_component,
 747                       space_split, term_dst))
 748             return 0;
 749         yaz_log(log_level_rpn, "Relation <");
 750
 751         *term_tmp++ = '(';
 752         for (i = 0; term_component[i]; )
 753         {
 754             int j = 0;
 755
 756             if (i)
 757                 *term_tmp++ = '|';
 758             while (j < i)
 759                 string_rel_add_char(&term_tmp, term_component, &j);
 760
 761             *term_tmp++ = '[';
 762
 763             *term_tmp++ = '^';
 764             string_rel_add_char(&term_tmp, term_component, &i);
 765             *term_tmp++ = '-';
 766
 767             *term_tmp++ = ']';
 768             *term_tmp++ = '.';
 769             *term_tmp++ = '*';
 770
 771             if ((term_tmp - term_dict) > IT_MAX_WORD)
 772                 break;
 773         }
 774         *term_tmp++ = ')';
 775         *term_tmp = '\0';
 776         break;
 777     case 2:
 778         if (!term_100(zh->reg->zebra_maps, reg_type,
 779                       term_sub, term_component,
 780                       space_split, term_dst))
 781             return 0;
 782         yaz_log(log_level_rpn, "Relation <=");
 783
 784         *term_tmp++ = '(';
 785         for (i = 0; term_component[i]; )
 786         {
 787             int j = 0;
 788
 789             while (j < i)
 790                 string_rel_add_char(&term_tmp, term_component, &j);
 791             *term_tmp++ = '[';
 792
 793             *term_tmp++ = '^';
 794             string_rel_add_char(&term_tmp, term_component, &i);
 795             *term_tmp++ = '-';
 796
 797             *term_tmp++ = ']';
 798             *term_tmp++ = '.';
 799             *term_tmp++ = '*';
 800
 801             *term_tmp++ = '|';
 802
 803             if ((term_tmp - term_dict) > IT_MAX_WORD)
 804                 break;
 805         }
 806         for (i = 0; term_component[i]; )
 807             string_rel_add_char(&term_tmp, term_component, &i);
 808         *term_tmp++ = ')';
 809         *term_tmp = '\0';
 810         break;
 811     case 5:
 812         if (!term_100 (zh->reg->zebra_maps, reg_type,
 813                        term_sub, term_component, space_split, term_dst))
 814             return 0;
 815         yaz_log(log_level_rpn, "Relation >");
 816
 817         *term_tmp++ = '(';
 818         for (i = 0; term_component[i];)
 819         {
 820             int j = 0;
 821
 822             while (j < i)
 823                 string_rel_add_char(&term_tmp, term_component, &j);
 824             *term_tmp++ = '[';
 825
 826             *term_tmp++ = '^';
 827             *term_tmp++ = '-';
 828             string_rel_add_char(&term_tmp, term_component, &i);
 829
 830             *term_tmp++ = ']';
 831             *term_tmp++ = '.';
 832             *term_tmp++ = '*';
 833
 834             *term_tmp++ = '|';
 835
 836             if ((term_tmp - term_dict) > IT_MAX_WORD)
 837                 break;
 838         }
 839         for (i = 0; term_component[i];)
 840             string_rel_add_char(&term_tmp, term_component, &i);
 841         *term_tmp++ = '.';
 842         *term_tmp++ = '+';
 843         *term_tmp++ = ')';
 844         *term_tmp = '\0';
 845         break;
 846     case 4:
 847         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 848                       term_component, space_split, term_dst))
 849             return 0;
 850         yaz_log(log_level_rpn, "Relation >=");
 851
 852         *term_tmp++ = '(';
 853         for (i = 0; term_component[i];)
 854         {
 855             int j = 0;
 856
 857             if (i)
 858                 *term_tmp++ = '|';
 859             while (j < i)
 860                 string_rel_add_char(&term_tmp, term_component, &j);
 861             *term_tmp++ = '[';
 862
 863             if (term_component[i+1])
 864             {
 865                 *term_tmp++ = '^';
 866                 *term_tmp++ = '-';
 867                 string_rel_add_char(&term_tmp, term_component, &i);
 868             }
 869             else
 870             {
 871                 string_rel_add_char(&term_tmp, term_component, &i);
 872                 *term_tmp++ = '-';
 873             }
 874             *term_tmp++ = ']';
 875             *term_tmp++ = '.';
 876             *term_tmp++ = '*';
 877
 878             if ((term_tmp - term_dict) > IT_MAX_WORD)
 879                 break;
 880         }
 881         *term_tmp++ = ')';
 882         *term_tmp = '\0';
 883         break;
 884     case 3:
 885     case 102:
 886     case -1:
 887         if (!**term_sub)
 888             return 1;
 889         yaz_log(log_level_rpn, "Relation =");
 890         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 891                       term_component, space_split, term_dst))
 892             return 0;
 893         strcat(term_tmp, "(");
 894         strcat(term_tmp, term_component);
 895         strcat(term_tmp, ")");
 896         break;
 897     case 103:
 898         yaz_log(log_level_rpn, "Relation always matches");
 899         /* skip to end of term (we don't care what it is) */
 900         while (**term_sub != '\0')
 901             (*term_sub)++;
 902         break;
 903     default:
 904         *error_code = YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
 905         return 0;
 906     }
 907     return 1;
 908 }
 909
 910 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 911                              const char **term_sub,
 912                              oid_value attributeSet, NMEM stream,
 913                              struct grep_info *grep_info,
 914                              int reg_type, int complete_flag,
 915                              int num_bases, char **basenames,
 916                              char *term_dst,
 917                              const char *xpath_use,
 918                              struct ord_list **ol);
 919
 920 static ZEBRA_RES term_limits_APT(ZebraHandle zh,
 921                                  Z_AttributesPlusTerm *zapt,
 922                                  zint *hits_limit_value,
 923                                  const char **term_ref_id_str,
 924                                  NMEM nmem)
 925 {
 926     AttrType term_ref_id_attr;
 927     AttrType hits_limit_attr;
 928     int term_ref_id_int;
 929
 930     attr_init_APT(&hits_limit_attr, zapt, 11);
 931     *hits_limit_value  = attr_find(&hits_limit_attr, NULL);
 932
 933     attr_init_APT(&term_ref_id_attr, zapt, 10);
 934     term_ref_id_int = attr_find_ex(&term_ref_id_attr, NULL, term_ref_id_str);
 935     if (term_ref_id_int >= 0)
 936     {
 937         char *res = nmem_malloc(nmem, 20);
 938         sprintf(res, "%d", term_ref_id_int);
 939         *term_ref_id_str = res;
 940     }
 941
 942     /* no limit given ? */
 943     if (*hits_limit_value == -1)
 944     {
 945         if (*term_ref_id_str)
 946         {
 947             /* use global if term_ref is present */
 948             *hits_limit_value = zh->approx_limit;
 949         }
 950         else
 951         {
 952             /* no counting if term_ref is not present */
 953             *hits_limit_value = 0;
 954         }
 955     }
 956     else if (*hits_limit_value == 0)
 957     {
 958         /* 0 is the same as global limit */
 959         *hits_limit_value = zh->approx_limit;
 960     }
 961     yaz_log(YLOG_DEBUG, "term_limits_APT ref_id=%s limit=" ZINT_FORMAT,
 962             *term_ref_id_str ? *term_ref_id_str : "none",
 963             *hits_limit_value);
 964     return ZEBRA_OK;
 965 }
 966
 967 static ZEBRA_RES term_trunc(ZebraHandle zh,
 968                             Z_AttributesPlusTerm *zapt,
 969                             const char **term_sub,
 970                             oid_value attributeSet, NMEM stream,
 971                             struct grep_info *grep_info,
 972                             int reg_type, int complete_flag,
 973                             int num_bases, char **basenames,
 974                             char *term_dst,
 975                             const char *rank_type,
 976                             const char *xpath_use,
 977                             NMEM rset_nmem,
 978                             RSET *rset,
 979                             struct rset_key_control *kc)
 980 {
 981     ZEBRA_RES res;
 982     struct ord_list *ol;
 983     zint hits_limit_value;
 984     const char *term_ref_id_str = 0;
 985     *rset = 0;
 986
 987     term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str, stream);
 988     grep_info->isam_p_indx = 0;
 989     res = string_term(zh, zapt, term_sub, attributeSet, stream, grep_info,
 990                       reg_type, complete_flag, num_bases, basenames,
 991                       term_dst, xpath_use, &ol);
 992     if (res != ZEBRA_OK)
 993         return res;
 994     if (!*term_sub)  /* no more terms ? */
 995         return res;
 996     yaz_log(log_level_rpn, "term: %s", term_dst);
 997     *rset = rset_trunc(zh, grep_info->isam_p_buf,
 998                        grep_info->isam_p_indx, term_dst,
 999                        strlen(term_dst), rank_type, 1 /* preserve pos */,
1000                        zapt->term->which, rset_nmem,
1001                        kc, kc->scope, ol, reg_type, hits_limit_value,
1002                        term_ref_id_str);
1003     if (!*rset)
1004         return ZEBRA_FAIL;
1005     return ZEBRA_OK;
1006 }
1007
1008 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1009                              const char **term_sub,
1010                              oid_value attributeSet, NMEM stream,
1011                              struct grep_info *grep_info,
1012                              int reg_type, int complete_flag,
1013                              int num_bases, char **basenames,
1014                              char *term_dst,
1015                              const char *xpath_use,
1016                              struct ord_list **ol)
1017 {
1018     char term_dict[2*IT_MAX_WORD+4000];
1019     int j, r, base_no;
1020     AttrType truncation;
1021     int truncation_value;
1022     const char *termp;
1023     struct rpn_char_map_info rcmi;
1024     int space_split = complete_flag ? 0 : 1;
1025
1026     int bases_ok = 0;     /* no of databases with OK attribute */
1027
1028     *ol = ord_list_create(stream);
1029
1030     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1031     attr_init_APT(&truncation, zapt, 5);
1032     truncation_value = attr_find(&truncation, NULL);
1033     yaz_log(log_level_rpn, "truncation value %d", truncation_value);
1034
1035     for (base_no = 0; base_no < num_bases; base_no++)
1036     {
1037         int ord = -1;
1038         int regex_range = 0;
1039         int max_pos, prefix_len = 0;
1040         int relation_error;
1041         char ord_buf[32];
1042         int ord_len, i;
1043
1044         termp = *term_sub; /* start of term for each database */
1045
1046         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1047         {
1048             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1049                            basenames[base_no]);
1050             return ZEBRA_FAIL;
1051         }
1052
1053         if (zebra_apt_get_ord(zh, zapt, reg_type, xpath_use,
1054                               attributeSet, &ord) != ZEBRA_OK)
1055             continue;
1056
1057         bases_ok++;
1058
1059         *ol = ord_list_append(stream, *ol, ord);
1060         ord_len = key_SU_encode (ord, ord_buf);
1061
1062         term_dict[prefix_len++] = '(';
1063         for (i = 0; i<ord_len; i++)
1064         {
1065             term_dict[prefix_len++] = 1;  /* our internal regexp escape char */
1066             term_dict[prefix_len++] = ord_buf[i];
1067         }
1068         term_dict[prefix_len++] = ')';
1069         term_dict[prefix_len] = '\0';
1070         j = prefix_len;
1071         switch (truncation_value)
1072         {
1073         case -1:         /* not specified */
1074         case 100:        /* do not truncate */
1075             if (!string_relation(zh, zapt, &termp, term_dict,
1076                                  attributeSet,
1077                                  reg_type, space_split, term_dst,
1078                                  &relation_error))
1079             {
1080                 if (relation_error)
1081                 {
1082                     zebra_setError(zh, relation_error, 0);
1083                     return ZEBRA_FAIL;
1084                 }
1085                 *term_sub = 0;
1086                 return ZEBRA_OK;
1087             }
1088             break;
1089         case 1:          /* right truncation */
1090             term_dict[j++] = '(';
1091             if (!term_100(zh->reg->zebra_maps, reg_type,
1092                           &termp, term_dict + j, space_split, term_dst))
1093             {
1094                 *term_sub = 0;
1095                 return ZEBRA_OK;
1096             }
1097             strcat(term_dict, ".*)");
1098             break;
1099         case 2:          /* keft truncation */
1100             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1101             if (!term_100(zh->reg->zebra_maps, reg_type,
1102                           &termp, term_dict + j, space_split, term_dst))
1103             {
1104                 *term_sub = 0;
1105                 return ZEBRA_OK;
1106             }
1107             strcat(term_dict, ")");
1108             break;
1109         case 3:          /* left&right truncation */
1110             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1111             if (!term_100(zh->reg->zebra_maps, reg_type,
1112                           &termp, term_dict + j, space_split, term_dst))
1113             {
1114                 *term_sub = 0;
1115                 return ZEBRA_OK;
1116             }
1117             strcat(term_dict, ".*)");
1118             break;
1119         case 101:        /* process # in term */
1120             term_dict[j++] = '(';
1121             if (!term_101(zh->reg->zebra_maps, reg_type,
1122                           &termp, term_dict + j, space_split, term_dst))
1123             {
1124                 *term_sub = 0;
1125                 return ZEBRA_OK;
1126             }
1127             strcat(term_dict, ")");
1128             break;
1129         case 102:        /* Regexp-1 */
1130             term_dict[j++] = '(';
1131             if (!term_102(zh->reg->zebra_maps, reg_type,
1132                           &termp, term_dict + j, space_split, term_dst))
1133             {
1134                 *term_sub = 0;
1135                 return ZEBRA_OK;
1136             }
1137             strcat(term_dict, ")");
1138             break;
1139         case 103:       /* Regexp-2 */
1140             regex_range = 1;
1141             term_dict[j++] = '(';
1142             if (!term_103(zh->reg->zebra_maps, reg_type,
1143                           &termp, term_dict + j, &regex_range,
1144                           space_split, term_dst))
1145             {
1146                 *term_sub = 0;
1147                 return ZEBRA_OK;
1148             }
1149             strcat(term_dict, ")");
1150             break;
1151         case 104:        /* process # and ! in term */
1152             term_dict[j++] = '(';
1153             if (!term_104(zh->reg->zebra_maps, reg_type,
1154                           &termp, term_dict + j, space_split, term_dst))
1155             {
1156                 *term_sub = 0;
1157                 return ZEBRA_OK;
1158             }
1159             strcat(term_dict, ")");
1160             break;
1161         case 105:        /* process * and ! in term */
1162             term_dict[j++] = '(';
1163             if (!term_105(zh->reg->zebra_maps, reg_type,
1164                           &termp, term_dict + j, space_split, term_dst, 1))
1165             {
1166                 *term_sub = 0;
1167                 return ZEBRA_OK;
1168             }
1169             strcat(term_dict, ")");
1170             break;
1171         case 106:        /* process * and ! in term */
1172             term_dict[j++] = '(';
1173             if (!term_105(zh->reg->zebra_maps, reg_type,
1174                           &termp, term_dict + j, space_split, term_dst, 0))
1175             {
1176                 *term_sub = 0;
1177                 return ZEBRA_OK;
1178             }
1179             strcat(term_dict, ")");
1180             break;
1181         default:
1182             zebra_setError_zint(zh,
1183                                 YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE,
1184                                 truncation_value);
1185             return ZEBRA_FAIL;
1186         }
1187         if (1)
1188         {
1189             char buf[80];
1190             const char *input = term_dict + prefix_len;
1191             esc_str(buf, sizeof(buf), input, strlen(input));
1192         }
1193         yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_dict+prefix_len);
1194         r = dict_lookup_grep(zh->reg->dict, term_dict, regex_range,
1195                              grep_info, &max_pos,
1196                              ord_len /* number of "exact" chars */,
1197                              grep_handle);
1198         if (r)
1199             yaz_log(YLOG_WARN, "dict_lookup_grep fail %d", r);
1200     }
1201     if (!bases_ok)
1202         return ZEBRA_FAIL;
1203     *term_sub = termp;
1204     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1205     return ZEBRA_OK;
1206 }
1207
1208
1209 /* convert APT search term to UTF8 */
1210 static ZEBRA_RES zapt_term_to_utf8(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1211                                    char *termz)
1212 {
1213     size_t sizez;
1214     Z_Term *term = zapt->term;
1215
1216     switch (term->which)
1217     {
1218     case Z_Term_general:
1219         if (zh->iconv_to_utf8 != 0)
1220         {
1221             char *inbuf = (char *) term->u.general->buf;
1222             size_t inleft = term->u.general->len;
1223             char *outbuf = termz;
1224             size_t outleft = IT_MAX_WORD-1;
1225             size_t ret;
1226
1227             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1228                         &outbuf, &outleft);
1229             if (ret == (size_t)(-1))
1230             {
1231                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1232                 zebra_setError(
1233                     zh,
1234                     YAZ_BIB1_QUERY_TERM_INCLUDES_CHARS_THAT_DO_NOT_TRANSLATE_INTO_,
1235                     0);
1236                 return ZEBRA_FAIL;
1237             }
1238             *outbuf = 0;
1239         }
1240         else
1241         {
1242             sizez = term->u.general->len;
1243             if (sizez > IT_MAX_WORD-1)
1244                 sizez = IT_MAX_WORD-1;
1245             memcpy (termz, term->u.general->buf, sizez);
1246             termz[sizez] = '\0';
1247         }
1248         break;
1249     case Z_Term_characterString:
1250         sizez = strlen(term->u.characterString);
1251         if (sizez > IT_MAX_WORD-1)
1252             sizez = IT_MAX_WORD-1;
1253         memcpy (termz, term->u.characterString, sizez);
1254         termz[sizez] = '\0';
1255         break;
1256     default:
1257         zebra_setError(zh, YAZ_BIB1_UNSUPP_CODED_VALUE_FOR_TERM, 0);
1258         return ZEBRA_FAIL;
1259     }
1260     return ZEBRA_OK;
1261 }
1262
1263 /* convert APT SCAN term to internal cmap */
1264 static ZEBRA_RES trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1265                                  char *termz, int reg_type)
1266 {
1267     char termz0[IT_MAX_WORD];
1268
1269     if (zapt_term_to_utf8(zh, zapt, termz0) == ZEBRA_FAIL)
1270         return ZEBRA_FAIL;    /* error */
1271     else
1272     {
1273         const char **map;
1274         const char *cp = (const char *) termz0;
1275         const char *cp_end = cp + strlen(cp);
1276         const char *src;
1277         int i = 0;
1278         const char *space_map = NULL;
1279         int len;
1280
1281         while ((len = (cp_end - cp)) > 0)
1282         {
1283             map = zebra_maps_input(zh->reg->zebra_maps, reg_type, &cp, len, 0);
1284             if (**map == *CHR_SPACE)
1285                 space_map = *map;
1286             else
1287             {
1288                 if (i && space_map)
1289                     for (src = space_map; *src; src++)
1290                         termz[i++] = *src;
1291                 space_map = NULL;
1292                 for (src = *map; *src; src++)
1293                     termz[i++] = *src;
1294             }
1295         }
1296         termz[i] = '\0';
1297     }
1298     return ZEBRA_OK;
1299 }
1300
1301 static void grep_info_delete(struct grep_info *grep_info)
1302 {
1303 #ifdef TERM_COUNT
1304     xfree(grep_info->term_no);
1305 #endif
1306     xfree(grep_info->isam_p_buf);
1307 }
1308
1309 static ZEBRA_RES grep_info_prepare(ZebraHandle zh,
1310                                    Z_AttributesPlusTerm *zapt,
1311                                    struct grep_info *grep_info,
1312                                    int reg_type)
1313 {
1314     AttrType termset;
1315     int termset_value_numeric;
1316     const char *termset_value_string;
1317
1318 #ifdef TERM_COUNT
1319     grep_info->term_no = 0;
1320 #endif
1321     grep_info->isam_p_size = 0;
1322     grep_info->isam_p_buf = NULL;
1323     grep_info->zh = zh;
1324     grep_info->reg_type = reg_type;
1325     grep_info->termset = 0;
1326     if (!zapt)
1327         return ZEBRA_OK;
1328     attr_init_APT(&termset, zapt, 8);
1329     termset_value_numeric =
1330         attr_find_ex(&termset, NULL, &termset_value_string);
1331     if (termset_value_numeric != -1)
1332     {
1333 #if TERMSET_DISABLE
1334         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, "termset");
1335         return ZEBRA_FAIL;
1336 #else
1337         char resname[32];
1338         const char *termset_name = 0;
1339         if (termset_value_numeric != -2)
1340         {
1341
1342             sprintf(resname, "%d", termset_value_numeric);
1343             termset_name = resname;
1344         }
1345         else
1346             termset_name = termset_value_string;
1347         yaz_log(log_level_rpn, "creating termset set %s", termset_name);
1348         grep_info->termset = resultSetAdd(zh, termset_name, 1);
1349         if (!grep_info->termset)
1350         {
1351             zebra_setError(zh, YAZ_BIB1_ILLEGAL_RESULT_SET_NAME, termset_name);
1352             return ZEBRA_FAIL;
1353         }
1354 #endif
1355     }
1356     return ZEBRA_OK;
1357 }
1358
1359 /**
1360   \brief Create result set(s) for list of terms
1361   \param zh Zebra Handle
1362   \param zapt Attributes Plust Term (RPN leaf)
1363   \param termz term as used in query but converted to UTF-8
1364   \param attributeSet default attribute set
1365   \param stream memory for result
1366   \param reg_type register type ('w', 'p',..)
1367   \param complete_flag whether it's phrases or not
1368   \param rank_type term flags for ranking
1369   \param xpath_use use attribute for X-Path (-1 for no X-path)
1370   \param num_bases number of databases
1371   \param basenames array of databases
1372   \param rset_nmem memory for result sets
1373   \param result_sets output result set for each term in list (output)
1374   \param num_result_sets number of output result sets
1375   \param kc rset key control to be used for created result sets
1376 */
1377 static ZEBRA_RES term_list_trunc(ZebraHandle zh,
1378                                  Z_AttributesPlusTerm *zapt,
1379                                  const char *termz,
1380                                  oid_value attributeSet,
1381                                  NMEM stream,
1382                                  int reg_type, int complete_flag,
1383                                  const char *rank_type,
1384                                  const char *xpath_use,
1385                                  int num_bases, char **basenames,
1386                                  NMEM rset_nmem,
1387                                  RSET **result_sets, int *num_result_sets,
1388                                  struct rset_key_control *kc)
1389 {
1390     char term_dst[IT_MAX_WORD+1];
1391     struct grep_info grep_info;
1392     const char *termp = termz;
1393     int alloc_sets = 0;
1394
1395     *num_result_sets = 0;
1396     *term_dst = 0;
1397     if (grep_info_prepare(zh, zapt, &grep_info, reg_type) == ZEBRA_FAIL)
1398         return ZEBRA_FAIL;
1399     while(1)
1400     {
1401         ZEBRA_RES res;
1402
1403         if (alloc_sets == *num_result_sets)
1404         {
1405             int add = 10;
1406             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1407                                               sizeof(*rnew));
1408             if (alloc_sets)
1409                 memcpy(rnew, *result_sets, alloc_sets * sizeof(*rnew));
1410             alloc_sets = alloc_sets + add;
1411             *result_sets = rnew;
1412         }
1413         res = term_trunc(zh, zapt, &termp, attributeSet,
1414                          stream, &grep_info,
1415                          reg_type, complete_flag,
1416                          num_bases, basenames,
1417                          term_dst, rank_type,
1418                          xpath_use, rset_nmem,
1419                          &(*result_sets)[*num_result_sets],
1420                          kc);
1421         if (res != ZEBRA_OK)
1422         {
1423             int i;
1424             for (i = 0; i < *num_result_sets; i++)
1425                 rset_delete((*result_sets)[i]);
1426             grep_info_delete (&grep_info);
1427             return res;
1428         }
1429         if ((*result_sets)[*num_result_sets] == 0)
1430             break;
1431         (*num_result_sets)++;
1432
1433         if (!*termp)
1434             break;
1435     }
1436     grep_info_delete(&grep_info);
1437     return ZEBRA_OK;
1438 }
1439
1440 static ZEBRA_RES rpn_search_APT_position(ZebraHandle zh,
1441                                          Z_AttributesPlusTerm *zapt,
1442                                          oid_value attributeSet,
1443                                          int reg_type,
1444                                          int num_bases, char **basenames,
1445                                          NMEM rset_nmem,
1446                                          RSET *rset,
1447                                          struct rset_key_control *kc)
1448 {
1449     RSET *f_set;
1450     int base_no;
1451     int position_value;
1452     int num_sets = 0;
1453     AttrType position;
1454
1455     attr_init_APT(&position, zapt, 3);
1456     position_value = attr_find(&position, NULL);
1457     switch(position_value)
1458     {
1459     case 3:
1460     case -1:
1461         return ZEBRA_OK;
1462     case 1:
1463     case 2:
1464         break;
1465     default:
1466         zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_POSITION_ATTRIBUTE,
1467                             position_value);
1468         return ZEBRA_FAIL;
1469     }
1470
1471     if (!zebra_maps_is_first_in_field(zh->reg->zebra_maps, reg_type))
1472     {
1473         zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_POSITION_ATTRIBUTE,
1474                             position_value);
1475         return ZEBRA_FAIL;
1476     }
1477
1478     if (!zh->reg->isamb && !zh->reg->isamc)
1479     {
1480         zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_POSITION_ATTRIBUTE,
1481                             position_value);
1482         return ZEBRA_FAIL;
1483     }
1484     f_set = xmalloc(sizeof(RSET) * num_bases);
1485     for (base_no = 0; base_no < num_bases; base_no++)
1486     {
1487         int ord = -1;
1488         char ord_buf[32];
1489         char term_dict[100];
1490         int ord_len;
1491         char *val;
1492         ISAM_P isam_p;
1493
1494         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1495         {
1496             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1497                            basenames[base_no]);
1498             return ZEBRA_FAIL;
1499         }
1500
1501         if (zebra_apt_get_ord(zh, zapt, reg_type, 0,
1502                               attributeSet, &ord) != ZEBRA_OK)
1503             continue;
1504
1505         ord_len = key_SU_encode (ord, ord_buf);
1506         memcpy(term_dict, ord_buf, ord_len);
1507         strcpy(term_dict+ord_len, FIRST_IN_FIELD_STR);
1508         val = dict_lookup(zh->reg->dict, term_dict);
1509         if (!val)
1510             continue;
1511         assert(*val == sizeof(ISAM_P));
1512         memcpy(&isam_p, val+1, sizeof(isam_p));
1513
1514
1515         if (zh->reg->isamb)
1516             f_set[num_sets++] = rsisamb_create(rset_nmem, kc, kc->scope,
1517                                                zh->reg->isamb, isam_p, 0);
1518         else if (zh->reg->isamc)
1519             f_set[num_sets++] = rsisamc_create(rset_nmem, kc, kc->scope,
1520                                                zh->reg->isamc, isam_p, 0);
1521     }
1522     if (num_sets)
1523     {
1524         *rset = rset_create_or(rset_nmem, kc, kc->scope,
1525                                0 /* termid */, num_sets, f_set);
1526     }
1527     xfree(f_set);
1528     return ZEBRA_OK;
1529 }
1530
1531 static ZEBRA_RES rpn_search_APT_phrase(ZebraHandle zh,
1532                                        Z_AttributesPlusTerm *zapt,
1533                                        const char *termz_org,
1534                                        oid_value attributeSet,
1535                                        NMEM stream,
1536                                        int reg_type, int complete_flag,
1537                                        const char *rank_type,
1538                                        const char *xpath_use,
1539                                        int num_bases, char **basenames,
1540                                        NMEM rset_nmem,
1541                                        RSET *rset,
1542                                        struct rset_key_control *kc)
1543 {
1544     RSET *result_sets = 0;
1545     int num_result_sets = 0;
1546     ZEBRA_RES res =
1547         term_list_trunc(zh, zapt, termz_org, attributeSet,
1548                         stream, reg_type, complete_flag,
1549                         rank_type, xpath_use,
1550                         num_bases, basenames,
1551                         rset_nmem,
1552                         &result_sets, &num_result_sets, kc);
1553
1554     if (res != ZEBRA_OK)
1555         return res;
1556
1557     if (num_result_sets > 0)
1558     {
1559         RSET first_set = 0;
1560         res = rpn_search_APT_position(zh, zapt, attributeSet,
1561                                       reg_type,
1562                                       num_bases, basenames,
1563                                       rset_nmem, &first_set,
1564                                       kc);
1565         if (res != ZEBRA_OK)
1566             return res;
1567         if (first_set)
1568         {
1569             RSET *nsets = nmem_malloc(stream,
1570                                       sizeof(RSET) * (num_result_sets+1));
1571             nsets[0] = first_set;
1572             memcpy(nsets+1, result_sets, sizeof(RSET) * num_result_sets);
1573             result_sets = nsets;
1574             num_result_sets++;
1575         }
1576     }
1577     if (num_result_sets == 0)
1578         *rset = rset_create_null(rset_nmem, kc, 0);
1579     else if (num_result_sets == 1)
1580         *rset = result_sets[0];
1581     else
1582         *rset = rset_create_prox(rset_nmem, kc, kc->scope,
1583                                  num_result_sets, result_sets,
1584                                  1 /* ordered */, 0 /* exclusion */,
1585                                  3 /* relation */, 1 /* distance */);
1586     if (!*rset)
1587         return ZEBRA_FAIL;
1588     return ZEBRA_OK;
1589 }
1590
1591 static ZEBRA_RES rpn_search_APT_or_list(ZebraHandle zh,
1592                                         Z_AttributesPlusTerm *zapt,
1593                                         const char *termz_org,
1594                                         oid_value attributeSet,
1595                                         NMEM stream,
1596                                         int reg_type, int complete_flag,
1597                                         const char *rank_type,
1598                                         const char *xpath_use,
1599                                         int num_bases, char **basenames,
1600                                         NMEM rset_nmem,
1601                                         RSET *rset,
1602                                         struct rset_key_control *kc)
1603 {
1604     RSET *result_sets = 0;
1605     int num_result_sets = 0;
1606     int i;
1607     ZEBRA_RES res =
1608         term_list_trunc(zh, zapt, termz_org, attributeSet,
1609                         stream, reg_type, complete_flag,
1610                         rank_type, xpath_use,
1611                         num_bases, basenames,
1612                         rset_nmem,
1613                         &result_sets, &num_result_sets, kc);
1614     if (res != ZEBRA_OK)
1615         return res;
1616
1617     for (i = 0; i<num_result_sets; i++)
1618     {
1619         RSET first_set = 0;
1620         res = rpn_search_APT_position(zh, zapt, attributeSet,
1621                                       reg_type,
1622                                       num_bases, basenames,
1623                                       rset_nmem, &first_set,
1624                                       kc);
1625         if (res != ZEBRA_OK)
1626         {
1627             for (i = 0; i<num_result_sets; i++)
1628                 rset_delete(result_sets[i]);
1629             return res;
1630         }
1631
1632         if (first_set)
1633         {
1634             RSET tmp_set[2];
1635
1636             tmp_set[0] = first_set;
1637             tmp_set[1] = result_sets[i];
1638
1639             result_sets[i] = rset_create_prox(
1640                 rset_nmem, kc, kc->scope,
1641                 2, tmp_set,
1642                 1 /* ordered */, 0 /* exclusion */,
1643                 3 /* relation */, 1 /* distance */);
1644         }
1645     }
1646     if (num_result_sets == 0)
1647         *rset = rset_create_null(rset_nmem, kc, 0);
1648     else if (num_result_sets == 1)
1649         *rset = result_sets[0];
1650     else
1651         *rset = rset_create_or(rset_nmem, kc, kc->scope, 0 /* termid */,
1652                                num_result_sets, result_sets);
1653     if (!*rset)
1654         return ZEBRA_FAIL;
1655     return ZEBRA_OK;
1656 }
1657
1658 static ZEBRA_RES rpn_search_APT_and_list(ZebraHandle zh,
1659                                          Z_AttributesPlusTerm *zapt,
1660                                          const char *termz_org,
1661                                          oid_value attributeSet,
1662                                          NMEM stream,
1663                                          int reg_type, int complete_flag,
1664                                          const char *rank_type,
1665                                          const char *xpath_use,
1666                                          int num_bases, char **basenames,
1667                                          NMEM rset_nmem,
1668                                          RSET *rset,
1669                                          struct rset_key_control *kc)
1670 {
1671     RSET *result_sets = 0;
1672     int num_result_sets = 0;
1673     int i;
1674     ZEBRA_RES res =
1675         term_list_trunc(zh, zapt, termz_org, attributeSet,
1676                         stream, reg_type, complete_flag,
1677                         rank_type, xpath_use,
1678                         num_bases, basenames,
1679                         rset_nmem,
1680                         &result_sets, &num_result_sets,
1681                         kc);
1682     if (res != ZEBRA_OK)
1683         return res;
1684     for (i = 0; i<num_result_sets; i++)
1685     {
1686         RSET first_set = 0;
1687         res = rpn_search_APT_position(zh, zapt, attributeSet,
1688                                       reg_type,
1689                                       num_bases, basenames,
1690                                       rset_nmem, &first_set,
1691                                       kc);
1692         if (res != ZEBRA_OK)
1693         {
1694             for (i = 0; i<num_result_sets; i++)
1695                 rset_delete(result_sets[i]);
1696             return res;
1697         }
1698
1699         if (first_set)
1700         {
1701             RSET tmp_set[2];
1702
1703             tmp_set[0] = first_set;
1704             tmp_set[1] = result_sets[i];
1705
1706             result_sets[i] = rset_create_prox(
1707                 rset_nmem, kc, kc->scope,
1708                 2, tmp_set,
1709                 1 /* ordered */, 0 /* exclusion */,
1710                 3 /* relation */, 1 /* distance */);
1711         }
1712     }
1713
1714
1715     if (num_result_sets == 0)
1716         *rset = rset_create_null(rset_nmem, kc, 0);
1717     else if (num_result_sets == 1)
1718         *rset = result_sets[0];
1719     else
1720         *rset = rset_create_and(rset_nmem, kc, kc->scope,
1721                                 num_result_sets, result_sets);
1722     if (!*rset)
1723         return ZEBRA_FAIL;
1724     return ZEBRA_OK;
1725 }
1726
1727 static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1728                             const char **term_sub,
1729                             char *term_dict,
1730                             oid_value attributeSet,
1731                             struct grep_info *grep_info,
1732                             int *max_pos,
1733                             int reg_type,
1734                             char *term_dst,
1735                             int *error_code)
1736 {
1737     AttrType relation;
1738     int relation_value;
1739     int term_value;
1740     int r;
1741     char *term_tmp = term_dict + strlen(term_dict);
1742
1743     *error_code = 0;
1744     attr_init_APT(&relation, zapt, 2);
1745     relation_value = attr_find(&relation, NULL);
1746
1747     yaz_log(log_level_rpn, "numeric relation value=%d", relation_value);
1748
1749     switch (relation_value)
1750     {
1751     case 1:
1752         yaz_log(log_level_rpn, "Relation <");
1753         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1754                       term_dst))
1755             return 0;
1756         term_value = atoi (term_tmp);
1757         gen_regular_rel(term_tmp, term_value-1, 1);
1758         break;
1759     case 2:
1760         yaz_log(log_level_rpn, "Relation <=");
1761         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1762                       term_dst))
1763             return 0;
1764         term_value = atoi (term_tmp);
1765         gen_regular_rel(term_tmp, term_value, 1);
1766         break;
1767     case 4:
1768         yaz_log(log_level_rpn, "Relation >=");
1769         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1770                       term_dst))
1771             return 0;
1772         term_value = atoi (term_tmp);
1773         gen_regular_rel(term_tmp, term_value, 0);
1774         break;
1775     case 5:
1776         yaz_log(log_level_rpn, "Relation >");
1777         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1778                       term_dst))
1779             return 0;
1780         term_value = atoi (term_tmp);
1781         gen_regular_rel(term_tmp, term_value+1, 0);
1782         break;
1783     case -1:
1784     case 3:
1785         yaz_log(log_level_rpn, "Relation =");
1786         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1787                       term_dst))
1788             return 0;
1789         term_value = atoi (term_tmp);
1790         sprintf(term_tmp, "(0*%d)", term_value);
1791         break;
1792     case 103:
1793         /* term_tmp untouched.. */
1794         while (**term_sub != '\0')
1795             (*term_sub)++;
1796         break;
1797     default:
1798         *error_code = YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
1799         return 0;
1800     }
1801     yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_tmp);
1802     r = dict_lookup_grep(zh->reg->dict, term_dict, 0, grep_info, max_pos,
1803                           0, grep_handle);
1804     if (r)
1805         yaz_log(YLOG_WARN, "dict_lookup_grep fail, rel = gt: %d", r);
1806     yaz_log(log_level_rpn, "%d positions", grep_info->isam_p_indx);
1807     return 1;
1808 }
1809
1810 static ZEBRA_RES numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1811                               const char **term_sub,
1812                               oid_value attributeSet, NMEM stream,
1813                               struct grep_info *grep_info,
1814                               int reg_type, int complete_flag,
1815                               int num_bases, char **basenames,
1816                               char *term_dst,
1817                               const char *xpath_use,
1818                               struct ord_list **ol)
1819 {
1820     char term_dict[2*IT_MAX_WORD+2];
1821     int base_no;
1822     const char *termp;
1823     struct rpn_char_map_info rcmi;
1824
1825     int bases_ok = 0;     /* no of databases with OK attribute */
1826
1827     *ol = ord_list_create(stream);
1828
1829     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1830
1831     for (base_no = 0; base_no < num_bases; base_no++)
1832     {
1833         int max_pos, prefix_len = 0;
1834         int relation_error = 0;
1835         int ord, ord_len, i;
1836         char ord_buf[32];
1837
1838         termp = *term_sub;
1839
1840         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1841         {
1842             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1843                            basenames[base_no]);
1844             return ZEBRA_FAIL;
1845         }
1846
1847         if (zebra_apt_get_ord(zh, zapt, reg_type, xpath_use,
1848                               attributeSet, &ord) != ZEBRA_OK)
1849             continue;
1850         bases_ok++;
1851
1852         *ol = ord_list_append(stream, *ol, ord);
1853
1854         ord_len = key_SU_encode (ord, ord_buf);
1855
1856         term_dict[prefix_len++] = '(';
1857         for (i = 0; i < ord_len; i++)
1858         {
1859             term_dict[prefix_len++] = 1;
1860             term_dict[prefix_len++] = ord_buf[i];
1861         }
1862         term_dict[prefix_len++] = ')';
1863         term_dict[prefix_len] = '\0';
1864
1865         if (!numeric_relation(zh, zapt, &termp, term_dict,
1866                               attributeSet, grep_info, &max_pos, reg_type,
1867                               term_dst, &relation_error))
1868         {
1869             if (relation_error)
1870             {
1871                 zebra_setError(zh, relation_error, 0);
1872                 return ZEBRA_FAIL;
1873             }
1874             *term_sub = 0;
1875             return ZEBRA_OK;
1876         }
1877     }
1878     if (!bases_ok)
1879         return ZEBRA_FAIL;
1880     *term_sub = termp;
1881     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1882     return ZEBRA_OK;
1883 }
1884
1885
1886 static ZEBRA_RES rpn_search_APT_numeric(ZebraHandle zh,
1887                                         Z_AttributesPlusTerm *zapt,
1888                                         const char *termz,
1889                                         oid_value attributeSet,
1890                                         NMEM stream,
1891                                         int reg_type, int complete_flag,
1892                                         const char *rank_type,
1893                                         const char *xpath_use,
1894                                         int num_bases, char **basenames,
1895                                         NMEM rset_nmem,
1896                                         RSET *rset,
1897                                         struct rset_key_control *kc)
1898 {
1899     char term_dst[IT_MAX_WORD+1];
1900     const char *termp = termz;
1901     RSET *result_sets = 0;
1902     int num_result_sets = 0;
1903     ZEBRA_RES res;
1904     struct grep_info grep_info;
1905     int alloc_sets = 0;
1906     zint hits_limit_value;
1907     const char *term_ref_id_str = 0;
1908
1909     term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str, stream);
1910
1911     yaz_log(log_level_rpn, "APT_numeric t='%s'", termz);
1912     if (grep_info_prepare(zh, zapt, &grep_info, reg_type) == ZEBRA_FAIL)
1913         return ZEBRA_FAIL;
1914     while (1)
1915     {
1916         struct ord_list *ol;
1917         if (alloc_sets == num_result_sets)
1918         {
1919             int add = 10;
1920             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1921                                               sizeof(*rnew));
1922             if (alloc_sets)
1923                 memcpy(rnew, result_sets, alloc_sets * sizeof(*rnew));
1924             alloc_sets = alloc_sets + add;
1925             result_sets = rnew;
1926         }
1927         yaz_log(YLOG_DEBUG, "APT_numeric termp=%s", termp);
1928         grep_info.isam_p_indx = 0;
1929         res = numeric_term(zh, zapt, &termp, attributeSet, stream, &grep_info,
1930                            reg_type, complete_flag, num_bases, basenames,
1931                            term_dst, xpath_use, &ol);
1932         if (res == ZEBRA_FAIL || termp == 0)
1933             break;
1934         yaz_log(YLOG_DEBUG, "term: %s", term_dst);
1935         result_sets[num_result_sets] =
1936             rset_trunc(zh, grep_info.isam_p_buf,
1937                        grep_info.isam_p_indx, term_dst,
1938                        strlen(term_dst), rank_type,
1939                        0 /* preserve position */,
1940                        zapt->term->which, rset_nmem,
1941                        kc, kc->scope, ol, reg_type,
1942                        hits_limit_value,
1943                        term_ref_id_str);
1944         if (!result_sets[num_result_sets])
1945             break;
1946         num_result_sets++;
1947         if (!*termp)
1948             break;
1949     }
1950     grep_info_delete(&grep_info);
1951
1952     if (res != ZEBRA_OK)
1953         return res;
1954     if (num_result_sets == 0)
1955         *rset = rset_create_null(rset_nmem, kc, 0);
1956     else if (num_result_sets == 1)
1957         *rset = result_sets[0];
1958     else
1959         *rset = rset_create_and(rset_nmem, kc, kc->scope,
1960                                 num_result_sets, result_sets);
1961     if (!*rset)
1962         return ZEBRA_FAIL;
1963     return ZEBRA_OK;
1964 }
1965
1966 static ZEBRA_RES rpn_search_APT_local(ZebraHandle zh,
1967                                       Z_AttributesPlusTerm *zapt,
1968                                       const char *termz,
1969                                       oid_value attributeSet,
1970                                       NMEM stream,
1971                                       const char *rank_type, NMEM rset_nmem,
1972                                       RSET *rset,
1973                                       struct rset_key_control *kc)
1974 {
1975     RSFD rsfd;
1976     struct it_key key;
1977     int sys;
1978     *rset = rset_create_temp(rset_nmem, kc, kc->scope,
1979                              res_get (zh->res, "setTmpDir"),0 );
1980     rsfd = rset_open(*rset, RSETF_WRITE);
1981
1982     sys = atoi(termz);
1983     if (sys <= 0)
1984         sys = 1;
1985     key.mem[0] = sys;
1986     key.mem[1] = 1;
1987     key.len = 2;
1988     rset_write (rsfd, &key);
1989     rset_close (rsfd);
1990     return ZEBRA_OK;
1991 }
1992
1993 static ZEBRA_RES rpn_sort_spec(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1994                                oid_value attributeSet, NMEM stream,
1995                                Z_SortKeySpecList *sort_sequence,
1996                                const char *rank_type,
1997                                NMEM rset_nmem,
1998                                RSET *rset,
1999                                struct rset_key_control *kc)
2000 {
2001     int i;
2002     int sort_relation_value;
2003     AttrType sort_relation_type;
2004     Z_SortKeySpec *sks;
2005     Z_SortKey *sk;
2006     int oid[OID_SIZE];
2007     oident oe;
2008     char termz[20];
2009
2010     attr_init_APT(&sort_relation_type, zapt, 7);
2011     sort_relation_value = attr_find(&sort_relation_type, &attributeSet);
2012
2013     if (!sort_sequence->specs)
2014     {
2015         sort_sequence->num_specs = 10;
2016         sort_sequence->specs = (Z_SortKeySpec **)
2017             nmem_malloc(stream, sort_sequence->num_specs *
2018                          sizeof(*sort_sequence->specs));
2019         for (i = 0; i<sort_sequence->num_specs; i++)
2020             sort_sequence->specs[i] = 0;
2021     }
2022     if (zapt->term->which != Z_Term_general)
2023         i = 0;
2024     else
2025         i = atoi_n ((char *) zapt->term->u.general->buf,
2026                     zapt->term->u.general->len);
2027     if (i >= sort_sequence->num_specs)
2028         i = 0;
2029     sprintf(termz, "%d", i);
2030
2031     oe.proto = PROTO_Z3950;
2032     oe.oclass = CLASS_ATTSET;
2033     oe.value = attributeSet;
2034     if (!oid_ent_to_oid (&oe, oid))
2035         return ZEBRA_FAIL;
2036
2037     sks = (Z_SortKeySpec *) nmem_malloc(stream, sizeof(*sks));
2038     sks->sortElement = (Z_SortElement *)
2039         nmem_malloc(stream, sizeof(*sks->sortElement));
2040     sks->sortElement->which = Z_SortElement_generic;
2041     sk = sks->sortElement->u.generic = (Z_SortKey *)
2042         nmem_malloc(stream, sizeof(*sk));
2043     sk->which = Z_SortKey_sortAttributes;
2044     sk->u.sortAttributes = (Z_SortAttributes *)
2045         nmem_malloc(stream, sizeof(*sk->u.sortAttributes));
2046
2047     sk->u.sortAttributes->id = oid;
2048     sk->u.sortAttributes->list = zapt->attributes;
2049
2050     sks->sortRelation = (int *)
2051         nmem_malloc(stream, sizeof(*sks->sortRelation));
2052     if (sort_relation_value == 1)
2053         *sks->sortRelation = Z_SortKeySpec_ascending;
2054     else if (sort_relation_value == 2)
2055         *sks->sortRelation = Z_SortKeySpec_descending;
2056     else
2057         *sks->sortRelation = Z_SortKeySpec_ascending;
2058
2059     sks->caseSensitivity = (int *)
2060         nmem_malloc(stream, sizeof(*sks->caseSensitivity));
2061     *sks->caseSensitivity = 0;
2062
2063     sks->which = Z_SortKeySpec_null;
2064     sks->u.null = odr_nullval ();
2065     sort_sequence->specs[i] = sks;
2066     *rset = rset_create_null(rset_nmem, kc, 0);
2067     return ZEBRA_OK;
2068 }
2069
2070
2071 static int rpn_check_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2072                            oid_value attributeSet,
2073                            struct xpath_location_step *xpath, int max,
2074                            NMEM mem)
2075 {
2076     oid_value curAttributeSet = attributeSet;
2077     AttrType use;
2078     const char *use_string = 0;
2079
2080     attr_init_APT(&use, zapt, 1);
2081     attr_find_ex(&use, &curAttributeSet, &use_string);
2082
2083     if (!use_string || *use_string != '/')
2084         return -1;
2085
2086     return zebra_parse_xpath_str(use_string, xpath, max, mem);
2087 }
2088
2089
2090
2091 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
2092                         int reg_type, const char *term,
2093                         const char *xpath_use,
2094                         NMEM rset_nmem,
2095                         struct rset_key_control *kc)
2096 {
2097     RSET rset;
2098     struct grep_info grep_info;
2099     char term_dict[2048];
2100     char ord_buf[32];
2101     int prefix_len = 0;
2102     int ord = zebraExplain_lookup_attr_str(zh->reg->zei,
2103                                            zinfo_index_category_index,
2104                                            reg_type,
2105                                            xpath_use);
2106     int ord_len, i, r, max_pos;
2107     int term_type = Z_Term_characterString;
2108     const char *flags = "void";
2109
2110     if (grep_info_prepare(zh, 0 /* zapt */, &grep_info, '0') == ZEBRA_FAIL)
2111         return rset_create_null(rset_nmem, kc, 0);
2112
2113     if (ord < 0)
2114         return rset_create_null(rset_nmem, kc, 0);
2115     if (prefix_len)
2116         term_dict[prefix_len++] = '|';
2117     else
2118         term_dict[prefix_len++] = '(';
2119
2120     ord_len = key_SU_encode (ord, ord_buf);
2121     for (i = 0; i<ord_len; i++)
2122     {
2123         term_dict[prefix_len++] = 1;
2124         term_dict[prefix_len++] = ord_buf[i];
2125     }
2126     term_dict[prefix_len++] = ')';
2127     strcpy(term_dict+prefix_len, term);
2128
2129     grep_info.isam_p_indx = 0;
2130     r = dict_lookup_grep(zh->reg->dict, term_dict, 0,
2131                           &grep_info, &max_pos, 0, grep_handle);
2132     yaz_log(YLOG_DEBUG, "%s %d positions", term,
2133              grep_info.isam_p_indx);
2134     rset = rset_trunc(zh, grep_info.isam_p_buf,
2135                       grep_info.isam_p_indx, term, strlen(term),
2136                       flags, 1, term_type,rset_nmem,
2137                       kc, kc->scope, 0, reg_type, 0 /* hits_limit */,
2138                       0 /* term_ref_id_str */);
2139     grep_info_delete(&grep_info);
2140     return rset;
2141 }
2142
2143 static
2144 ZEBRA_RES rpn_search_xpath(ZebraHandle zh,
2145                            int num_bases, char **basenames,
2146                            NMEM stream, const char *rank_type, RSET rset,
2147                            int xpath_len, struct xpath_location_step *xpath,
2148                            NMEM rset_nmem,
2149                            RSET *rset_out,
2150                            struct rset_key_control *kc)
2151 {
2152     int base_no;
2153     int i;
2154     int always_matches = rset ? 0 : 1;
2155
2156     if (xpath_len < 0)
2157     {
2158         *rset_out = rset;
2159         return ZEBRA_OK;
2160     }
2161
2162     yaz_log(YLOG_DEBUG, "xpath len=%d", xpath_len);
2163     for (i = 0; i<xpath_len; i++)
2164     {
2165         yaz_log(log_level_rpn, "XPATH %d %s", i, xpath[i].part);
2166
2167     }
2168
2169     /*
2170       //a    ->    a/.*
2171       //a/b  ->    b/a/.*
2172       /a     ->    a/
2173       /a/b   ->    b/a/
2174
2175       /      ->    none
2176
2177    a[@attr = value]/b[@other = othervalue]
2178
2179  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
2180  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
2181  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
2182  /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y)
2183  /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y)
2184  /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x)
2185
2186     */
2187
2188     dict_grep_cmap (zh->reg->dict, 0, 0);
2189
2190     for (base_no = 0; base_no < num_bases; base_no++)
2191     {
2192         int level = xpath_len;
2193         int first_path = 1;
2194
2195         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2196         {
2197             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
2198                            basenames[base_no]);
2199             *rset_out = rset;
2200             return ZEBRA_FAIL;
2201         }
2202         while (--level >= 0)
2203         {
2204             WRBUF xpath_rev = wrbuf_alloc();
2205             int i;
2206             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
2207
2208             for (i = level; i >= 1; --i)
2209             {
2210                 const char *cp = xpath[i].part;
2211                 if (*cp)
2212                 {
2213                     for (; *cp; cp++)
2214                     {
2215                         if (*cp == '*')
2216                             wrbuf_puts(xpath_rev, "[^/]*");
2217                         else if (*cp == ' ')
2218                             wrbuf_puts(xpath_rev, "\001 ");
2219                         else
2220                             wrbuf_putc(xpath_rev, *cp);
2221
2222                         /* wrbuf_putc does not null-terminate , but
2223                            wrbuf_puts below ensures it does.. so xpath_rev
2224                            is OK iff length is > 0 */
2225                     }
2226                     wrbuf_puts(xpath_rev, "/");
2227                 }
2228                 else if (i == 1)  /* // case */
2229                     wrbuf_puts(xpath_rev, ".*");
2230             }
2231             if (xpath[level].predicate &&
2232                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
2233                 xpath[level].predicate->u.relation.name[0])
2234             {
2235                 WRBUF wbuf = wrbuf_alloc();
2236                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
2237                 if (xpath[level].predicate->u.relation.value)
2238                 {
2239                     const char *cp = xpath[level].predicate->u.relation.value;
2240                     wrbuf_putc(wbuf, '=');
2241
2242                     while (*cp)
2243                     {
2244                         if (strchr(REGEX_CHARS, *cp))
2245                             wrbuf_putc(wbuf, '\\');
2246                         wrbuf_putc(wbuf, *cp);
2247                         cp++;
2248                     }
2249                 }
2250                 wrbuf_puts(wbuf, "");
2251                 rset_attr = xpath_trunc(
2252                     zh, stream, '0', wrbuf_buf(wbuf), ZEBRA_XPATH_ATTR_NAME,
2253                     rset_nmem, kc);
2254                 wrbuf_free(wbuf, 1);
2255             }
2256             else
2257             {
2258                 if (!first_path)
2259                 {
2260                     wrbuf_free(xpath_rev, 1);
2261                     continue;
2262                 }
2263             }
2264             yaz_log(log_level_rpn, "xpath_rev (%d) = %.*s", level,
2265                     wrbuf_len(xpath_rev), wrbuf_buf(xpath_rev));
2266             if (wrbuf_len(xpath_rev))
2267             {
2268                 rset_start_tag = xpath_trunc(zh, stream, '0',
2269                                              wrbuf_buf(xpath_rev),
2270                                              ZEBRA_XPATH_ELM_BEGIN,
2271                                              rset_nmem, kc);
2272                 if (always_matches)
2273                     rset = rset_start_tag;
2274                 else
2275                 {
2276                     rset_end_tag = xpath_trunc(zh, stream, '0',
2277                                                wrbuf_buf(xpath_rev),
2278                                                ZEBRA_XPATH_ELM_END,
2279                                                rset_nmem, kc);
2280
2281                     rset = rset_create_between(rset_nmem, kc, kc->scope,
2282                                                rset_start_tag, rset,
2283                                                rset_end_tag, rset_attr);
2284                 }
2285             }
2286             wrbuf_free(xpath_rev, 1);
2287             first_path = 0;
2288         }
2289     }
2290     *rset_out = rset;
2291     return ZEBRA_OK;
2292 }
2293
2294 #define MAX_XPATH_STEPS 10
2295
2296 static ZEBRA_RES rpn_search_APT(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2297                                 oid_value attributeSet, NMEM stream,
2298                                 Z_SortKeySpecList *sort_sequence,
2299                                 int num_bases, char **basenames,
2300                                 NMEM rset_nmem,
2301                                 RSET *rset,
2302                                 struct rset_key_control *kc)
2303 {
2304     ZEBRA_RES res = ZEBRA_OK;
2305     unsigned reg_id;
2306     char *search_type = NULL;
2307     char rank_type[128];
2308     int complete_flag;
2309     int sort_flag;
2310     char termz[IT_MAX_WORD+1];
2311     int xpath_len;
2312     const char *xpath_use = 0;
2313     struct xpath_location_step xpath[MAX_XPATH_STEPS];
2314
2315     if (!log_level_set)
2316     {
2317         log_level_rpn = yaz_log_module_level("rpn");
2318         log_level_set = 1;
2319     }
2320     zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2321                     rank_type, &complete_flag, &sort_flag);
2322
2323     yaz_log(YLOG_DEBUG, "reg_id=%c", reg_id);
2324     yaz_log(YLOG_DEBUG, "complete_flag=%d", complete_flag);
2325     yaz_log(YLOG_DEBUG, "search_type=%s", search_type);
2326     yaz_log(YLOG_DEBUG, "rank_type=%s", rank_type);
2327
2328     if (zapt_term_to_utf8(zh, zapt, termz) == ZEBRA_FAIL)
2329         return ZEBRA_FAIL;
2330
2331     if (sort_flag)
2332         return rpn_sort_spec(zh, zapt, attributeSet, stream, sort_sequence,
2333                              rank_type, rset_nmem, rset, kc);
2334     /* consider if an X-Path query is used */
2335     xpath_len = rpn_check_xpath(zh, zapt, attributeSet,
2336                                 xpath, MAX_XPATH_STEPS, stream);
2337     if (xpath_len >= 0)
2338     {
2339         if (xpath[xpath_len-1].part[0] == '@')
2340             xpath_use = ZEBRA_XPATH_ATTR_CDATA;  /* last step is attribute  */
2341         else
2342             xpath_use = ZEBRA_XPATH_CDATA;  /* searching for cdata */
2343
2344         if (1)
2345         {
2346             AttrType relation;
2347             int relation_value;
2348
2349             attr_init_APT(&relation, zapt, 2);
2350             relation_value = attr_find(&relation, NULL);
2351
2352             if (relation_value == 103) /* alwaysmatches */
2353             {
2354                 *rset = 0; /* signal no "term" set */
2355                 return rpn_search_xpath(zh, num_bases, basenames,
2356                                         stream, rank_type, *rset,
2357                                         xpath_len, xpath, rset_nmem, rset, kc);
2358             }
2359         }
2360     }
2361
2362     /* search using one of the various search type strategies
2363        termz is our UTF-8 search term
2364        attributeSet is top-level default attribute set
2365        stream is ODR for search
2366        reg_id is the register type
2367        complete_flag is 1 for complete subfield, 0 for incomplete
2368        xpath_use is use-attribute to be used for X-Path search, 0 for none
2369     */
2370     if (!strcmp(search_type, "phrase"))
2371     {
2372         res = rpn_search_APT_phrase(zh, zapt, termz, attributeSet, stream,
2373                                     reg_id, complete_flag, rank_type,
2374                                     xpath_use,
2375                                     num_bases, basenames, rset_nmem,
2376                                     rset, kc);
2377     }
2378     else if (!strcmp(search_type, "and-list"))
2379     {
2380         res = rpn_search_APT_and_list(zh, zapt, termz, attributeSet, stream,
2381                                       reg_id, complete_flag, rank_type,
2382                                       xpath_use,
2383                                       num_bases, basenames, rset_nmem,
2384                                       rset, kc);
2385     }
2386     else if (!strcmp(search_type, "or-list"))
2387     {
2388         res = rpn_search_APT_or_list(zh, zapt, termz, attributeSet, stream,
2389                                      reg_id, complete_flag, rank_type,
2390                                      xpath_use,
2391                                      num_bases, basenames, rset_nmem,
2392                                      rset, kc);
2393     }
2394     else if (!strcmp(search_type, "local"))
2395     {
2396         res = rpn_search_APT_local(zh, zapt, termz, attributeSet, stream,
2397                                    rank_type, rset_nmem, rset, kc);
2398     }
2399     else if (!strcmp(search_type, "numeric"))
2400     {
2401         res = rpn_search_APT_numeric(zh, zapt, termz, attributeSet, stream,
2402                                      reg_id, complete_flag, rank_type,
2403                                      xpath_use,
2404                                      num_bases, basenames, rset_nmem,
2405                                      rset, kc);
2406     }
2407     else
2408     {
2409         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2410         res = ZEBRA_FAIL;
2411     }
2412     if (res != ZEBRA_OK)
2413         return res;
2414     if (!*rset)
2415         return ZEBRA_FAIL;
2416     return rpn_search_xpath(zh, num_bases, basenames,
2417                             stream, rank_type, *rset,
2418                             xpath_len, xpath, rset_nmem, rset, kc);
2419 }
2420
2421 static ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2422                                       oid_value attributeSet,
2423                                       NMEM stream, NMEM rset_nmem,
2424                                       Z_SortKeySpecList *sort_sequence,
2425                                       int num_bases, char **basenames,
2426                                       RSET **result_sets, int *num_result_sets,
2427                                       Z_Operator *parent_op,
2428                                       struct rset_key_control *kc);
2429
2430 ZEBRA_RES rpn_search_top(ZebraHandle zh, Z_RPNStructure *zs,
2431                          oid_value attributeSet,
2432                          NMEM stream, NMEM rset_nmem,
2433                          Z_SortKeySpecList *sort_sequence,
2434                          int num_bases, char **basenames,
2435                          RSET *result_set)
2436 {
2437     RSET *result_sets = 0;
2438     int num_result_sets = 0;
2439     ZEBRA_RES res;
2440     struct rset_key_control *kc = zebra_key_control_create(zh);
2441
2442     res = rpn_search_structure(zh, zs, attributeSet,
2443                                stream, rset_nmem,
2444                                sort_sequence,
2445                                num_bases, basenames,
2446                                &result_sets, &num_result_sets,
2447                                0 /* no parent op */,
2448                                kc);
2449     if (res != ZEBRA_OK)
2450     {
2451         int i;
2452         for (i = 0; i<num_result_sets; i++)
2453             rset_delete(result_sets[i]);
2454         *result_set = 0;
2455     }
2456     else
2457     {
2458         assert(num_result_sets == 1);
2459         assert(result_sets);
2460         assert(*result_sets);
2461         *result_set = *result_sets;
2462     }
2463     (*kc->dec)(kc);
2464     return res;
2465 }
2466
2467 ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2468                                oid_value attributeSet,
2469                                NMEM stream, NMEM rset_nmem,
2470                                Z_SortKeySpecList *sort_sequence,
2471                                int num_bases, char **basenames,
2472                                RSET **result_sets, int *num_result_sets,
2473                                Z_Operator *parent_op,
2474                                struct rset_key_control *kc)
2475 {
2476     *num_result_sets = 0;
2477     if (zs->which == Z_RPNStructure_complex)
2478     {
2479         ZEBRA_RES res;
2480         Z_Operator *zop = zs->u.complex->roperator;
2481         RSET *result_sets_l = 0;
2482         int num_result_sets_l = 0;
2483         RSET *result_sets_r = 0;
2484         int num_result_sets_r = 0;
2485
2486         res = rpn_search_structure(zh, zs->u.complex->s1,
2487                                    attributeSet, stream, rset_nmem,
2488                                    sort_sequence,
2489                                    num_bases, basenames,
2490                                    &result_sets_l, &num_result_sets_l,
2491                                    zop, kc);
2492         if (res != ZEBRA_OK)
2493         {
2494             int i;
2495             for (i = 0; i<num_result_sets_l; i++)
2496                 rset_delete(result_sets_l[i]);
2497             return res;
2498         }
2499         res = rpn_search_structure(zh, zs->u.complex->s2,
2500                                    attributeSet, stream, rset_nmem,
2501                                    sort_sequence,
2502                                    num_bases, basenames,
2503                                    &result_sets_r, &num_result_sets_r,
2504                                    zop, kc);
2505         if (res != ZEBRA_OK)
2506         {
2507             int i;
2508             for (i = 0; i<num_result_sets_l; i++)
2509                 rset_delete(result_sets_l[i]);
2510             for (i = 0; i<num_result_sets_r; i++)
2511                 rset_delete(result_sets_r[i]);
2512             return res;
2513         }
2514
2515         /* make a new list of result for all children */
2516         *num_result_sets = num_result_sets_l + num_result_sets_r;
2517         *result_sets = nmem_malloc(stream, *num_result_sets *
2518                                    sizeof(**result_sets));
2519         memcpy(*result_sets, result_sets_l,
2520                num_result_sets_l * sizeof(**result_sets));
2521         memcpy(*result_sets + num_result_sets_l, result_sets_r,
2522                num_result_sets_r * sizeof(**result_sets));
2523
2524         if (!parent_op || parent_op->which != zop->which
2525             || (zop->which != Z_Operator_and &&
2526                 zop->which != Z_Operator_or))
2527         {
2528             /* parent node different from this one (or non-present) */
2529             /* we must combine result sets now */
2530             RSET rset;
2531             switch (zop->which)
2532             {
2533             case Z_Operator_and:
2534                 rset = rset_create_and(rset_nmem, kc,
2535                                        kc->scope,
2536                                        *num_result_sets, *result_sets);
2537                 break;
2538             case Z_Operator_or:
2539                 rset = rset_create_or(rset_nmem, kc,
2540                                       kc->scope, 0, /* termid */
2541                                       *num_result_sets, *result_sets);
2542                 break;
2543             case Z_Operator_and_not:
2544                 rset = rset_create_not(rset_nmem, kc,
2545                                        kc->scope,
2546                                        (*result_sets)[0],
2547                                        (*result_sets)[1]);
2548                 break;
2549             case Z_Operator_prox:
2550                 if (zop->u.prox->which != Z_ProximityOperator_known)
2551                 {
2552                     zebra_setError(zh,
2553                                    YAZ_BIB1_UNSUPP_PROX_UNIT_CODE,
2554                                    0);
2555                     return ZEBRA_FAIL;
2556                 }
2557                 if (*zop->u.prox->u.known != Z_ProxUnit_word)
2558                 {
2559                     zebra_setError_zint(zh,
2560                                         YAZ_BIB1_UNSUPP_PROX_UNIT_CODE,
2561                                         *zop->u.prox->u.known);
2562                     return ZEBRA_FAIL;
2563                 }
2564                 else
2565                 {
2566                     rset = rset_create_prox(rset_nmem, kc,
2567                                             kc->scope,
2568                                             *num_result_sets, *result_sets,
2569                                             *zop->u.prox->ordered,
2570                                             (!zop->u.prox->exclusion ?
2571                                              0 : *zop->u.prox->exclusion),
2572                                             *zop->u.prox->relationType,
2573                                             *zop->u.prox->distance );
2574                 }
2575                 break;
2576             default:
2577                 zebra_setError(zh, YAZ_BIB1_OPERATOR_UNSUPP, 0);
2578                 return ZEBRA_FAIL;
2579             }
2580             *num_result_sets = 1;
2581             *result_sets = nmem_malloc(stream, *num_result_sets *
2582                                        sizeof(**result_sets));
2583             (*result_sets)[0] = rset;
2584         }
2585     }
2586     else if (zs->which == Z_RPNStructure_simple)
2587     {
2588         RSET rset;
2589         ZEBRA_RES res;
2590
2591         if (zs->u.simple->which == Z_Operand_APT)
2592         {
2593             yaz_log(YLOG_DEBUG, "rpn_search_APT");
2594             res = rpn_search_APT(zh, zs->u.simple->u.attributesPlusTerm,
2595                                  attributeSet, stream, sort_sequence,
2596                                  num_bases, basenames, rset_nmem, &rset,
2597                                  kc);
2598             if (res != ZEBRA_OK)
2599                 return res;
2600         }
2601         else if (zs->u.simple->which == Z_Operand_resultSetId)
2602         {
2603             yaz_log(YLOG_DEBUG, "rpn_search_ref");
2604             rset = resultSetRef(zh, zs->u.simple->u.resultSetId);
2605             if (!rset)
2606             {
2607                 zebra_setError(zh,
2608                                YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
2609                                zs->u.simple->u.resultSetId);
2610                 return ZEBRA_FAIL;
2611             }
2612             rset_dup(rset);
2613         }
2614         else
2615         {
2616             zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2617             return ZEBRA_FAIL;
2618         }
2619         *num_result_sets = 1;
2620         *result_sets = nmem_malloc(stream, *num_result_sets *
2621                                    sizeof(**result_sets));
2622         (*result_sets)[0] = rset;
2623     }
2624     else
2625     {
2626         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2627         return ZEBRA_FAIL;
2628     }
2629     return ZEBRA_OK;
2630 }
2631
2632 struct scan_info_entry {
2633     char *term;
2634     ISAM_P isam_p;
2635 };
2636
2637 struct scan_info {
2638     struct scan_info_entry *list;
2639     ODR odr;
2640     int before, after;
2641     char prefix[20];
2642 };
2643
2644 static int scan_handle (char *name, const char *info, int pos, void *client)
2645 {
2646     int len_prefix, idx;
2647     struct scan_info *scan_info = (struct scan_info *) client;
2648
2649     len_prefix = strlen(scan_info->prefix);
2650     if (memcmp (name, scan_info->prefix, len_prefix))
2651         return 1;
2652     if (pos > 0)
2653         idx = scan_info->after - pos + scan_info->before;
2654     else
2655         idx = - pos - 1;
2656
2657     /* skip special terms.. of no interest */
2658     if (name[len_prefix] < 4)
2659         return 1;
2660
2661     if (idx < 0)
2662         return 0;
2663     scan_info->list[idx].term = (char *)
2664         odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
2665     strcpy(scan_info->list[idx].term, name + len_prefix);
2666     assert (*info == sizeof(ISAM_P));
2667     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
2668     return 0;
2669 }
2670
2671 void zebra_term_untrans_iconv(ZebraHandle zh, NMEM stream, int reg_type,
2672                               char **dst, const char *src)
2673 {
2674     char term_src[IT_MAX_WORD];
2675     char term_dst[IT_MAX_WORD];
2676
2677     zebra_term_untrans (zh, reg_type, term_src, src);
2678
2679     if (zh->iconv_from_utf8 != 0)
2680     {
2681         int len;
2682         char *inbuf = term_src;
2683         size_t inleft = strlen(term_src);
2684         char *outbuf = term_dst;
2685         size_t outleft = sizeof(term_dst)-1;
2686         size_t ret;
2687
2688         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2689                          &outbuf, &outleft);
2690         if (ret == (size_t)(-1))
2691             len = 0;
2692         else
2693             len = outbuf - term_dst;
2694         *dst = nmem_malloc(stream, len + 1);
2695         if (len > 0)
2696             memcpy (*dst, term_dst, len);
2697         (*dst)[len] = '\0';
2698     }
2699     else
2700         *dst = nmem_strdup(stream, term_src);
2701 }
2702
2703 static void count_set(ZebraHandle zh, RSET rset, zint *count)
2704 {
2705     zint psysno = 0;
2706     struct it_key key;
2707     RSFD rfd;
2708
2709     yaz_log(YLOG_DEBUG, "count_set");
2710
2711     rset->hits_limit = zh->approx_limit;
2712
2713     *count = 0;
2714     rfd = rset_open(rset, RSETF_READ);
2715     while (rset_read(rfd, &key,0 /* never mind terms */))
2716     {
2717         if (key.mem[0] != psysno)
2718         {
2719             psysno = key.mem[0];
2720             if (rfd->counted_items >= rset->hits_limit)
2721                 break;
2722         }
2723     }
2724     rset_close (rfd);
2725     *count = rset->hits_count;
2726 }
2727
2728 #define RPN_MAX_ORDS 32
2729
2730 ZEBRA_RES rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2731                    oid_value attributeset,
2732                    int num_bases, char **basenames,
2733                    int *position, int *num_entries, ZebraScanEntry **list,
2734                    int *is_partial, RSET limit_set, int return_zero)
2735 {
2736     int i;
2737     int pos = *position;
2738     int num = *num_entries;
2739     int before;
2740     int after;
2741     int base_no;
2742     char termz[IT_MAX_WORD+20];
2743     struct scan_info *scan_info_array;
2744     ZebraScanEntry *glist;
2745     int ords[RPN_MAX_ORDS], ord_no = 0;
2746     int ptr[RPN_MAX_ORDS];
2747
2748     unsigned index_type;
2749     char *search_type = NULL;
2750     char rank_type[128];
2751     int complete_flag;
2752     int sort_flag;
2753     NMEM rset_nmem = NULL;
2754     struct rset_key_control *kc = 0;
2755
2756     *list = 0;
2757     *is_partial = 0;
2758
2759     if (attributeset == VAL_NONE)
2760         attributeset = VAL_BIB1;
2761
2762     if (!limit_set)
2763     {
2764         AttrType termset;
2765         int termset_value_numeric;
2766         const char *termset_value_string;
2767         attr_init_APT(&termset, zapt, 8);
2768         termset_value_numeric =
2769             attr_find_ex(&termset, NULL, &termset_value_string);
2770         if (termset_value_numeric != -1)
2771         {
2772             char resname[32];
2773             const char *termset_name = 0;
2774
2775             if (termset_value_numeric != -2)
2776             {
2777
2778                 sprintf(resname, "%d", termset_value_numeric);
2779                 termset_name = resname;
2780             }
2781             else
2782                 termset_name = termset_value_string;
2783
2784             limit_set = resultSetRef (zh, termset_name);
2785         }
2786     }
2787
2788     yaz_log(YLOG_DEBUG, "position = %d, num = %d set=%d",
2789             pos, num, attributeset);
2790
2791     if (zebra_maps_attr(zh->reg->zebra_maps, zapt, &index_type, &search_type,
2792                         rank_type, &complete_flag, &sort_flag))
2793     {
2794         *num_entries = 0;
2795         zebra_setError(zh, YAZ_BIB1_UNSUPP_ATTRIBUTE_TYPE, 0);
2796         return ZEBRA_FAIL;
2797     }
2798     for (base_no = 0; base_no < num_bases && ord_no < RPN_MAX_ORDS; base_no++)
2799     {
2800         int ord;
2801
2802         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2803         {
2804             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
2805                            basenames[base_no]);
2806             *num_entries = 0;
2807             return ZEBRA_FAIL;
2808         }
2809         if (zebra_apt_get_ord(zh, zapt, index_type, 0, attributeset, &ord)
2810             != ZEBRA_OK)
2811             continue;
2812         ords[ord_no++] = ord;
2813     }
2814     if (ord_no == 0)
2815     {
2816         *num_entries = 0;
2817         return ZEBRA_FAIL;
2818     }
2819     /* prepare dictionary scanning */
2820     if (num < 1)
2821     {
2822         *num_entries = 0;
2823         return ZEBRA_OK;
2824     }
2825     before = pos-1;
2826     if (before < 0)
2827         before = 0;
2828     after = 1+num-pos;
2829     if (after < 0)
2830         after = 0;
2831     yaz_log(YLOG_DEBUG, "rpn_scan pos=%d num=%d before=%d "
2832             "after=%d before+after=%d",
2833             pos, num, before, after, before+after);
2834     scan_info_array = (struct scan_info *)
2835         odr_malloc(stream, ord_no * sizeof(*scan_info_array));
2836     for (i = 0; i < ord_no; i++)
2837     {
2838         int j, prefix_len = 0;
2839         int before_tmp = before, after_tmp = after;
2840         struct scan_info *scan_info = scan_info_array + i;
2841         struct rpn_char_map_info rcmi;
2842
2843         rpn_char_map_prepare (zh->reg, index_type, &rcmi);
2844
2845         scan_info->before = before;
2846         scan_info->after = after;
2847         scan_info->odr = stream;
2848
2849         scan_info->list = (struct scan_info_entry *)
2850             odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
2851         for (j = 0; j<before+after; j++)
2852             scan_info->list[j].term = NULL;
2853
2854         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2855         termz[prefix_len] = 0;
2856         strcpy(scan_info->prefix, termz);
2857
2858         if (trans_scan_term(zh, zapt, termz+prefix_len, index_type) ==
2859             ZEBRA_FAIL)
2860             return ZEBRA_FAIL;
2861
2862         dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
2863                   scan_info, scan_handle);
2864     }
2865     glist = (ZebraScanEntry *)
2866         odr_malloc(stream, (before+after)*sizeof(*glist));
2867
2868     rset_nmem = nmem_create();
2869     kc = zebra_key_control_create(zh);
2870
2871     /* consider terms after main term */
2872     for (i = 0; i < ord_no; i++)
2873         ptr[i] = before;
2874
2875     *is_partial = 0;
2876     for (i = 0; i<after; i++)
2877     {
2878         int j, j0 = -1;
2879         const char *mterm = NULL;
2880         const char *tst;
2881         RSET rset = 0;
2882         int lo = i + pos-1; /* offset in result list */
2883
2884         /* find: j0 is the first of the minimal values */
2885         for (j = 0; j < ord_no; j++)
2886         {
2887             if (ptr[j] < before+after && ptr[j] >= 0 &&
2888                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2889                 (!mterm || strcmp (tst, mterm) < 0))
2890             {
2891                 j0 = j;
2892                 mterm = tst;
2893             }
2894         }
2895         if (j0 == -1)
2896             break;  /* no value found, stop */
2897
2898         /* get result set for first one , but only if it's within bounds */
2899         if (lo >= 0)
2900         {
2901             /* get result set for first term */
2902             zebra_term_untrans_iconv(zh, stream->mem, index_type,
2903                                      &glist[lo].term, mterm);
2904             rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2905                               glist[lo].term, strlen(glist[lo].term),
2906                               NULL, 0, zapt->term->which, rset_nmem,
2907                               kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2908                               0 /* term_ref_id_str */);
2909         }
2910         ptr[j0]++; /* move index for this set .. */
2911         /* get result set for remaining scan terms */
2912         for (j = j0+1; j<ord_no; j++)
2913         {
2914             if (ptr[j] < before+after && ptr[j] >= 0 &&
2915                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2916                 !strcmp (tst, mterm))
2917             {
2918                 if (lo >= 0)
2919                 {
2920                     RSET rsets[2];
2921
2922                     rsets[0] = rset;
2923                     rsets[1] =
2924                         rset_trunc(
2925                             zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
2926                             glist[lo].term,
2927                             strlen(glist[lo].term), NULL, 0,
2928                             zapt->term->which,rset_nmem,
2929                             kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2930                             0 /* term_ref_id_str */ );
2931                     rset = rset_create_or(rset_nmem, kc,
2932                                           kc->scope, 0 /* termid */,
2933                                           2, rsets);
2934                 }
2935                 ptr[j]++;
2936             }
2937         }
2938         if (lo >= 0)
2939         {
2940             zint count;
2941             /* merge with limit_set if given */
2942             if (limit_set)
2943             {
2944                 RSET rsets[2];
2945                 rsets[0] = rset;
2946                 rsets[1] = rset_dup(limit_set);
2947
2948                 rset = rset_create_and(rset_nmem, kc, kc->scope, 2, rsets);
2949             }
2950             /* count it */
2951             count_set(zh, rset, &count);
2952             glist[lo].occurrences = count;
2953             rset_delete(rset);
2954         }
2955     }
2956     if (i < after)
2957     {
2958         *num_entries -= (after-i);
2959         *is_partial = 1;
2960         if (*num_entries < 0)
2961         {
2962             (*kc->dec)(kc);
2963             nmem_destroy(rset_nmem);
2964             *num_entries = 0;
2965             return ZEBRA_OK;
2966         }
2967     }
2968     /* consider terms before main term */
2969     for (i = 0; i<ord_no; i++)
2970         ptr[i] = 0;
2971
2972     for (i = 0; i<before; i++)
2973     {
2974         int j, j0 = -1;
2975         const char *mterm = NULL;
2976         const char *tst;
2977         RSET rset;
2978         int lo = before-1-i; /* offset in result list */
2979         zint count;
2980
2981         for (j = 0; j <ord_no; j++)
2982         {
2983             if (ptr[j] < before && ptr[j] >= 0 &&
2984                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2985                 (!mterm || strcmp (tst, mterm) > 0))
2986             {
2987                 j0 = j;
2988                     mterm = tst;
2989             }
2990         }
2991         if (j0 == -1)
2992             break;
2993
2994         zebra_term_untrans_iconv(zh, stream->mem, index_type,
2995                                  &glist[lo].term, mterm);
2996
2997         rset = rset_trunc
2998             (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
2999              glist[lo].term, strlen(glist[lo].term),
3000              NULL, 0, zapt->term->which, rset_nmem,
3001              kc, kc->scope, 0, index_type, 0 /* hits_limit */,
3002              0 /* term_ref_id_str */);
3003
3004         ptr[j0]++;
3005
3006         for (j = j0+1; j<ord_no; j++)
3007         {
3008             if (ptr[j] < before && ptr[j] >= 0 &&
3009                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
3010                 !strcmp (tst, mterm))
3011             {
3012                 RSET rsets[2];
3013
3014                 rsets[0] = rset;
3015                 rsets[1] = rset_trunc(
3016                     zh,
3017                     &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
3018                     glist[lo].term,
3019                     strlen(glist[lo].term), NULL, 0,
3020                     zapt->term->which, rset_nmem,
3021                     kc, kc->scope, 0, index_type, 0 /* hits_limit */,
3022                     0 /* term_ref_id_str */);
3023                 rset = rset_create_or(rset_nmem, kc,
3024                                       kc->scope, 0 /* termid */, 2, rsets);
3025
3026                 ptr[j]++;
3027             }
3028         }
3029         if (limit_set)
3030         {
3031             RSET rsets[2];
3032             rsets[0] = rset;
3033             rsets[1] = rset_dup(limit_set);
3034
3035             rset = rset_create_and(rset_nmem, kc, kc->scope, 2, rsets);
3036         }
3037         count_set(zh, rset, &count);
3038         glist[lo].occurrences = count;
3039         rset_delete (rset);
3040     }
3041     (*kc->dec)(kc);
3042     nmem_destroy(rset_nmem);
3043     i = before-i;
3044     if (i)
3045     {
3046         *is_partial = 1;
3047         *position -= i;
3048         *num_entries -= i;
3049         if (*num_entries <= 0)
3050         {
3051             *num_entries = 0;
3052             return ZEBRA_OK;
3053         }
3054     }
3055
3056     *list = glist + i;               /* list is set to first 'real' entry */
3057
3058     yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
3059             *position, *num_entries);
3060     return ZEBRA_OK;
3061 }
3062
3063 /*
3064  * Local variables:
3065  * c-basic-offset: 4
3066  * indent-tabs-mode: nil
3067  * End:
3068  * vim: shiftwidth=4 tabstop=8 expandtab
3069  */
3070