index/zrpn.c

   1 /* $Id: zrpn.c,v 1.215 2006-05-19 23:20:24 adam Exp $
   2    Copyright (C) 1995-2006
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with Zebra; see the file LICENSE.zebra.  If not, write to the
  19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  20 02111-1307, USA.
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #ifdef WIN32
  26 #include <io.h>
  27 #endif
  28 #if HAVE_UNISTD_H
  29 #include <unistd.h>
  30 #endif
  31 #include <ctype.h>
  32
  33 #include <yaz/diagbib1.h>
  34 #include "index.h"
  35 #include <zebra_xpath.h>
  36 #include <attrfind.h>
  37 #include <charmap.h>
  38 #include <rset.h>
  39
  40 struct rpn_char_map_info
  41 {
  42     ZebraMaps zm;
  43     int reg_type;
  44 };
  45
  46 static int log_level_set = 0;
  47 static int log_level_rpn = 0;
  48
  49 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  50 {
  51     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  52     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  53 #if 0
  54     if (out && *out)
  55     {
  56         const char *outp = *out;
  57         yaz_log(YLOG_LOG, "---");
  58         while (*outp)
  59         {
  60             yaz_log(YLOG_LOG, "%02X", *outp);
  61             outp++;
  62         }
  63     }
  64 #endif
  65     return out;
  66 }
  67
  68 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  69                                   struct rpn_char_map_info *map_info)
  70 {
  71     map_info->zm = reg->zebra_maps;
  72     map_info->reg_type = reg_type;
  73     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  74 }
  75
  76 #define TERM_COUNT
  77
  78 struct grep_info {
  79 #ifdef TERM_COUNT
  80     int *term_no;
  81 #endif
  82     ISAM_P *isam_p_buf;
  83     int isam_p_size;
  84     int isam_p_indx;
  85     ZebraHandle zh;
  86     int reg_type;
  87     ZebraSet termset;
  88 };
  89
  90 void zebra_term_untrans(ZebraHandle zh, int reg_type,
  91                         char *dst, const char *src)
  92 {
  93     int len = 0;
  94     while (*src)
  95     {
  96         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
  97                                            reg_type, &src);
  98         if (!cp)
  99         {
 100             if (len < IT_MAX_WORD-1)
 101                 dst[len++] = *src;
 102             src++;
 103         }
 104         else
 105             while (*cp && len < IT_MAX_WORD-1)
 106                 dst[len++] = *cp++;
 107     }
 108     dst[len] = '\0';
 109 }
 110
 111 static void add_isam_p(const char *name, const char *info,
 112                        struct grep_info *p)
 113 {
 114     if (!log_level_set)
 115     {
 116         log_level_rpn = yaz_log_module_level("rpn");
 117         log_level_set = 1;
 118     }
 119     if (p->isam_p_indx == p->isam_p_size)
 120     {
 121         ISAM_P *new_isam_p_buf;
 122 #ifdef TERM_COUNT
 123         int *new_term_no;
 124 #endif
 125         p->isam_p_size = 2*p->isam_p_size + 100;
 126         new_isam_p_buf = (ISAM_P *) xmalloc(sizeof(*new_isam_p_buf) *
 127                                             p->isam_p_size);
 128         if (p->isam_p_buf)
 129         {
 130             memcpy(new_isam_p_buf, p->isam_p_buf,
 131                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 132             xfree(p->isam_p_buf);
 133         }
 134         p->isam_p_buf = new_isam_p_buf;
 135
 136 #ifdef TERM_COUNT
 137         new_term_no = (int *) xmalloc(sizeof(*new_term_no) * p->isam_p_size);
 138         if (p->term_no)
 139         {
 140             memcpy(new_term_no, p->isam_p_buf,
 141                     p->isam_p_indx * sizeof(*p->term_no));
 142             xfree(p->term_no);
 143         }
 144         p->term_no = new_term_no;
 145 #endif
 146     }
 147     assert(*info == sizeof(*p->isam_p_buf));
 148     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 149
 150 #if 1
 151     if (p->termset)
 152     {
 153         const char *db;
 154         char term_tmp[IT_MAX_WORD];
 155         int ord = 0;
 156         const char *index_name;
 157         int len = key_SU_decode (&ord, (const unsigned char *) name);
 158
 159         zebra_term_untrans  (p->zh, p->reg_type, term_tmp, name+len+1);
 160         yaz_log(log_level_rpn, "grep: %d %c %s", ord, name[len], term_tmp);
 161         zebraExplain_lookup_ord(p->zh->reg->zei,
 162                                 ord, 0 /* index_type */, &db, &index_name);
 163         yaz_log(log_level_rpn, "grep:  db=%s index=%s", db, index_name);
 164
 165         resultSetAddTerm(p->zh, p->termset, name[len], db,
 166                          index_name, term_tmp);
 167     }
 168 #endif
 169     (p->isam_p_indx)++;
 170 }
 171
 172 static int grep_handle(char *name, const char *info, void *p)
 173 {
 174     add_isam_p(name, info, (struct grep_info *) p);
 175     return 0;
 176 }
 177
 178 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 179                     const char *ct1, const char *ct2, int first)
 180 {
 181     const char *s1, *s0 = *src;
 182     const char **map;
 183
 184     /* skip white space */
 185     while (*s0)
 186     {
 187         if (ct1 && strchr(ct1, *s0))
 188             break;
 189         if (ct2 && strchr(ct2, *s0))
 190             break;
 191         s1 = s0;
 192         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 193         if (**map != *CHR_SPACE)
 194             break;
 195         s0 = s1;
 196     }
 197     *src = s0;
 198     return *s0;
 199 }
 200
 201
 202 static void esc_str(char *out_buf, int out_size,
 203                     const char *in_buf, int in_size)
 204 {
 205     int k;
 206
 207     assert(out_buf);
 208     assert(in_buf);
 209     assert(out_size > 20);
 210     *out_buf = '\0';
 211     for (k = 0; k<in_size; k++)
 212     {
 213         int c = in_buf[k] & 0xff;
 214         int pc;
 215         if (c < 32 || c > 126)
 216             pc = '?';
 217         else
 218             pc = c;
 219         sprintf(out_buf +strlen(out_buf), "%02X:%c  ", c, pc);
 220         if (strlen(out_buf) > out_size-20)
 221         {
 222             strcat(out_buf, "..");
 223             break;
 224         }
 225     }
 226 }
 227
 228 #define REGEX_CHARS " []()|.*+?!"
 229
 230 /* term_100: handle term, where trunc = none(no operators at all) */
 231 static int term_100(ZebraMaps zebra_maps, int reg_type,
 232                     const char **src, char *dst, int space_split,
 233                     char *dst_term)
 234 {
 235     const char *s0;
 236     const char **map;
 237     int i = 0;
 238     int j = 0;
 239
 240     const char *space_start = 0;
 241     const char *space_end = 0;
 242
 243     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 244         return 0;
 245     s0 = *src;
 246     while (*s0)
 247     {
 248         const char *s1 = s0;
 249         int q_map_match = 0;
 250         map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 251                                 &q_map_match);
 252         if (space_split)
 253         {
 254             if (**map == *CHR_SPACE)
 255                 break;
 256         }
 257         else  /* complete subfield only. */
 258         {
 259             if (**map == *CHR_SPACE)
 260             {   /* save space mapping for later  .. */
 261                 space_start = s1;
 262                 space_end = s0;
 263                 continue;
 264             }
 265             else if (space_start)
 266             {   /* reload last space */
 267                 while (space_start < space_end)
 268                 {
 269                     if (strchr(REGEX_CHARS, *space_start))
 270                         dst[i++] = '\\';
 271                     dst_term[j++] = *space_start;
 272                     dst[i++] = *space_start++;
 273                 }
 274                 /* and reset */
 275                 space_start = space_end = 0;
 276             }
 277         }
 278         /* add non-space char */
 279         memcpy(dst_term+j, s1, s0 - s1);
 280         j += (s0 - s1);
 281         if (!q_map_match)
 282         {
 283             while (s1 < s0)
 284             {
 285                 if (strchr(REGEX_CHARS, *s1))
 286                     dst[i++] = '\\';
 287                 dst[i++] = *s1++;
 288             }
 289         }
 290         else
 291         {
 292             char tmpbuf[80];
 293             esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 294
 295             strcpy(dst + i, map[0]);
 296             i += strlen(map[0]);
 297         }
 298     }
 299     dst[i] = '\0';
 300     dst_term[j] = '\0';
 301     *src = s0;
 302     return i;
 303 }
 304
 305 /* term_101: handle term, where trunc = Process # */
 306 static int term_101(ZebraMaps zebra_maps, int reg_type,
 307                     const char **src, char *dst, int space_split,
 308                     char *dst_term)
 309 {
 310     const char *s0;
 311     const char **map;
 312     int i = 0;
 313     int j = 0;
 314
 315     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 316         return 0;
 317     s0 = *src;
 318     while (*s0)
 319     {
 320         if (*s0 == '#')
 321         {
 322             dst[i++] = '.';
 323             dst[i++] = '*';
 324             dst_term[j++] = *s0++;
 325         }
 326         else
 327         {
 328             const char *s1 = s0;
 329             int q_map_match = 0;
 330             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 331                                     &q_map_match);
 332             if (space_split && **map == *CHR_SPACE)
 333                 break;
 334
 335             /* add non-space char */
 336             memcpy(dst_term+j, s1, s0 - s1);
 337             j += (s0 - s1);
 338             if (!q_map_match)
 339             {
 340                 while (s1 < s0)
 341                 {
 342                     if (strchr(REGEX_CHARS, *s1))
 343                         dst[i++] = '\\';
 344                     dst[i++] = *s1++;
 345                 }
 346             }
 347             else
 348             {
 349                 char tmpbuf[80];
 350                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 351
 352                 strcpy(dst + i, map[0]);
 353                 i += strlen(map[0]);
 354             }
 355         }
 356     }
 357     dst[i] = '\0';
 358     dst_term[j++] = '\0';
 359     *src = s0;
 360     return i;
 361 }
 362
 363 /* term_103: handle term, where trunc = re-2 (regular expressions) */
 364 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 365                     char *dst, int *errors, int space_split,
 366                     char *dst_term)
 367 {
 368     int i = 0;
 369     int j = 0;
 370     const char *s0;
 371     const char **map;
 372
 373     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 374         return 0;
 375     s0 = *src;
 376     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 377         isdigit(((const unsigned char *)s0)[1]))
 378     {
 379         *errors = s0[1] - '0';
 380         s0 += 3;
 381         if (*errors > 3)
 382             *errors = 3;
 383     }
 384     while (*s0)
 385     {
 386         if (strchr("^\\()[].*+?|-", *s0))
 387         {
 388             dst_term[j++] = *s0;
 389             dst[i++] = *s0++;
 390         }
 391         else
 392         {
 393             const char *s1 = s0;
 394             int q_map_match = 0;
 395             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 396                                     &q_map_match);
 397             if (space_split && **map == *CHR_SPACE)
 398                 break;
 399
 400             /* add non-space char */
 401             memcpy(dst_term+j, s1, s0 - s1);
 402             j += (s0 - s1);
 403             if (!q_map_match)
 404             {
 405                 while (s1 < s0)
 406                 {
 407                     if (strchr(REGEX_CHARS, *s1))
 408                         dst[i++] = '\\';
 409                     dst[i++] = *s1++;
 410                 }
 411             }
 412             else
 413             {
 414                 char tmpbuf[80];
 415                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 416
 417                 strcpy(dst + i, map[0]);
 418                 i += strlen(map[0]);
 419             }
 420         }
 421     }
 422     dst[i] = '\0';
 423     dst_term[j] = '\0';
 424     *src = s0;
 425
 426     return i;
 427 }
 428
 429 /* term_103: handle term, where trunc = re-1 (regular expressions) */
 430 static int term_102(ZebraMaps zebra_maps, int reg_type, const char **src,
 431                     char *dst, int space_split, char *dst_term)
 432 {
 433     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 434                     dst_term);
 435 }
 436
 437
 438 /* term_104: handle term, where trunc = Process # and ! */
 439 static int term_104(ZebraMaps zebra_maps, int reg_type,
 440                     const char **src, char *dst, int space_split,
 441                     char *dst_term)
 442 {
 443     const char *s0;
 444     const char **map;
 445     int i = 0;
 446     int j = 0;
 447
 448     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 449         return 0;
 450     s0 = *src;
 451     while (*s0)
 452     {
 453         if (*s0 == '?')
 454         {
 455             dst_term[j++] = *s0++;
 456             if (*s0 >= '0' && *s0 <= '9')
 457             {
 458                 int limit = 0;
 459                 while (*s0 >= '0' && *s0 <= '9')
 460                 {
 461                     limit = limit * 10 + (*s0 - '0');
 462                     dst_term[j++] = *s0++;
 463                 }
 464                 if (limit > 20)
 465                     limit = 20;
 466                 while (--limit >= 0)
 467                 {
 468                     dst[i++] = '.';
 469                     dst[i++] = '?';
 470                 }
 471             }
 472             else
 473             {
 474                 dst[i++] = '.';
 475                 dst[i++] = '*';
 476             }
 477         }
 478         else if (*s0 == '*')
 479         {
 480             dst[i++] = '.';
 481             dst[i++] = '*';
 482             dst_term[j++] = *s0++;
 483         }
 484         else if (*s0 == '#')
 485         {
 486             dst[i++] = '.';
 487             dst_term[j++] = *s0++;
 488         }
 489         else
 490         {
 491             const char *s1 = s0;
 492             int q_map_match = 0;
 493             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 494                                     &q_map_match);
 495             if (space_split && **map == *CHR_SPACE)
 496                 break;
 497
 498             /* add non-space char */
 499             memcpy(dst_term+j, s1, s0 - s1);
 500             j += (s0 - s1);
 501             if (!q_map_match)
 502             {
 503                 while (s1 < s0)
 504                 {
 505                     if (strchr(REGEX_CHARS, *s1))
 506                         dst[i++] = '\\';
 507                     dst[i++] = *s1++;
 508                 }
 509             }
 510             else
 511             {
 512                 char tmpbuf[80];
 513                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 514
 515                 strcpy(dst + i, map[0]);
 516                 i += strlen(map[0]);
 517             }
 518         }
 519     }
 520     dst[i] = '\0';
 521     dst_term[j++] = '\0';
 522     *src = s0;
 523     return i;
 524 }
 525
 526 /* term_105/106: handle term, where trunc = Process * and ! and right trunc */
 527 static int term_105(ZebraMaps zebra_maps, int reg_type,
 528                     const char **src, char *dst, int space_split,
 529                     char *dst_term, int right_truncate)
 530 {
 531     const char *s0;
 532     const char **map;
 533     int i = 0;
 534     int j = 0;
 535
 536     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 537         return 0;
 538     s0 = *src;
 539     while (*s0)
 540     {
 541         if (*s0 == '*')
 542         {
 543             dst[i++] = '.';
 544             dst[i++] = '*';
 545             dst_term[j++] = *s0++;
 546         }
 547         else if (*s0 == '!')
 548         {
 549             dst[i++] = '.';
 550             dst_term[j++] = *s0++;
 551         }
 552         else
 553         {
 554             const char *s1 = s0;
 555             int q_map_match = 0;
 556             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 557                                     &q_map_match);
 558             if (space_split && **map == *CHR_SPACE)
 559                 break;
 560
 561             /* add non-space char */
 562             memcpy(dst_term+j, s1, s0 - s1);
 563             j += (s0 - s1);
 564             if (!q_map_match)
 565             {
 566                 while (s1 < s0)
 567                 {
 568                     if (strchr(REGEX_CHARS, *s1))
 569                         dst[i++] = '\\';
 570                     dst[i++] = *s1++;
 571                 }
 572             }
 573             else
 574             {
 575                 char tmpbuf[80];
 576                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 577
 578                 strcpy(dst + i, map[0]);
 579                 i += strlen(map[0]);
 580             }
 581         }
 582     }
 583     if (right_truncate)
 584     {
 585         dst[i++] = '.';
 586         dst[i++] = '*';
 587     }
 588     dst[i] = '\0';
 589
 590     dst_term[j++] = '\0';
 591     *src = s0;
 592     return i;
 593 }
 594
 595
 596 /* gen_regular_rel - generate regular expression from relation
 597  *  val:     border value (inclusive)
 598  *  islt:    1 if <=; 0 if >=.
 599  */
 600 static void gen_regular_rel(char *dst, int val, int islt)
 601 {
 602     int dst_p;
 603     int w, d, i;
 604     int pos = 0;
 605     char numstr[20];
 606
 607     yaz_log(YLOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 608     if (val >= 0)
 609     {
 610         if (islt)
 611             strcpy(dst, "(-[0-9]+|(");
 612         else
 613             strcpy(dst, "((");
 614     }
 615     else
 616     {
 617         if (!islt)
 618         {
 619             strcpy(dst, "([0-9]+|-(");
 620             dst_p = strlen(dst);
 621             islt = 1;
 622         }
 623         else
 624         {
 625             strcpy(dst, "(-(");
 626             islt = 0;
 627         }
 628         val = -val;
 629     }
 630     dst_p = strlen(dst);
 631     sprintf(numstr, "%d", val);
 632     for (w = strlen(numstr); --w >= 0; pos++)
 633     {
 634         d = numstr[w];
 635         if (pos > 0)
 636         {
 637             if (islt)
 638             {
 639                 if (d == '0')
 640                     continue;
 641                 d--;
 642             }
 643             else
 644             {
 645                 if (d == '9')
 646                     continue;
 647                 d++;
 648             }
 649         }
 650
 651         strcpy(dst + dst_p, numstr);
 652         dst_p = strlen(dst) - pos - 1;
 653
 654         if (islt)
 655         {
 656             if (d != '0')
 657             {
 658                 dst[dst_p++] = '[';
 659                 dst[dst_p++] = '0';
 660                 dst[dst_p++] = '-';
 661                 dst[dst_p++] = d;
 662                 dst[dst_p++] = ']';
 663             }
 664             else
 665                 dst[dst_p++] = d;
 666         }
 667         else
 668         {
 669             if (d != '9')
 670             {
 671                 dst[dst_p++] = '[';
 672                 dst[dst_p++] = d;
 673                 dst[dst_p++] = '-';
 674                 dst[dst_p++] = '9';
 675                 dst[dst_p++] = ']';
 676             }
 677             else
 678                 dst[dst_p++] = d;
 679         }
 680         for (i = 0; i<pos; i++)
 681         {
 682             dst[dst_p++] = '[';
 683             dst[dst_p++] = '0';
 684             dst[dst_p++] = '-';
 685             dst[dst_p++] = '9';
 686             dst[dst_p++] = ']';
 687         }
 688         dst[dst_p++] = '|';
 689     }
 690     dst[dst_p] = '\0';
 691     if (islt)
 692     {
 693         /* match everything less than 10^(pos-1) */
 694         strcat(dst, "0*");
 695         for (i = 1; i<pos; i++)
 696             strcat(dst, "[0-9]?");
 697     }
 698     else
 699     {
 700         /* match everything greater than 10^pos */
 701         for (i = 0; i <= pos; i++)
 702             strcat(dst, "[0-9]");
 703         strcat(dst, "[0-9]*");
 704     }
 705     strcat(dst, "))");
 706 }
 707
 708 void string_rel_add_char(char **term_p, const char *src, int *indx)
 709 {
 710     if (src[*indx] == '\\')
 711         *(*term_p)++ = src[(*indx)++];
 712     *(*term_p)++ = src[(*indx)++];
 713 }
 714
 715 /*
 716  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 717  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 718  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 719  *              ([^-a].*|a[^-b].*|ab[c-].*)
 720  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 721  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 722  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 723  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 724  */
 725 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 726                            const char **term_sub, char *term_dict,
 727                            oid_value attributeSet,
 728                            int reg_type, int space_split, char *term_dst,
 729                            int *error_code)
 730 {
 731     AttrType relation;
 732     int relation_value;
 733     int i;
 734     char *term_tmp = term_dict + strlen(term_dict);
 735     char term_component[2*IT_MAX_WORD+20];
 736
 737     attr_init_APT(&relation, zapt, 2);
 738     relation_value = attr_find(&relation, NULL);
 739
 740     *error_code = 0;
 741     yaz_log(YLOG_DEBUG, "string relation value=%d", relation_value);
 742     switch (relation_value)
 743     {
 744     case 1:
 745         if (!term_100(zh->reg->zebra_maps, reg_type,
 746                       term_sub, term_component,
 747                       space_split, term_dst))
 748             return 0;
 749         yaz_log(log_level_rpn, "Relation <");
 750
 751         *term_tmp++ = '(';
 752         for (i = 0; term_component[i]; )
 753         {
 754             int j = 0;
 755
 756             if (i)
 757                 *term_tmp++ = '|';
 758             while (j < i)
 759                 string_rel_add_char(&term_tmp, term_component, &j);
 760
 761             *term_tmp++ = '[';
 762
 763             *term_tmp++ = '^';
 764             string_rel_add_char(&term_tmp, term_component, &i);
 765             *term_tmp++ = '-';
 766
 767             *term_tmp++ = ']';
 768             *term_tmp++ = '.';
 769             *term_tmp++ = '*';
 770
 771             if ((term_tmp - term_dict) > IT_MAX_WORD)
 772                 break;
 773         }
 774         *term_tmp++ = ')';
 775         *term_tmp = '\0';
 776         break;
 777     case 2:
 778         if (!term_100(zh->reg->zebra_maps, reg_type,
 779                       term_sub, term_component,
 780                       space_split, term_dst))
 781             return 0;
 782         yaz_log(log_level_rpn, "Relation <=");
 783
 784         *term_tmp++ = '(';
 785         for (i = 0; term_component[i]; )
 786         {
 787             int j = 0;
 788
 789             while (j < i)
 790                 string_rel_add_char(&term_tmp, term_component, &j);
 791             *term_tmp++ = '[';
 792
 793             *term_tmp++ = '^';
 794             string_rel_add_char(&term_tmp, term_component, &i);
 795             *term_tmp++ = '-';
 796
 797             *term_tmp++ = ']';
 798             *term_tmp++ = '.';
 799             *term_tmp++ = '*';
 800
 801             *term_tmp++ = '|';
 802
 803             if ((term_tmp - term_dict) > IT_MAX_WORD)
 804                 break;
 805         }
 806         for (i = 0; term_component[i]; )
 807             string_rel_add_char(&term_tmp, term_component, &i);
 808         *term_tmp++ = ')';
 809         *term_tmp = '\0';
 810         break;
 811     case 5:
 812         if (!term_100 (zh->reg->zebra_maps, reg_type,
 813                        term_sub, term_component, space_split, term_dst))
 814             return 0;
 815         yaz_log(log_level_rpn, "Relation >");
 816
 817         *term_tmp++ = '(';
 818         for (i = 0; term_component[i];)
 819         {
 820             int j = 0;
 821
 822             while (j < i)
 823                 string_rel_add_char(&term_tmp, term_component, &j);
 824             *term_tmp++ = '[';
 825
 826             *term_tmp++ = '^';
 827             *term_tmp++ = '-';
 828             string_rel_add_char(&term_tmp, term_component, &i);
 829
 830             *term_tmp++ = ']';
 831             *term_tmp++ = '.';
 832             *term_tmp++ = '*';
 833
 834             *term_tmp++ = '|';
 835
 836             if ((term_tmp - term_dict) > IT_MAX_WORD)
 837                 break;
 838         }
 839         for (i = 0; term_component[i];)
 840             string_rel_add_char(&term_tmp, term_component, &i);
 841         *term_tmp++ = '.';
 842         *term_tmp++ = '+';
 843         *term_tmp++ = ')';
 844         *term_tmp = '\0';
 845         break;
 846     case 4:
 847         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 848                       term_component, space_split, term_dst))
 849             return 0;
 850         yaz_log(log_level_rpn, "Relation >=");
 851
 852         *term_tmp++ = '(';
 853         for (i = 0; term_component[i];)
 854         {
 855             int j = 0;
 856
 857             if (i)
 858                 *term_tmp++ = '|';
 859             while (j < i)
 860                 string_rel_add_char(&term_tmp, term_component, &j);
 861             *term_tmp++ = '[';
 862
 863             if (term_component[i+1])
 864             {
 865                 *term_tmp++ = '^';
 866                 *term_tmp++ = '-';
 867                 string_rel_add_char(&term_tmp, term_component, &i);
 868             }
 869             else
 870             {
 871                 string_rel_add_char(&term_tmp, term_component, &i);
 872                 *term_tmp++ = '-';
 873             }
 874             *term_tmp++ = ']';
 875             *term_tmp++ = '.';
 876             *term_tmp++ = '*';
 877
 878             if ((term_tmp - term_dict) > IT_MAX_WORD)
 879                 break;
 880         }
 881         *term_tmp++ = ')';
 882         *term_tmp = '\0';
 883         break;
 884     case 3:
 885     case 102:
 886     case 103:
 887     case -1:
 888         if (!**term_sub)
 889             return 1;
 890         yaz_log(log_level_rpn, "Relation =");
 891         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 892                       term_component, space_split, term_dst))
 893             return 0;
 894         strcat(term_tmp, "(");
 895         strcat(term_tmp, term_component);
 896         strcat(term_tmp, ")");
 897         break;
 898     default:
 899         *error_code = YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
 900         return 0;
 901     }
 902     return 1;
 903 }
 904
 905 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 906                              const char **term_sub,
 907                              oid_value attributeSet, NMEM stream,
 908                              struct grep_info *grep_info,
 909                              int reg_type, int complete_flag,
 910                              int num_bases, char **basenames,
 911                              char *term_dst,
 912                              const char *xpath_use,
 913                              struct ord_list **ol);
 914
 915 static ZEBRA_RES term_limits_APT(ZebraHandle zh,
 916                                  Z_AttributesPlusTerm *zapt,
 917                                  zint *hits_limit_value,
 918                                  const char **term_ref_id_str,
 919                                  NMEM nmem)
 920 {
 921     AttrType term_ref_id_attr;
 922     AttrType hits_limit_attr;
 923     int term_ref_id_int;
 924
 925     attr_init_APT(&hits_limit_attr, zapt, 9);
 926     *hits_limit_value  = attr_find(&hits_limit_attr, NULL);
 927
 928     attr_init_APT(&term_ref_id_attr, zapt, 10);
 929     term_ref_id_int = attr_find_ex(&term_ref_id_attr, NULL, term_ref_id_str);
 930     if (term_ref_id_int >= 0)
 931     {
 932         char *res = nmem_malloc(nmem, 20);
 933         sprintf(res, "%d", term_ref_id_int);
 934         *term_ref_id_str = res;
 935     }
 936
 937     /* no limit given ? */
 938     if (*hits_limit_value == -1)
 939     {
 940         if (*term_ref_id_str)
 941         {
 942             /* use global if term_ref is present */
 943             *hits_limit_value = zh->approx_limit;
 944         }
 945         else
 946         {
 947             /* no counting if term_ref is not present */
 948             *hits_limit_value = 0;
 949         }
 950     }
 951     else if (*hits_limit_value == 0)
 952     {
 953         /* 0 is the same as global limit */
 954         *hits_limit_value = zh->approx_limit;
 955     }
 956     yaz_log(YLOG_DEBUG, "term_limits_APT ref_id=%s limit=" ZINT_FORMAT,
 957             *term_ref_id_str ? *term_ref_id_str : "none",
 958             *hits_limit_value);
 959     return ZEBRA_OK;
 960 }
 961
 962 static ZEBRA_RES term_trunc(ZebraHandle zh,
 963                             Z_AttributesPlusTerm *zapt,
 964                             const char **term_sub,
 965                             oid_value attributeSet, NMEM stream,
 966                             struct grep_info *grep_info,
 967                             int reg_type, int complete_flag,
 968                             int num_bases, char **basenames,
 969                             char *term_dst,
 970                             const char *rank_type,
 971                             const char *xpath_use,
 972                             NMEM rset_nmem,
 973                             RSET *rset,
 974                             struct rset_key_control *kc)
 975 {
 976     ZEBRA_RES res;
 977     struct ord_list *ol;
 978     zint hits_limit_value;
 979     const char *term_ref_id_str = 0;
 980     *rset = 0;
 981
 982     term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str,
 983                     stream);
 984     grep_info->isam_p_indx = 0;
 985     res = string_term(zh, zapt, term_sub, attributeSet, stream, grep_info,
 986                       reg_type, complete_flag, num_bases, basenames,
 987                       term_dst, xpath_use, &ol);
 988     if (res != ZEBRA_OK)
 989         return res;
 990     if (!*term_sub)  /* no more terms ? */
 991         return res;
 992     yaz_log(log_level_rpn, "term: %s", term_dst);
 993     *rset = rset_trunc(zh, grep_info->isam_p_buf,
 994                        grep_info->isam_p_indx, term_dst,
 995                        strlen(term_dst), rank_type, 1 /* preserve pos */,
 996                        zapt->term->which, rset_nmem,
 997                        kc, kc->scope, ol, reg_type, hits_limit_value,
 998                        term_ref_id_str);
 999     if (!*rset)
1000         return ZEBRA_FAIL;
1001     return ZEBRA_OK;
1002 }
1003
1004 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1005                              const char **term_sub,
1006                              oid_value attributeSet, NMEM stream,
1007                              struct grep_info *grep_info,
1008                              int reg_type, int complete_flag,
1009                              int num_bases, char **basenames,
1010                              char *term_dst,
1011                              const char *xpath_use,
1012                              struct ord_list **ol)
1013 {
1014     char term_dict[2*IT_MAX_WORD+4000];
1015     int j, r, base_no;
1016     AttrType truncation;
1017     int truncation_value;
1018     oid_value curAttributeSet = attributeSet;
1019     const char *termp;
1020     struct rpn_char_map_info rcmi;
1021     int space_split = complete_flag ? 0 : 1;
1022
1023     int bases_ok = 0;     /* no of databases with OK attribute */
1024
1025     *ol = ord_list_create(stream);
1026
1027     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1028     attr_init_APT(&truncation, zapt, 5);
1029     truncation_value = attr_find(&truncation, NULL);
1030     yaz_log(log_level_rpn, "truncation value %d", truncation_value);
1031
1032     for (base_no = 0; base_no < num_bases; base_no++)
1033     {
1034         int ord = -1;
1035         int attr_ok = 0;
1036         int regex_range = 0;
1037         int init_pos = 0;
1038 #if 0
1039         attent attp;
1040         data1_local_attribute id_xpath_attr;
1041         data1_local_attribute *local_attr;
1042 #endif
1043         int max_pos, prefix_len = 0;
1044         int relation_error;
1045         char ord_buf[32];
1046         int ord_len, i;
1047
1048         termp = *term_sub;
1049
1050         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1051         {
1052             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1053                            basenames[base_no]);
1054             return ZEBRA_FAIL;
1055         }
1056
1057         if (zebra_apt_get_ord(zh, zapt, reg_type, xpath_use,
1058                               curAttributeSet, &ord)
1059             != ZEBRA_OK)
1060         {
1061             break;
1062         }
1063         *ol = ord_list_append(stream, *ol, ord);
1064
1065         if (prefix_len)
1066             term_dict[prefix_len++] = '|';
1067         else
1068             term_dict[prefix_len++] = '(';
1069
1070         ord_len = key_SU_encode (ord, ord_buf);
1071         for (i = 0; i<ord_len; i++)
1072         {
1073             term_dict[prefix_len++] = 1;
1074                 term_dict[prefix_len++] = ord_buf[i];
1075         }
1076         if (ord_len > init_pos)
1077             init_pos = ord_len;
1078
1079         bases_ok++;
1080         if (prefix_len)
1081             attr_ok = 1;
1082
1083         term_dict[prefix_len++] = ')';
1084         term_dict[prefix_len] = '\0';
1085         j = prefix_len;
1086         switch (truncation_value)
1087         {
1088         case -1:         /* not specified */
1089         case 100:        /* do not truncate */
1090             if (!string_relation (zh, zapt, &termp, term_dict,
1091                                   attributeSet,
1092                                   reg_type, space_split, term_dst,
1093                                   &relation_error))
1094             {
1095                 if (relation_error)
1096                 {
1097                     zebra_setError(zh, relation_error, 0);
1098                     return ZEBRA_FAIL;
1099                 }
1100                 *term_sub = 0;
1101                 return ZEBRA_OK;
1102             }
1103             break;
1104         case 1:          /* right truncation */
1105             term_dict[j++] = '(';
1106             if (!term_100(zh->reg->zebra_maps, reg_type,
1107                           &termp, term_dict + j, space_split, term_dst))
1108             {
1109                 *term_sub = 0;
1110                 return ZEBRA_OK;
1111             }
1112             strcat(term_dict, ".*)");
1113             break;
1114         case 2:          /* keft truncation */
1115             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1116             if (!term_100(zh->reg->zebra_maps, reg_type,
1117                           &termp, term_dict + j, space_split, term_dst))
1118             {
1119                 *term_sub = 0;
1120                 return ZEBRA_OK;
1121             }
1122             strcat(term_dict, ")");
1123             break;
1124         case 3:          /* left&right truncation */
1125             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1126             if (!term_100(zh->reg->zebra_maps, reg_type,
1127                           &termp, term_dict + j, space_split, term_dst))
1128             {
1129                 *term_sub = 0;
1130                 return ZEBRA_OK;
1131             }
1132             strcat(term_dict, ".*)");
1133             break;
1134         case 101:        /* process # in term */
1135             term_dict[j++] = '(';
1136             if (!term_101(zh->reg->zebra_maps, reg_type,
1137                           &termp, term_dict + j, space_split, term_dst))
1138             {
1139                 *term_sub = 0;
1140                 return ZEBRA_OK;
1141             }
1142             strcat(term_dict, ")");
1143             break;
1144         case 102:        /* Regexp-1 */
1145             term_dict[j++] = '(';
1146             if (!term_102(zh->reg->zebra_maps, reg_type,
1147                           &termp, term_dict + j, space_split, term_dst))
1148             {
1149                 *term_sub = 0;
1150                 return ZEBRA_OK;
1151             }
1152             strcat(term_dict, ")");
1153             break;
1154         case 103:       /* Regexp-2 */
1155             regex_range = 1;
1156             term_dict[j++] = '(';
1157             if (!term_103(zh->reg->zebra_maps, reg_type,
1158                           &termp, term_dict + j, &regex_range,
1159                           space_split, term_dst))
1160             {
1161                 *term_sub = 0;
1162                 return ZEBRA_OK;
1163             }
1164             strcat(term_dict, ")");
1165             break;
1166         case 104:        /* process # and ! in term */
1167             term_dict[j++] = '(';
1168             if (!term_104(zh->reg->zebra_maps, reg_type,
1169                           &termp, term_dict + j, space_split, term_dst))
1170             {
1171                 *term_sub = 0;
1172                 return ZEBRA_OK;
1173             }
1174             strcat(term_dict, ")");
1175             break;
1176         case 105:        /* process * and ! in term */
1177             term_dict[j++] = '(';
1178             if (!term_105(zh->reg->zebra_maps, reg_type,
1179                           &termp, term_dict + j, space_split, term_dst, 1))
1180             {
1181                 *term_sub = 0;
1182                 return ZEBRA_OK;
1183             }
1184             strcat(term_dict, ")");
1185             break;
1186         case 106:        /* process * and ! in term */
1187             term_dict[j++] = '(';
1188             if (!term_105(zh->reg->zebra_maps, reg_type,
1189                           &termp, term_dict + j, space_split, term_dst, 0))
1190             {
1191                 *term_sub = 0;
1192                 return ZEBRA_OK;
1193             }
1194             strcat(term_dict, ")");
1195             break;
1196         default:
1197             zebra_setError_zint(zh,
1198                                 YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE,
1199                                 truncation_value);
1200             return ZEBRA_FAIL;
1201         }
1202         if (attr_ok)
1203         {
1204             char buf[80];
1205             const char *input = term_dict + prefix_len;
1206             esc_str(buf, sizeof(buf), input, strlen(input));
1207         }
1208         if (attr_ok)
1209         {
1210             yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_dict+prefix_len);
1211             r = dict_lookup_grep(zh->reg->dict, term_dict, regex_range,
1212                                  grep_info, &max_pos, init_pos,
1213                                  grep_handle);
1214             if (r)
1215                 yaz_log(YLOG_WARN, "dict_lookup_grep fail %d", r);
1216         }
1217     }
1218     if (!bases_ok)
1219         return ZEBRA_FAIL;
1220     *term_sub = termp;
1221     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1222     return ZEBRA_OK;
1223 }
1224
1225
1226 /* convert APT search term to UTF8 */
1227 static ZEBRA_RES zapt_term_to_utf8(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1228                                    char *termz)
1229 {
1230     size_t sizez;
1231     Z_Term *term = zapt->term;
1232
1233     switch (term->which)
1234     {
1235     case Z_Term_general:
1236         if (zh->iconv_to_utf8 != 0)
1237         {
1238             char *inbuf = (char *) term->u.general->buf;
1239             size_t inleft = term->u.general->len;
1240             char *outbuf = termz;
1241             size_t outleft = IT_MAX_WORD-1;
1242             size_t ret;
1243
1244             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1245                         &outbuf, &outleft);
1246             if (ret == (size_t)(-1))
1247             {
1248                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1249                 zebra_setError(
1250                     zh,
1251                     YAZ_BIB1_QUERY_TERM_INCLUDES_CHARS_THAT_DO_NOT_TRANSLATE_INTO_,
1252                     0);
1253                 return ZEBRA_FAIL;
1254             }
1255             *outbuf = 0;
1256         }
1257         else
1258         {
1259             sizez = term->u.general->len;
1260             if (sizez > IT_MAX_WORD-1)
1261                 sizez = IT_MAX_WORD-1;
1262             memcpy (termz, term->u.general->buf, sizez);
1263             termz[sizez] = '\0';
1264         }
1265         break;
1266     case Z_Term_characterString:
1267         sizez = strlen(term->u.characterString);
1268         if (sizez > IT_MAX_WORD-1)
1269             sizez = IT_MAX_WORD-1;
1270         memcpy (termz, term->u.characterString, sizez);
1271         termz[sizez] = '\0';
1272         break;
1273     default:
1274         zebra_setError(zh, YAZ_BIB1_UNSUPP_CODED_VALUE_FOR_TERM, 0);
1275         return ZEBRA_FAIL;
1276     }
1277     return ZEBRA_OK;
1278 }
1279
1280 /* convert APT SCAN term to internal cmap */
1281 static ZEBRA_RES trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1282                                  char *termz, int reg_type)
1283 {
1284     char termz0[IT_MAX_WORD];
1285
1286     if (zapt_term_to_utf8(zh, zapt, termz0) == ZEBRA_FAIL)
1287         return ZEBRA_FAIL;    /* error */
1288     else
1289     {
1290         const char **map;
1291         const char *cp = (const char *) termz0;
1292         const char *cp_end = cp + strlen(cp);
1293         const char *src;
1294         int i = 0;
1295         const char *space_map = NULL;
1296         int len;
1297
1298         while ((len = (cp_end - cp)) > 0)
1299         {
1300             map = zebra_maps_input(zh->reg->zebra_maps, reg_type, &cp, len, 0);
1301             if (**map == *CHR_SPACE)
1302                 space_map = *map;
1303             else
1304             {
1305                 if (i && space_map)
1306                     for (src = space_map; *src; src++)
1307                         termz[i++] = *src;
1308                 space_map = NULL;
1309                 for (src = *map; *src; src++)
1310                     termz[i++] = *src;
1311             }
1312         }
1313         termz[i] = '\0';
1314     }
1315     return ZEBRA_OK;
1316 }
1317
1318 static void grep_info_delete(struct grep_info *grep_info)
1319 {
1320 #ifdef TERM_COUNT
1321     xfree(grep_info->term_no);
1322 #endif
1323     xfree(grep_info->isam_p_buf);
1324 }
1325
1326 static ZEBRA_RES grep_info_prepare(ZebraHandle zh,
1327                                    Z_AttributesPlusTerm *zapt,
1328                                    struct grep_info *grep_info,
1329                                    int reg_type)
1330 {
1331     AttrType termset;
1332     int termset_value_numeric;
1333     const char *termset_value_string;
1334
1335 #ifdef TERM_COUNT
1336     grep_info->term_no = 0;
1337 #endif
1338     grep_info->isam_p_size = 0;
1339     grep_info->isam_p_buf = NULL;
1340     grep_info->zh = zh;
1341     grep_info->reg_type = reg_type;
1342     grep_info->termset = 0;
1343
1344     if (!zapt)
1345         return ZEBRA_OK;
1346     attr_init_APT(&termset, zapt, 8);
1347     termset_value_numeric =
1348         attr_find_ex(&termset, NULL, &termset_value_string);
1349     if (termset_value_numeric != -1)
1350     {
1351         char resname[32];
1352         const char *termset_name = 0;
1353         if (termset_value_numeric != -2)
1354         {
1355
1356             sprintf(resname, "%d", termset_value_numeric);
1357             termset_name = resname;
1358         }
1359         else
1360             termset_name = termset_value_string;
1361         yaz_log(log_level_rpn, "creating termset set %s", termset_name);
1362         grep_info->termset = resultSetAdd(zh, termset_name, 1);
1363         if (!grep_info->termset)
1364         {
1365             zebra_setError(zh, YAZ_BIB1_ILLEGAL_RESULT_SET_NAME, termset_name);
1366             return ZEBRA_FAIL;
1367         }
1368     }
1369     return ZEBRA_OK;
1370 }
1371
1372 /**
1373   \brief Create result set(s) for list of terms
1374   \param zh Zebra Handle
1375   \param termz term as used in query but converted to UTF-8
1376   \param attributeSet default attribute set
1377   \param stream memory for result
1378   \param reg_type register type ('w', 'p',..)
1379   \param complete_flag whether it's phrases or not
1380   \param rank_type term flags for ranking
1381   \param xpath_use use attribute for X-Path (-1 for no X-path)
1382   \param num_bases number of databases
1383   \param basenames array of databases
1384   \param rset_mem memory for result sets
1385   \param result_sets output result set for each term in list (output)
1386   \param number number of output result sets
1387   \param kc rset key control to be used for created result sets
1388 */
1389 static ZEBRA_RES term_list_trunc(ZebraHandle zh,
1390                                  Z_AttributesPlusTerm *zapt,
1391                                  const char *termz,
1392                                  oid_value attributeSet,
1393                                  NMEM stream,
1394                                  int reg_type, int complete_flag,
1395                                  const char *rank_type,
1396                                  const char *xpath_use,
1397                                  int num_bases, char **basenames,
1398                                  NMEM rset_nmem,
1399                                  RSET **result_sets, int *num_result_sets,
1400                                  struct rset_key_control *kc)
1401 {
1402     char term_dst[IT_MAX_WORD+1];
1403     struct grep_info grep_info;
1404     const char *termp = termz;
1405     int alloc_sets = 0;
1406     int empty_term = *termz ? 0 : 1;
1407
1408     empty_term = 0;
1409     *num_result_sets = 0;
1410     *term_dst = 0;
1411     if (grep_info_prepare(zh, zapt, &grep_info, reg_type) == ZEBRA_FAIL)
1412         return ZEBRA_FAIL;
1413     while(1)
1414     {
1415         ZEBRA_RES res;
1416
1417         if (alloc_sets == *num_result_sets)
1418         {
1419             int add = 10;
1420             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1421                                               sizeof(*rnew));
1422             if (alloc_sets)
1423                 memcpy(rnew, *result_sets, alloc_sets * sizeof(*rnew));
1424             alloc_sets = alloc_sets + add;
1425             *result_sets = rnew;
1426         }
1427         res = term_trunc(zh, zapt, &termp, attributeSet,
1428                          stream, &grep_info,
1429                          reg_type, complete_flag,
1430                          num_bases, basenames,
1431                          term_dst, rank_type,
1432                          xpath_use, rset_nmem,
1433                          &(*result_sets)[*num_result_sets],
1434                          kc);
1435         if (res != ZEBRA_OK)
1436         {
1437             int i;
1438             for (i = 0; i < *num_result_sets; i++)
1439                 rset_delete((*result_sets)[i]);
1440             grep_info_delete (&grep_info);
1441             return res;
1442         }
1443         if ((*result_sets)[*num_result_sets] == 0)
1444             break;
1445         (*num_result_sets)++;
1446
1447         if (empty_term)
1448             break;
1449         if (!*termp)
1450             break;
1451     }
1452     grep_info_delete(&grep_info);
1453     return ZEBRA_OK;
1454 }
1455
1456 static ZEBRA_RES rpn_search_APT_phrase(ZebraHandle zh,
1457                                        Z_AttributesPlusTerm *zapt,
1458                                        const char *termz_org,
1459                                        oid_value attributeSet,
1460                                        NMEM stream,
1461                                        int reg_type, int complete_flag,
1462                                        const char *rank_type,
1463                                        const char *xpath_use,
1464                                        int num_bases, char **basenames,
1465                                        NMEM rset_nmem,
1466                                        RSET *rset,
1467                                        struct rset_key_control *kc)
1468 {
1469     RSET *result_sets = 0;
1470     int num_result_sets = 0;
1471     ZEBRA_RES res =
1472         term_list_trunc(zh, zapt, termz_org, attributeSet,
1473                         stream, reg_type, complete_flag,
1474                         rank_type, xpath_use,
1475                         num_bases, basenames,
1476                         rset_nmem,
1477                         &result_sets, &num_result_sets, kc);
1478     if (res != ZEBRA_OK)
1479         return res;
1480     if (num_result_sets == 0)
1481         *rset = rsnull_create (rset_nmem, kc, 0);
1482     else if (num_result_sets == 1)
1483         *rset = result_sets[0];
1484     else
1485         *rset = rsprox_create(rset_nmem, kc, kc->scope,
1486                               num_result_sets, result_sets,
1487                               1 /* ordered */, 0 /* exclusion */,
1488                               3 /* relation */, 1 /* distance */);
1489     if (!*rset)
1490         return ZEBRA_FAIL;
1491     return ZEBRA_OK;
1492 }
1493
1494 static ZEBRA_RES rpn_search_APT_or_list(ZebraHandle zh,
1495                                         Z_AttributesPlusTerm *zapt,
1496                                         const char *termz_org,
1497                                         oid_value attributeSet,
1498                                         NMEM stream,
1499                                         int reg_type, int complete_flag,
1500                                         const char *rank_type,
1501                                         const char *xpath_use,
1502                                         int num_bases, char **basenames,
1503                                         NMEM rset_nmem,
1504                                         RSET *rset,
1505                                         struct rset_key_control *kc)
1506 {
1507     RSET *result_sets = 0;
1508     int num_result_sets = 0;
1509     ZEBRA_RES res =
1510         term_list_trunc(zh, zapt, termz_org, attributeSet,
1511                         stream, reg_type, complete_flag,
1512                         rank_type, xpath_use,
1513                         num_bases, basenames,
1514                         rset_nmem,
1515                         &result_sets, &num_result_sets, kc);
1516     if (res != ZEBRA_OK)
1517         return res;
1518     if (num_result_sets == 0)
1519         *rset = rsnull_create (rset_nmem, kc, 0);
1520     else if (num_result_sets == 1)
1521         *rset = result_sets[0];
1522     else
1523         *rset = rsmulti_or_create(rset_nmem, kc, kc->scope, 0 /* termid */,
1524                                   num_result_sets, result_sets);
1525     if (!*rset)
1526         return ZEBRA_FAIL;
1527     return ZEBRA_OK;
1528 }
1529
1530 static ZEBRA_RES rpn_search_APT_and_list(ZebraHandle zh,
1531                                          Z_AttributesPlusTerm *zapt,
1532                                          const char *termz_org,
1533                                          oid_value attributeSet,
1534                                          NMEM stream,
1535                                          int reg_type, int complete_flag,
1536                                          const char *rank_type,
1537                                          const char *xpath_use,
1538                                          int num_bases, char **basenames,
1539                                          NMEM rset_nmem,
1540                                          RSET *rset,
1541                                          struct rset_key_control *kc)
1542 {
1543     RSET *result_sets = 0;
1544     int num_result_sets = 0;
1545     ZEBRA_RES res =
1546         term_list_trunc(zh, zapt, termz_org, attributeSet,
1547                         stream, reg_type, complete_flag,
1548                         rank_type, xpath_use,
1549                         num_bases, basenames,
1550                         rset_nmem,
1551                         &result_sets, &num_result_sets,
1552                         kc);
1553     if (res != ZEBRA_OK)
1554         return res;
1555     if (num_result_sets == 0)
1556         *rset = rsnull_create (rset_nmem, kc, 0);
1557     else if (num_result_sets == 1)
1558         *rset = result_sets[0];
1559     else
1560         *rset = rsmulti_and_create(rset_nmem, kc, kc->scope,
1561                                    num_result_sets, result_sets);
1562     if (!*rset)
1563         return ZEBRA_FAIL;
1564     return ZEBRA_OK;
1565 }
1566
1567 static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1568                             const char **term_sub,
1569                             char *term_dict,
1570                             oid_value attributeSet,
1571                             struct grep_info *grep_info,
1572                             int *max_pos,
1573                             int reg_type,
1574                             char *term_dst,
1575                             int *error_code)
1576 {
1577     AttrType relation;
1578     int relation_value;
1579     int term_value;
1580     int r;
1581     char *term_tmp = term_dict + strlen(term_dict);
1582
1583     *error_code = 0;
1584     attr_init_APT(&relation, zapt, 2);
1585     relation_value = attr_find(&relation, NULL);
1586
1587     yaz_log(log_level_rpn, "numeric relation value=%d", relation_value);
1588
1589     if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1590                   term_dst))
1591         return 0;
1592     term_value = atoi (term_tmp);
1593     switch (relation_value)
1594     {
1595     case 1:
1596         yaz_log(log_level_rpn, "Relation <");
1597         gen_regular_rel(term_tmp, term_value-1, 1);
1598         break;
1599     case 2:
1600         yaz_log(log_level_rpn, "Relation <=");
1601         gen_regular_rel(term_tmp, term_value, 1);
1602         break;
1603     case 4:
1604         yaz_log(log_level_rpn, "Relation >=");
1605         gen_regular_rel(term_tmp, term_value, 0);
1606         break;
1607     case 5:
1608         yaz_log(log_level_rpn, "Relation >");
1609         gen_regular_rel(term_tmp, term_value+1, 0);
1610         break;
1611     case -1:
1612     case 3:
1613         yaz_log(log_level_rpn, "Relation =");
1614         sprintf(term_tmp, "(0*%d)", term_value);
1615         break;
1616     default:
1617         *error_code = YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
1618         return 0;
1619     }
1620     yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_tmp);
1621     r = dict_lookup_grep(zh->reg->dict, term_dict, 0, grep_info, max_pos,
1622                           0, grep_handle);
1623     if (r)
1624         yaz_log(YLOG_WARN, "dict_lookup_grep fail, rel = gt: %d", r);
1625     yaz_log(log_level_rpn, "%d positions", grep_info->isam_p_indx);
1626     return 1;
1627 }
1628
1629 static ZEBRA_RES numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1630                               const char **term_sub,
1631                               oid_value attributeSet,
1632                               struct grep_info *grep_info,
1633                               int reg_type, int complete_flag,
1634                               int num_bases, char **basenames,
1635                               char *term_dst,
1636                               const char *xpath_use,
1637                               NMEM stream)
1638 {
1639     char term_dict[2*IT_MAX_WORD+2];
1640     int base_no;
1641     oid_value curAttributeSet = attributeSet;
1642     const char *termp;
1643     struct rpn_char_map_info rcmi;
1644
1645     int bases_ok = 0;     /* no of databases with OK attribute */
1646
1647     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1648
1649     for (base_no = 0; base_no < num_bases; base_no++)
1650     {
1651         int max_pos, prefix_len = 0;
1652         int relation_error = 0;
1653         int ord, ord_len, i;
1654         char ord_buf[32];
1655
1656         termp = *term_sub;
1657
1658         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1659         {
1660             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1661                            basenames[base_no]);
1662             return ZEBRA_FAIL;
1663         }
1664
1665         if (zebra_apt_get_ord(zh, zapt, reg_type, xpath_use,
1666                               curAttributeSet, &ord)
1667             != ZEBRA_OK)
1668         {
1669             break;
1670         }
1671
1672         if (prefix_len)
1673             term_dict[prefix_len++] = '|';
1674         else
1675             term_dict[prefix_len++] = '(';
1676
1677         ord_len = key_SU_encode (ord, ord_buf);
1678         for (i = 0; i < ord_len; i++)
1679         {
1680             term_dict[prefix_len++] = 1;
1681                 term_dict[prefix_len++] = ord_buf[i];
1682         }
1683         bases_ok++;
1684         term_dict[prefix_len++] = ')';
1685         term_dict[prefix_len] = '\0';
1686         if (!numeric_relation(zh, zapt, &termp, term_dict,
1687                               attributeSet, grep_info, &max_pos, reg_type,
1688                               term_dst, &relation_error))
1689         {
1690             if (relation_error)
1691             {
1692                 zebra_setError(zh, relation_error, 0);
1693                 return ZEBRA_FAIL;
1694             }
1695             *term_sub = 0;
1696             return ZEBRA_OK;
1697         }
1698     }
1699     if (!bases_ok)
1700         return ZEBRA_FAIL;
1701     *term_sub = termp;
1702     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1703     return ZEBRA_OK;
1704 }
1705
1706
1707 static ZEBRA_RES rpn_search_APT_numeric(ZebraHandle zh,
1708                                         Z_AttributesPlusTerm *zapt,
1709                                         const char *termz,
1710                                         oid_value attributeSet,
1711                                         NMEM stream,
1712                                         int reg_type, int complete_flag,
1713                                         const char *rank_type,
1714                                         const char *xpath_use,
1715                                         int num_bases, char **basenames,
1716                                         NMEM rset_nmem,
1717                                         RSET *rset,
1718                                         struct rset_key_control *kc)
1719 {
1720     char term_dst[IT_MAX_WORD+1];
1721     const char *termp = termz;
1722     RSET *result_sets = 0;
1723     int num_result_sets = 0;
1724     ZEBRA_RES res;
1725     struct grep_info grep_info;
1726     int alloc_sets = 0;
1727     zint hits_limit_value;
1728     const char *term_ref_id_str = 0;
1729
1730     term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str, stream);
1731
1732     yaz_log(log_level_rpn, "APT_numeric t='%s'", termz);
1733     if (grep_info_prepare(zh, zapt, &grep_info, reg_type) == ZEBRA_FAIL)
1734         return ZEBRA_FAIL;
1735     while (1)
1736     {
1737         if (alloc_sets == num_result_sets)
1738         {
1739             int add = 10;
1740             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1741                                               sizeof(*rnew));
1742             if (alloc_sets)
1743                 memcpy(rnew, result_sets, alloc_sets * sizeof(*rnew));
1744             alloc_sets = alloc_sets + add;
1745             result_sets = rnew;
1746         }
1747         yaz_log(YLOG_DEBUG, "APT_numeric termp=%s", termp);
1748         grep_info.isam_p_indx = 0;
1749         res = numeric_term(zh, zapt, &termp, attributeSet, &grep_info,
1750                            reg_type, complete_flag, num_bases, basenames,
1751                            term_dst, xpath_use,
1752                            stream);
1753         if (res == ZEBRA_FAIL || termp == 0)
1754             break;
1755         yaz_log(YLOG_DEBUG, "term: %s", term_dst);
1756         result_sets[num_result_sets] =
1757             rset_trunc(zh, grep_info.isam_p_buf,
1758                        grep_info.isam_p_indx, term_dst,
1759                        strlen(term_dst), rank_type,
1760                        0 /* preserve position */,
1761                        zapt->term->which, rset_nmem,
1762                        kc, kc->scope, 0, reg_type,
1763                        hits_limit_value,
1764                        term_ref_id_str);
1765         if (!result_sets[num_result_sets])
1766             break;
1767         num_result_sets++;
1768     }
1769     grep_info_delete(&grep_info);
1770     if (termp)
1771     {
1772         int i;
1773         for (i = 0; i<num_result_sets; i++)
1774             rset_delete(result_sets[i]);
1775         return ZEBRA_FAIL;
1776     }
1777     if (num_result_sets == 0)
1778         *rset = rsnull_create(rset_nmem, kc, 0);
1779     if (num_result_sets == 1)
1780         *rset = result_sets[0];
1781     else
1782         *rset = rsmulti_and_create(rset_nmem, kc, kc->scope,
1783                                    num_result_sets, result_sets);
1784     if (!*rset)
1785         return ZEBRA_FAIL;
1786     return ZEBRA_OK;
1787 }
1788
1789 static ZEBRA_RES rpn_search_APT_local(ZebraHandle zh,
1790                                       Z_AttributesPlusTerm *zapt,
1791                                       const char *termz,
1792                                       oid_value attributeSet,
1793                                       NMEM stream,
1794                                       const char *rank_type, NMEM rset_nmem,
1795                                       RSET *rset,
1796                                       struct rset_key_control *kc)
1797 {
1798     RSFD rsfd;
1799     struct it_key key;
1800     int sys;
1801     *rset = rstemp_create(rset_nmem, kc, kc->scope,
1802                           res_get (zh->res, "setTmpDir"),0 );
1803     rsfd = rset_open(*rset, RSETF_WRITE);
1804
1805     sys = atoi(termz);
1806     if (sys <= 0)
1807         sys = 1;
1808     key.mem[0] = sys;
1809     key.mem[1] = 1;
1810     key.len = 2;
1811     rset_write (rsfd, &key);
1812     rset_close (rsfd);
1813     return ZEBRA_OK;
1814 }
1815
1816 static ZEBRA_RES rpn_sort_spec(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1817                                oid_value attributeSet, NMEM stream,
1818                                Z_SortKeySpecList *sort_sequence,
1819                                const char *rank_type,
1820                                NMEM rset_nmem,
1821                                RSET *rset,
1822                                struct rset_key_control *kc)
1823 {
1824     int i;
1825     int sort_relation_value;
1826     AttrType sort_relation_type;
1827     Z_SortKeySpec *sks;
1828     Z_SortKey *sk;
1829     int oid[OID_SIZE];
1830     oident oe;
1831     char termz[20];
1832
1833     attr_init_APT(&sort_relation_type, zapt, 7);
1834     sort_relation_value = attr_find(&sort_relation_type, &attributeSet);
1835
1836     if (!sort_sequence->specs)
1837     {
1838         sort_sequence->num_specs = 10;
1839         sort_sequence->specs = (Z_SortKeySpec **)
1840             nmem_malloc(stream, sort_sequence->num_specs *
1841                          sizeof(*sort_sequence->specs));
1842         for (i = 0; i<sort_sequence->num_specs; i++)
1843             sort_sequence->specs[i] = 0;
1844     }
1845     if (zapt->term->which != Z_Term_general)
1846         i = 0;
1847     else
1848         i = atoi_n ((char *) zapt->term->u.general->buf,
1849                     zapt->term->u.general->len);
1850     if (i >= sort_sequence->num_specs)
1851         i = 0;
1852     sprintf(termz, "%d", i);
1853
1854     oe.proto = PROTO_Z3950;
1855     oe.oclass = CLASS_ATTSET;
1856     oe.value = attributeSet;
1857     if (!oid_ent_to_oid (&oe, oid))
1858         return ZEBRA_FAIL;
1859
1860     sks = (Z_SortKeySpec *) nmem_malloc(stream, sizeof(*sks));
1861     sks->sortElement = (Z_SortElement *)
1862         nmem_malloc(stream, sizeof(*sks->sortElement));
1863     sks->sortElement->which = Z_SortElement_generic;
1864     sk = sks->sortElement->u.generic = (Z_SortKey *)
1865         nmem_malloc(stream, sizeof(*sk));
1866     sk->which = Z_SortKey_sortAttributes;
1867     sk->u.sortAttributes = (Z_SortAttributes *)
1868         nmem_malloc(stream, sizeof(*sk->u.sortAttributes));
1869
1870     sk->u.sortAttributes->id = oid;
1871     sk->u.sortAttributes->list = zapt->attributes;
1872
1873     sks->sortRelation = (int *)
1874         nmem_malloc(stream, sizeof(*sks->sortRelation));
1875     if (sort_relation_value == 1)
1876         *sks->sortRelation = Z_SortKeySpec_ascending;
1877     else if (sort_relation_value == 2)
1878         *sks->sortRelation = Z_SortKeySpec_descending;
1879     else
1880         *sks->sortRelation = Z_SortKeySpec_ascending;
1881
1882     sks->caseSensitivity = (int *)
1883         nmem_malloc(stream, sizeof(*sks->caseSensitivity));
1884     *sks->caseSensitivity = 0;
1885
1886     sks->which = Z_SortKeySpec_null;
1887     sks->u.null = odr_nullval ();
1888     sort_sequence->specs[i] = sks;
1889     *rset = rsnull_create (rset_nmem, kc, 0);
1890     return ZEBRA_OK;
1891 }
1892
1893
1894 static int parse_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1895                        oid_value attributeSet,
1896                        struct xpath_location_step *xpath, int max, NMEM mem)
1897 {
1898     oid_value curAttributeSet = attributeSet;
1899     AttrType use;
1900     const char *use_string = 0;
1901
1902     attr_init_APT(&use, zapt, 1);
1903     attr_find_ex(&use, &curAttributeSet, &use_string);
1904
1905     if (!use_string || *use_string != '/')
1906         return -1;
1907
1908     return zebra_parse_xpath_str(use_string, xpath, max, mem);
1909 }
1910
1911
1912
1913 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
1914                         int reg_type, const char *term,
1915                         const char *xpath_use,
1916                         oid_value curAttributeSet, NMEM rset_nmem,
1917                         struct rset_key_control *kc)
1918 {
1919     RSET rset;
1920     struct grep_info grep_info;
1921     char term_dict[2048];
1922     char ord_buf[32];
1923     int prefix_len = 0;
1924     int ord = zebraExplain_lookup_attr_str(zh->reg->zei, reg_type,
1925                                            xpath_use);
1926     int ord_len, i, r, max_pos;
1927     int term_type = Z_Term_characterString;
1928     const char *flags = "void";
1929
1930     if (grep_info_prepare(zh, 0 /* zapt */, &grep_info, '0') == ZEBRA_FAIL)
1931         return rsnull_create(rset_nmem, kc, 0);
1932
1933     if (ord < 0)
1934         return rsnull_create(rset_nmem, kc, 0);
1935     if (prefix_len)
1936         term_dict[prefix_len++] = '|';
1937     else
1938         term_dict[prefix_len++] = '(';
1939
1940     ord_len = key_SU_encode (ord, ord_buf);
1941     for (i = 0; i<ord_len; i++)
1942     {
1943         term_dict[prefix_len++] = 1;
1944         term_dict[prefix_len++] = ord_buf[i];
1945     }
1946     term_dict[prefix_len++] = ')';
1947     strcpy(term_dict+prefix_len, term);
1948
1949     grep_info.isam_p_indx = 0;
1950     r = dict_lookup_grep(zh->reg->dict, term_dict, 0,
1951                           &grep_info, &max_pos, 0, grep_handle);
1952     yaz_log(YLOG_DEBUG, "%s %d positions", term,
1953              grep_info.isam_p_indx);
1954     rset = rset_trunc(zh, grep_info.isam_p_buf,
1955                       grep_info.isam_p_indx, term, strlen(term),
1956                       flags, 1, term_type,rset_nmem,
1957                       kc, kc->scope, 0, reg_type, 0 /* hits_limit */,
1958                       0 /* term_ref_id_str */);
1959     grep_info_delete(&grep_info);
1960     return rset;
1961 }
1962
1963 static
1964 ZEBRA_RES rpn_search_xpath(ZebraHandle zh,
1965                            oid_value attributeSet,
1966                            int num_bases, char **basenames,
1967                            NMEM stream, const char *rank_type, RSET rset,
1968                            int xpath_len, struct xpath_location_step *xpath,
1969                            NMEM rset_nmem,
1970                            RSET *rset_out,
1971                            struct rset_key_control *kc)
1972 {
1973     oid_value curAttributeSet = attributeSet;
1974     int base_no;
1975     int i;
1976
1977     if (xpath_len < 0)
1978     {
1979         *rset_out = rset;
1980         return ZEBRA_OK;
1981     }
1982
1983     yaz_log(YLOG_DEBUG, "xpath len=%d", xpath_len);
1984     for (i = 0; i<xpath_len; i++)
1985     {
1986         yaz_log(log_level_rpn, "XPATH %d %s", i, xpath[i].part);
1987
1988     }
1989
1990     curAttributeSet = VAL_IDXPATH;
1991
1992     /*
1993       //a    ->    a/.*
1994       //a/b  ->    b/a/.*
1995       /a     ->    a/
1996       /a/b   ->    b/a/
1997
1998       /      ->    none
1999
2000    a[@attr = value]/b[@other = othervalue]
2001
2002  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
2003  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
2004  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
2005  /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y)
2006  /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y)
2007  /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x)
2008
2009     */
2010
2011     dict_grep_cmap (zh->reg->dict, 0, 0);
2012
2013     for (base_no = 0; base_no < num_bases; base_no++)
2014     {
2015         int level = xpath_len;
2016         int first_path = 1;
2017
2018         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2019         {
2020             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
2021                            basenames[base_no]);
2022             *rset_out = rset;
2023             return ZEBRA_FAIL;
2024         }
2025         while (--level >= 0)
2026         {
2027             char xpath_rev[128];
2028             int i, len;
2029             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
2030
2031             *xpath_rev = 0;
2032             len = 0;
2033             for (i = level; i >= 1; --i)
2034             {
2035                 const char *cp = xpath[i].part;
2036                 if (*cp)
2037                 {
2038                     for (;*cp; cp++)
2039                         if (*cp == '*')
2040                         {
2041                             memcpy (xpath_rev + len, "[^/]*", 5);
2042                             len += 5;
2043                         }
2044                         else if (*cp == ' ')
2045                         {
2046
2047                             xpath_rev[len++] = 1;
2048                             xpath_rev[len++] = ' ';
2049                         }
2050
2051                         else
2052                             xpath_rev[len++] = *cp;
2053                     xpath_rev[len++] = '/';
2054                 }
2055                 else if (i == 1)  /* // case */
2056                 {
2057                     xpath_rev[len++] = '.';
2058                     xpath_rev[len++] = '*';
2059                 }
2060             }
2061             xpath_rev[len] = 0;
2062
2063             if (xpath[level].predicate &&
2064                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
2065                 xpath[level].predicate->u.relation.name[0])
2066             {
2067                 WRBUF wbuf = wrbuf_alloc();
2068                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
2069                 if (xpath[level].predicate->u.relation.value)
2070                 {
2071                     const char *cp = xpath[level].predicate->u.relation.value;
2072                     wrbuf_putc(wbuf, '=');
2073
2074                     while (*cp)
2075                     {
2076                         if (strchr(REGEX_CHARS, *cp))
2077                             wrbuf_putc(wbuf, '\\');
2078                         wrbuf_putc(wbuf, *cp);
2079                         cp++;
2080                     }
2081                 }
2082                 wrbuf_puts(wbuf, "");
2083                 rset_attr = xpath_trunc(
2084                     zh, stream, '0', wrbuf_buf(wbuf), ZEBRA_XPATH_ATTR_NAME,
2085                     curAttributeSet, rset_nmem, kc);
2086                 wrbuf_free(wbuf, 1);
2087             }
2088             else
2089             {
2090                 if (!first_path)
2091                     continue;
2092             }
2093             yaz_log(log_level_rpn, "xpath_rev (%d) = %s", level, xpath_rev);
2094             if (strlen(xpath_rev))
2095             {
2096                 rset_start_tag = xpath_trunc(zh, stream, '0',
2097                                              xpath_rev,
2098                                              ZEBRA_XPATH_ELM_BEGIN,
2099                                              curAttributeSet,
2100                                              rset_nmem, kc);
2101
2102                 rset_end_tag = xpath_trunc(zh, stream, '0',
2103                                            xpath_rev,
2104                                            ZEBRA_XPATH_ELM_END,
2105                                            curAttributeSet,
2106                                            rset_nmem, kc);
2107
2108                 rset = rsbetween_create(rset_nmem, kc, kc->scope,
2109                                         rset_start_tag, rset,
2110                                         rset_end_tag, rset_attr);
2111             }
2112             first_path = 0;
2113         }
2114     }
2115     *rset_out = rset;
2116     return ZEBRA_OK;
2117 }
2118
2119 static ZEBRA_RES rpn_search_APT(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2120                                 oid_value attributeSet, NMEM stream,
2121                                 Z_SortKeySpecList *sort_sequence,
2122                                 int num_bases, char **basenames,
2123                                 NMEM rset_nmem,
2124                                 RSET *rset,
2125                                 struct rset_key_control *kc)
2126 {
2127     ZEBRA_RES res = ZEBRA_OK;
2128     unsigned reg_id;
2129     char *search_type = NULL;
2130     char rank_type[128];
2131     int complete_flag;
2132     int sort_flag;
2133     char termz[IT_MAX_WORD+1];
2134     int xpath_len;
2135     const char *xpath_use = 0;
2136     struct xpath_location_step xpath[10];
2137
2138     if (!log_level_set)
2139     {
2140         log_level_rpn = yaz_log_module_level("rpn");
2141         log_level_set = 1;
2142     }
2143     zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2144                     rank_type, &complete_flag, &sort_flag);
2145
2146     yaz_log(YLOG_DEBUG, "reg_id=%c", reg_id);
2147     yaz_log(YLOG_DEBUG, "complete_flag=%d", complete_flag);
2148     yaz_log(YLOG_DEBUG, "search_type=%s", search_type);
2149     yaz_log(YLOG_DEBUG, "rank_type=%s", rank_type);
2150
2151     if (zapt_term_to_utf8(zh, zapt, termz) == ZEBRA_FAIL)
2152         return ZEBRA_FAIL;
2153
2154     if (sort_flag)
2155         return rpn_sort_spec(zh, zapt, attributeSet, stream, sort_sequence,
2156                              rank_type, rset_nmem, rset, kc);
2157     /* consider if an X-Path query is used */
2158     xpath_len = parse_xpath(zh, zapt, attributeSet, xpath, 10, stream);
2159     if (xpath_len >= 0)
2160     {
2161         if (xpath[xpath_len-1].part[0] == '@')
2162             xpath_use = ZEBRA_XPATH_ATTR_CDATA;  /* last step is attribute  */
2163         else
2164             xpath_use = ZEBRA_XPATH_CDATA;  /* searching for cdata */
2165     }
2166
2167     /* search using one of the various search type strategies
2168        termz is our UTF-8 search term
2169        attributeSet is top-level default attribute set
2170        stream is ODR for search
2171        reg_id is the register type
2172        complete_flag is 1 for complete subfield, 0 for incomplete
2173        xpath_use is use-attribute to be used for X-Path search, 0 for none
2174     */
2175     if (!strcmp(search_type, "phrase"))
2176     {
2177         res = rpn_search_APT_phrase(zh, zapt, termz, attributeSet, stream,
2178                                     reg_id, complete_flag, rank_type,
2179                                     xpath_use,
2180                                     num_bases, basenames, rset_nmem,
2181                                     rset, kc);
2182     }
2183     else if (!strcmp(search_type, "and-list"))
2184     {
2185         res = rpn_search_APT_and_list(zh, zapt, termz, attributeSet, stream,
2186                                       reg_id, complete_flag, rank_type,
2187                                       xpath_use,
2188                                       num_bases, basenames, rset_nmem,
2189                                       rset, kc);
2190     }
2191     else if (!strcmp(search_type, "or-list"))
2192     {
2193         res = rpn_search_APT_or_list(zh, zapt, termz, attributeSet, stream,
2194                                      reg_id, complete_flag, rank_type,
2195                                      xpath_use,
2196                                      num_bases, basenames, rset_nmem,
2197                                      rset, kc);
2198     }
2199     else if (!strcmp(search_type, "local"))
2200     {
2201         res = rpn_search_APT_local(zh, zapt, termz, attributeSet, stream,
2202                                    rank_type, rset_nmem, rset, kc);
2203     }
2204     else if (!strcmp(search_type, "numeric"))
2205     {
2206         res = rpn_search_APT_numeric(zh, zapt, termz, attributeSet, stream,
2207                                      reg_id, complete_flag, rank_type,
2208                                      xpath_use,
2209                                      num_bases, basenames, rset_nmem,
2210                                      rset, kc);
2211     }
2212     else if (!strcmp(search_type, "always"))
2213     {
2214         *termz = '\0';
2215         res = rpn_search_APT_phrase(zh, zapt, termz, attributeSet, stream,
2216                                     reg_id, complete_flag, rank_type,
2217                                     xpath_use,
2218                                     num_bases, basenames, rset_nmem,
2219                                     rset, kc);
2220     }
2221     else
2222     {
2223         zebra_setError(zh, YAZ_BIB1_UNSUPP_STRUCTURE_ATTRIBUTE, 0);
2224         res = ZEBRA_FAIL;
2225     }
2226     if (res != ZEBRA_OK)
2227         return res;
2228     if (!*rset)
2229         return ZEBRA_FAIL;
2230     return rpn_search_xpath(zh, attributeSet, num_bases, basenames,
2231                             stream, rank_type, *rset,
2232                             xpath_len, xpath, rset_nmem, rset, kc);
2233 }
2234
2235 static ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2236                                       oid_value attributeSet,
2237                                       NMEM stream, NMEM rset_nmem,
2238                                       Z_SortKeySpecList *sort_sequence,
2239                                       int num_bases, char **basenames,
2240                                       RSET **result_sets, int *num_result_sets,
2241                                       Z_Operator *parent_op,
2242                                       struct rset_key_control *kc);
2243
2244 ZEBRA_RES rpn_search_top(ZebraHandle zh, Z_RPNStructure *zs,
2245                          oid_value attributeSet,
2246                          NMEM stream, NMEM rset_nmem,
2247                          Z_SortKeySpecList *sort_sequence,
2248                          int num_bases, char **basenames,
2249                          RSET *result_set)
2250 {
2251     RSET *result_sets = 0;
2252     int num_result_sets = 0;
2253     ZEBRA_RES res;
2254     struct rset_key_control *kc = zebra_key_control_create(zh);
2255
2256     res = rpn_search_structure(zh, zs, attributeSet,
2257                                stream, rset_nmem,
2258                                sort_sequence,
2259                                num_bases, basenames,
2260                                &result_sets, &num_result_sets,
2261                                0 /* no parent op */,
2262                                kc);
2263     if (res != ZEBRA_OK)
2264     {
2265         int i;
2266         for (i = 0; i<num_result_sets; i++)
2267             rset_delete(result_sets[i]);
2268         *result_set = 0;
2269     }
2270     else
2271     {
2272         assert(num_result_sets == 1);
2273         assert(result_sets);
2274         assert(*result_sets);
2275         *result_set = *result_sets;
2276     }
2277     (*kc->dec)(kc);
2278     return res;
2279 }
2280
2281 ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2282                                oid_value attributeSet,
2283                                NMEM stream, NMEM rset_nmem,
2284                                Z_SortKeySpecList *sort_sequence,
2285                                int num_bases, char **basenames,
2286                                RSET **result_sets, int *num_result_sets,
2287                                Z_Operator *parent_op,
2288                                struct rset_key_control *kc)
2289 {
2290     *num_result_sets = 0;
2291     if (zs->which == Z_RPNStructure_complex)
2292     {
2293         ZEBRA_RES res;
2294         Z_Operator *zop = zs->u.complex->roperator;
2295         RSET *result_sets_l = 0;
2296         int num_result_sets_l = 0;
2297         RSET *result_sets_r = 0;
2298         int num_result_sets_r = 0;
2299
2300         res = rpn_search_structure(zh, zs->u.complex->s1,
2301                                    attributeSet, stream, rset_nmem,
2302                                    sort_sequence,
2303                                    num_bases, basenames,
2304                                    &result_sets_l, &num_result_sets_l,
2305                                    zop, kc);
2306         if (res != ZEBRA_OK)
2307         {
2308             int i;
2309             for (i = 0; i<num_result_sets_l; i++)
2310                 rset_delete(result_sets_l[i]);
2311             return res;
2312         }
2313         res = rpn_search_structure(zh, zs->u.complex->s2,
2314                                    attributeSet, stream, rset_nmem,
2315                                    sort_sequence,
2316                                    num_bases, basenames,
2317                                    &result_sets_r, &num_result_sets_r,
2318                                    zop, kc);
2319         if (res != ZEBRA_OK)
2320         {
2321             int i;
2322             for (i = 0; i<num_result_sets_l; i++)
2323                 rset_delete(result_sets_l[i]);
2324             for (i = 0; i<num_result_sets_r; i++)
2325                 rset_delete(result_sets_r[i]);
2326             return res;
2327         }
2328
2329         /* make a new list of result for all children */
2330         *num_result_sets = num_result_sets_l + num_result_sets_r;
2331         *result_sets = nmem_malloc(stream, *num_result_sets *
2332                                    sizeof(**result_sets));
2333         memcpy(*result_sets, result_sets_l,
2334                num_result_sets_l * sizeof(**result_sets));
2335         memcpy(*result_sets + num_result_sets_l, result_sets_r,
2336                num_result_sets_r * sizeof(**result_sets));
2337
2338         if (!parent_op || parent_op->which != zop->which
2339             || (zop->which != Z_Operator_and &&
2340                 zop->which != Z_Operator_or))
2341         {
2342             /* parent node different from this one (or non-present) */
2343             /* we must combine result sets now */
2344             RSET rset;
2345             switch (zop->which)
2346             {
2347             case Z_Operator_and:
2348                 rset = rsmulti_and_create(rset_nmem, kc,
2349                                           kc->scope,
2350                                           *num_result_sets, *result_sets);
2351                 break;
2352             case Z_Operator_or:
2353                 rset = rsmulti_or_create(rset_nmem, kc,
2354                                          kc->scope, 0, /* termid */
2355                                          *num_result_sets, *result_sets);
2356                 break;
2357             case Z_Operator_and_not:
2358                 rset = rsbool_create_not(rset_nmem, kc,
2359                                          kc->scope,
2360                                          (*result_sets)[0],
2361                                          (*result_sets)[1]);
2362                 break;
2363             case Z_Operator_prox:
2364                 if (zop->u.prox->which != Z_ProximityOperator_known)
2365                 {
2366                     zebra_setError(zh,
2367                                    YAZ_BIB1_UNSUPP_PROX_UNIT_CODE,
2368                                    0);
2369                     return ZEBRA_FAIL;
2370                 }
2371                 if (*zop->u.prox->u.known != Z_ProxUnit_word)
2372                 {
2373                     zebra_setError_zint(zh,
2374                                         YAZ_BIB1_UNSUPP_PROX_UNIT_CODE,
2375                                         *zop->u.prox->u.known);
2376                     return ZEBRA_FAIL;
2377                 }
2378                 else
2379                 {
2380                     rset = rsprox_create(rset_nmem, kc,
2381                                          kc->scope,
2382                                          *num_result_sets, *result_sets,
2383                                          *zop->u.prox->ordered,
2384                                          (!zop->u.prox->exclusion ?
2385                                           0 : *zop->u.prox->exclusion),
2386                                          *zop->u.prox->relationType,
2387                                          *zop->u.prox->distance );
2388                 }
2389                 break;
2390             default:
2391                 zebra_setError(zh, YAZ_BIB1_OPERATOR_UNSUPP, 0);
2392                 return ZEBRA_FAIL;
2393             }
2394             *num_result_sets = 1;
2395             *result_sets = nmem_malloc(stream, *num_result_sets *
2396                                        sizeof(**result_sets));
2397             (*result_sets)[0] = rset;
2398         }
2399     }
2400     else if (zs->which == Z_RPNStructure_simple)
2401     {
2402         RSET rset;
2403         ZEBRA_RES res;
2404
2405         if (zs->u.simple->which == Z_Operand_APT)
2406         {
2407             yaz_log(YLOG_DEBUG, "rpn_search_APT");
2408             res = rpn_search_APT(zh, zs->u.simple->u.attributesPlusTerm,
2409                                  attributeSet, stream, sort_sequence,
2410                                  num_bases, basenames, rset_nmem, &rset,
2411                                  kc);
2412             if (res != ZEBRA_OK)
2413                 return res;
2414         }
2415         else if (zs->u.simple->which == Z_Operand_resultSetId)
2416         {
2417             yaz_log(YLOG_DEBUG, "rpn_search_ref");
2418             rset = resultSetRef(zh, zs->u.simple->u.resultSetId);
2419             if (!rset)
2420             {
2421                 zebra_setError(zh,
2422                                YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
2423                                zs->u.simple->u.resultSetId);
2424                 return ZEBRA_FAIL;
2425             }
2426             rset_dup(rset);
2427         }
2428         else
2429         {
2430             zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2431             return ZEBRA_FAIL;
2432         }
2433         *num_result_sets = 1;
2434         *result_sets = nmem_malloc(stream, *num_result_sets *
2435                                    sizeof(**result_sets));
2436         (*result_sets)[0] = rset;
2437     }
2438     else
2439     {
2440         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2441         return ZEBRA_FAIL;
2442     }
2443     return ZEBRA_OK;
2444 }
2445
2446 struct scan_info_entry {
2447     char *term;
2448     ISAM_P isam_p;
2449 };
2450
2451 struct scan_info {
2452     struct scan_info_entry *list;
2453     ODR odr;
2454     int before, after;
2455     char prefix[20];
2456 };
2457
2458 static int scan_handle (char *name, const char *info, int pos, void *client)
2459 {
2460     int len_prefix, idx;
2461     struct scan_info *scan_info = (struct scan_info *) client;
2462
2463     len_prefix = strlen(scan_info->prefix);
2464     if (memcmp (name, scan_info->prefix, len_prefix))
2465         return 1;
2466     if (pos > 0)
2467         idx = scan_info->after - pos + scan_info->before;
2468     else
2469         idx = - pos - 1;
2470
2471     if (idx < 0)
2472         return 0;
2473     scan_info->list[idx].term = (char *)
2474         odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
2475     strcpy(scan_info->list[idx].term, name + len_prefix);
2476     assert (*info == sizeof(ISAM_P));
2477     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
2478     return 0;
2479 }
2480
2481 void zebra_term_untrans_iconv(ZebraHandle zh, NMEM stream, int reg_type,
2482                               char **dst, const char *src)
2483 {
2484     char term_src[IT_MAX_WORD];
2485     char term_dst[IT_MAX_WORD];
2486
2487     zebra_term_untrans (zh, reg_type, term_src, src);
2488
2489     if (zh->iconv_from_utf8 != 0)
2490     {
2491         int len;
2492         char *inbuf = term_src;
2493         size_t inleft = strlen(term_src);
2494         char *outbuf = term_dst;
2495         size_t outleft = sizeof(term_dst)-1;
2496         size_t ret;
2497
2498         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2499                          &outbuf, &outleft);
2500         if (ret == (size_t)(-1))
2501             len = 0;
2502         else
2503             len = outbuf - term_dst;
2504         *dst = nmem_malloc(stream, len + 1);
2505         if (len > 0)
2506             memcpy (*dst, term_dst, len);
2507         (*dst)[len] = '\0';
2508     }
2509     else
2510         *dst = nmem_strdup(stream, term_src);
2511 }
2512
2513 static void count_set(ZebraHandle zh, RSET rset, zint *count)
2514 {
2515     zint psysno = 0;
2516     struct it_key key;
2517     RSFD rfd;
2518
2519     yaz_log(YLOG_DEBUG, "count_set");
2520
2521     rset->hits_limit = zh->approx_limit;
2522
2523     *count = 0;
2524     rfd = rset_open(rset, RSETF_READ);
2525     while (rset_read(rfd, &key,0 /* never mind terms */))
2526     {
2527         if (key.mem[0] != psysno)
2528         {
2529             psysno = key.mem[0];
2530             if (rfd->counted_items >= rset->hits_limit)
2531                 break;
2532         }
2533     }
2534     rset_close (rfd);
2535     *count = rset->hits_count;
2536 }
2537
2538 ZEBRA_RES rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2539                    oid_value attributeset,
2540                    int num_bases, char **basenames,
2541                    int *position, int *num_entries, ZebraScanEntry **list,
2542                    int *is_partial, RSET limit_set, int return_zero)
2543 {
2544     int i;
2545     int pos = *position;
2546     int num = *num_entries;
2547     int before;
2548     int after;
2549     int base_no;
2550     char termz[IT_MAX_WORD+20];
2551     struct scan_info *scan_info_array;
2552     ZebraScanEntry *glist;
2553     int ords[32], ord_no = 0;
2554     int ptr[32];
2555
2556     int bases_ok = 0;     /* no of databases with OK attribute */
2557     int errCode = 0;      /* err code (if any is not OK) */
2558     char *errString = 0;  /* addinfo */
2559
2560     unsigned index_type;
2561     char *search_type = NULL;
2562     char rank_type[128];
2563     int complete_flag;
2564     int sort_flag;
2565     NMEM rset_nmem = NULL;
2566     struct rset_key_control *kc = 0;
2567
2568     *list = 0;
2569     *is_partial = 0;
2570
2571     if (attributeset == VAL_NONE)
2572         attributeset = VAL_BIB1;
2573
2574     if (!limit_set)
2575     {
2576         AttrType termset;
2577         int termset_value_numeric;
2578         const char *termset_value_string;
2579         attr_init_APT(&termset, zapt, 8);
2580         termset_value_numeric =
2581             attr_find_ex(&termset, NULL, &termset_value_string);
2582         if (termset_value_numeric != -1)
2583         {
2584             char resname[32];
2585             const char *termset_name = 0;
2586
2587             if (termset_value_numeric != -2)
2588             {
2589
2590                 sprintf(resname, "%d", termset_value_numeric);
2591                 termset_name = resname;
2592             }
2593             else
2594                 termset_name = termset_value_string;
2595
2596             limit_set = resultSetRef (zh, termset_name);
2597         }
2598     }
2599
2600     yaz_log(YLOG_DEBUG, "position = %d, num = %d set=%d",
2601             pos, num, attributeset);
2602
2603     if (zebra_maps_attr(zh->reg->zebra_maps, zapt, &index_type, &search_type,
2604                         rank_type, &complete_flag, &sort_flag))
2605     {
2606         *num_entries = 0;
2607         zebra_setError(zh, YAZ_BIB1_UNSUPP_ATTRIBUTE_TYPE, 0);
2608         return ZEBRA_FAIL;
2609     }
2610     for (base_no = 0; base_no < num_bases && ord_no < 32; base_no++)
2611     {
2612         int ord;
2613
2614         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2615         {
2616             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
2617                            basenames[base_no]);
2618             *num_entries = 0;
2619             return ZEBRA_FAIL;
2620         }
2621
2622         if (zebra_apt_get_ord(zh, zapt, index_type, 0, attributeset, &ord)
2623             != ZEBRA_OK)
2624         {
2625             break;
2626         }
2627         ords[ord_no++] = ord;
2628     }
2629     if (!bases_ok && errCode)
2630     {
2631         zebra_setError(zh, errCode, errString);
2632         *num_entries = 0;
2633         return ZEBRA_FAIL;
2634     }
2635     if (ord_no == 0)
2636     {
2637         *num_entries = 0;
2638         return ZEBRA_OK;
2639     }
2640     /* prepare dictionary scanning */
2641     if (num < 1)
2642     {
2643         *num_entries = 0;
2644         return ZEBRA_OK;
2645     }
2646     before = pos-1;
2647     if (before < 0)
2648         before = 0;
2649     after = 1+num-pos;
2650     if (after < 0)
2651         after = 0;
2652     yaz_log(YLOG_DEBUG, "rpn_scan pos=%d num=%d before=%d "
2653             "after=%d before+after=%d",
2654             pos, num, before, after, before+after);
2655     scan_info_array = (struct scan_info *)
2656         odr_malloc(stream, ord_no * sizeof(*scan_info_array));
2657     for (i = 0; i < ord_no; i++)
2658     {
2659         int j, prefix_len = 0;
2660         int before_tmp = before, after_tmp = after;
2661         struct scan_info *scan_info = scan_info_array + i;
2662         struct rpn_char_map_info rcmi;
2663
2664         rpn_char_map_prepare (zh->reg, index_type, &rcmi);
2665
2666         scan_info->before = before;
2667         scan_info->after = after;
2668         scan_info->odr = stream;
2669
2670         scan_info->list = (struct scan_info_entry *)
2671             odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
2672         for (j = 0; j<before+after; j++)
2673             scan_info->list[j].term = NULL;
2674
2675         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2676         termz[prefix_len] = 0;
2677         strcpy(scan_info->prefix, termz);
2678
2679         if (trans_scan_term(zh, zapt, termz+prefix_len, index_type) ==
2680             ZEBRA_FAIL)
2681             return ZEBRA_FAIL;
2682
2683         dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
2684                   scan_info, scan_handle);
2685     }
2686     glist = (ZebraScanEntry *)
2687         odr_malloc(stream, (before+after)*sizeof(*glist));
2688
2689     rset_nmem = nmem_create();
2690     kc = zebra_key_control_create(zh);
2691
2692     /* consider terms after main term */
2693     for (i = 0; i < ord_no; i++)
2694         ptr[i] = before;
2695
2696     *is_partial = 0;
2697     for (i = 0; i<after; i++)
2698     {
2699         int j, j0 = -1;
2700         const char *mterm = NULL;
2701         const char *tst;
2702         RSET rset = 0;
2703         int lo = i + pos-1; /* offset in result list */
2704
2705         /* find: j0 is the first of the minimal values */
2706         for (j = 0; j < ord_no; j++)
2707         {
2708             if (ptr[j] < before+after && ptr[j] >= 0 &&
2709                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2710                 (!mterm || strcmp (tst, mterm) < 0))
2711             {
2712                 j0 = j;
2713                 mterm = tst;
2714             }
2715         }
2716         if (j0 == -1)
2717             break;  /* no value found, stop */
2718
2719         /* get result set for first one , but only if it's within bounds */
2720         if (lo >= 0)
2721         {
2722             /* get result set for first term */
2723             zebra_term_untrans_iconv(zh, stream->mem, index_type,
2724                                      &glist[lo].term, mterm);
2725             rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2726                               glist[lo].term, strlen(glist[lo].term),
2727                               NULL, 0, zapt->term->which, rset_nmem,
2728                               kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2729                               0 /* term_ref_id_str */);
2730         }
2731         ptr[j0]++; /* move index for this set .. */
2732         /* get result set for remaining scan terms */
2733         for (j = j0+1; j<ord_no; j++)
2734         {
2735             if (ptr[j] < before+after && ptr[j] >= 0 &&
2736                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2737                 !strcmp (tst, mterm))
2738             {
2739                 if (lo >= 0)
2740                 {
2741                     RSET rsets[2];
2742
2743                     rsets[0] = rset;
2744                     rsets[1] =
2745                         rset_trunc(
2746                             zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
2747                             glist[lo].term,
2748                             strlen(glist[lo].term), NULL, 0,
2749                             zapt->term->which,rset_nmem,
2750                             kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2751                             0 /* term_ref_id_str */ );
2752                     rset = rsmulti_or_create(rset_nmem, kc,
2753                                              kc->scope, 0 /* termid */,
2754                                              2, rsets);
2755                 }
2756                 ptr[j]++;
2757             }
2758         }
2759         if (lo >= 0)
2760         {
2761             zint count;
2762             /* merge with limit_set if given */
2763             if (limit_set)
2764             {
2765                 RSET rsets[2];
2766                 rsets[0] = rset;
2767                 rsets[1] = rset_dup(limit_set);
2768
2769                 rset = rsmulti_and_create(rset_nmem, kc,
2770                                           kc->scope,
2771                                           2, rsets);
2772             }
2773             /* count it */
2774             count_set(zh, rset, &count);
2775             glist[lo].occurrences = count;
2776             rset_delete(rset);
2777         }
2778     }
2779     if (i < after)
2780     {
2781         *num_entries -= (after-i);
2782         *is_partial = 1;
2783         if (*num_entries < 0)
2784         {
2785             (*kc->dec)(kc);
2786             nmem_destroy(rset_nmem);
2787             *num_entries = 0;
2788             return ZEBRA_OK;
2789         }
2790     }
2791     /* consider terms before main term */
2792     for (i = 0; i<ord_no; i++)
2793         ptr[i] = 0;
2794
2795     for (i = 0; i<before; i++)
2796     {
2797         int j, j0 = -1;
2798         const char *mterm = NULL;
2799         const char *tst;
2800         RSET rset;
2801         int lo = before-1-i; /* offset in result list */
2802         zint count;
2803
2804         for (j = 0; j <ord_no; j++)
2805         {
2806             if (ptr[j] < before && ptr[j] >= 0 &&
2807                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2808                 (!mterm || strcmp (tst, mterm) > 0))
2809             {
2810                 j0 = j;
2811                     mterm = tst;
2812             }
2813         }
2814         if (j0 == -1)
2815             break;
2816
2817         zebra_term_untrans_iconv(zh, stream->mem, index_type,
2818                                  &glist[lo].term, mterm);
2819
2820         rset = rset_trunc
2821             (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
2822              glist[lo].term, strlen(glist[lo].term),
2823              NULL, 0, zapt->term->which, rset_nmem,
2824              kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2825              0 /* term_ref_id_str */);
2826
2827         ptr[j0]++;
2828
2829         for (j = j0+1; j<ord_no; j++)
2830         {
2831             if (ptr[j] < before && ptr[j] >= 0 &&
2832                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2833                 !strcmp (tst, mterm))
2834             {
2835                 RSET rsets[2];
2836
2837                 rsets[0] = rset;
2838                 rsets[1] = rset_trunc(
2839                     zh,
2840                     &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
2841                     glist[lo].term,
2842                     strlen(glist[lo].term), NULL, 0,
2843                     zapt->term->which, rset_nmem,
2844                     kc, kc->scope, 0, index_type, 0 /* hits_limit */,
2845                     0 /* term_ref_id_str */);
2846                 rset = rsmulti_or_create(rset_nmem, kc,
2847                                          kc->scope, 0 /* termid */, 2, rsets);
2848
2849                 ptr[j]++;
2850             }
2851         }
2852         if (limit_set)
2853         {
2854             RSET rsets[2];
2855             rsets[0] = rset;
2856             rsets[1] = rset_dup(limit_set);
2857
2858             rset = rsmulti_and_create(rset_nmem, kc,
2859                                       kc->scope, 2, rsets);
2860         }
2861         count_set(zh, rset, &count);
2862         glist[lo].occurrences = count;
2863         rset_delete (rset);
2864     }
2865     (*kc->dec)(kc);
2866     nmem_destroy(rset_nmem);
2867     i = before-i;
2868     if (i)
2869     {
2870         *is_partial = 1;
2871         *position -= i;
2872         *num_entries -= i;
2873         if (*num_entries <= 0)
2874         {
2875             *num_entries = 0;
2876             return ZEBRA_OK;
2877         }
2878     }
2879
2880     *list = glist + i;               /* list is set to first 'real' entry */
2881
2882     yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
2883             *position, *num_entries);
2884     return ZEBRA_OK;
2885 }
2886
2887 /*
2888  * Local variables:
2889  * c-basic-offset: 4
2890  * indent-tabs-mode: nil
2891  * End:
2892  * vim: shiftwidth=4 tabstop=8 expandtab
2893  */
2894