index/zrpn.c

   1 /* $Id: zrpn.c,v 1.187 2005-05-03 13:57:44 adam Exp $
   2    Copyright (C) 1995-2005
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with Zebra; see the file LICENSE.zebra.  If not, write to the
  19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  20 02111-1307, USA.
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #ifdef WIN32
  26 #include <io.h>
  27 #else
  28 #include <unistd.h>
  29 #endif
  30 #include <ctype.h>
  31
  32 #include <yaz/diagbib1.h>
  33 #include "index.h"
  34 #include <zebra_xpath.h>
  35
  36 #include <charmap.h>
  37 #include <rset.h>
  38
  39 struct rpn_char_map_info
  40 {
  41     ZebraMaps zm;
  42     int reg_type;
  43 };
  44
  45 typedef struct
  46 {
  47     int type;
  48     int major;
  49     int minor;
  50     Z_AttributesPlusTerm *zapt;
  51 } AttrType;
  52
  53
  54 static int log_level_set = 0;
  55 static int log_level_rpn = 0;
  56
  57 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  58 {
  59     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  60     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  61 #if 0
  62     if (out && *out)
  63     {
  64         const char *outp = *out;
  65         yaz_log(YLOG_LOG, "---");
  66         while (*outp)
  67         {
  68             yaz_log(YLOG_LOG, "%02X", *outp);
  69             outp++;
  70         }
  71     }
  72 #endif
  73     return out;
  74 }
  75
  76 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  77                                   struct rpn_char_map_info *map_info)
  78 {
  79     map_info->zm = reg->zebra_maps;
  80     map_info->reg_type = reg_type;
  81     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  82 }
  83
  84 static int attr_find_ex(AttrType *src, oid_value *attributeSetP,
  85                          const char **string_value)
  86 {
  87     int num_attributes;
  88
  89     num_attributes = src->zapt->attributes->num_attributes;
  90     while (src->major < num_attributes)
  91     {
  92         Z_AttributeElement *element;
  93
  94         element = src->zapt->attributes->attributes[src->major];
  95         if (src->type == *element->attributeType)
  96         {
  97             switch (element->which)
  98             {
  99             case Z_AttributeValue_numeric:
 100                 ++(src->major);
 101                 if (element->attributeSet && attributeSetP)
 102                 {
 103                     oident *attrset;
 104
 105                     attrset = oid_getentbyoid(element->attributeSet);
 106                     *attributeSetP = attrset->value;
 107                 }
 108                 return *element->value.numeric;
 109                 break;
 110             case Z_AttributeValue_complex:
 111                 if (src->minor >= element->value.complex->num_list)
 112                     break;
 113                 if (element->attributeSet && attributeSetP)
 114                 {
 115                     oident *attrset;
 116
 117                     attrset = oid_getentbyoid(element->attributeSet);
 118                     *attributeSetP = attrset->value;
 119                 }
 120                 if (element->value.complex->list[src->minor]->which ==
 121                     Z_StringOrNumeric_numeric)
 122                 {
 123                     ++(src->minor);
 124                     return
 125                         *element->value.complex->list[src->minor-1]->u.numeric;
 126                 }
 127                 else if (element->value.complex->list[src->minor]->which ==
 128                          Z_StringOrNumeric_string)
 129                 {
 130                     if (!string_value)
 131                         break;
 132                     ++(src->minor);
 133                     *string_value =
 134                         element->value.complex->list[src->minor-1]->u.string;
 135                     return -2;
 136                 }
 137                 else
 138                     break;
 139             default:
 140                 assert(0);
 141             }
 142         }
 143         ++(src->major);
 144     }
 145     return -1;
 146 }
 147
 148 static int attr_find(AttrType *src, oid_value *attributeSetP)
 149 {
 150     return attr_find_ex(src, attributeSetP, 0);
 151 }
 152
 153 static void attr_init(AttrType *src, Z_AttributesPlusTerm *zapt,
 154                        int type)
 155 {
 156     src->zapt = zapt;
 157     src->type = type;
 158     src->major = 0;
 159     src->minor = 0;
 160 }
 161
 162 #define TERM_COUNT
 163
 164 struct grep_info {
 165 #ifdef TERM_COUNT
 166     int *term_no;
 167 #endif
 168     ISAM_P *isam_p_buf;
 169     int isam_p_size;
 170     int isam_p_indx;
 171     ZebraHandle zh;
 172     int reg_type;
 173     ZebraSet termset;
 174 };
 175
 176 static void term_untrans(ZebraHandle zh, int reg_type,
 177                            char *dst, const char *src)
 178 {
 179     int len = 0;
 180     while (*src)
 181     {
 182         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
 183                                            reg_type, &src);
 184         if (!cp && len < IT_MAX_WORD-1)
 185             dst[len++] = *src++;
 186         else
 187             while (*cp && len < IT_MAX_WORD-1)
 188                 dst[len++] = *cp++;
 189     }
 190     dst[len] = '\0';
 191 }
 192
 193 static void add_isam_p(const char *name, const char *info,
 194                        struct grep_info *p)
 195 {
 196     if (!log_level_set)
 197     {
 198         log_level_rpn = yaz_log_module_level("rpn");
 199         log_level_set = 1;
 200     }
 201     if (p->isam_p_indx == p->isam_p_size)
 202     {
 203         ISAM_P *new_isam_p_buf;
 204 #ifdef TERM_COUNT
 205         int *new_term_no;
 206 #endif
 207         p->isam_p_size = 2*p->isam_p_size + 100;
 208         new_isam_p_buf = (ISAM_P *) xmalloc(sizeof(*new_isam_p_buf) *
 209                                             p->isam_p_size);
 210         if (p->isam_p_buf)
 211         {
 212             memcpy(new_isam_p_buf, p->isam_p_buf,
 213                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 214             xfree(p->isam_p_buf);
 215         }
 216         p->isam_p_buf = new_isam_p_buf;
 217
 218 #ifdef TERM_COUNT
 219         new_term_no = (int *) xmalloc(sizeof(*new_term_no) * p->isam_p_size);
 220         if (p->term_no)
 221         {
 222             memcpy(new_term_no, p->isam_p_buf,
 223                     p->isam_p_indx * sizeof(*p->term_no));
 224             xfree(p->term_no);
 225         }
 226         p->term_no = new_term_no;
 227 #endif
 228     }
 229     assert(*info == sizeof(*p->isam_p_buf));
 230     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 231
 232 #if 1
 233     if (p->termset)
 234     {
 235         const char *db;
 236         int set, use;
 237         char term_tmp[IT_MAX_WORD];
 238         int su_code = 0;
 239         int len = key_SU_decode (&su_code, name);
 240
 241         term_untrans  (p->zh, p->reg_type, term_tmp, name+len+1);
 242         yaz_log(log_level_rpn, "grep: %d %c %s", su_code, name[len], term_tmp);
 243         zebraExplain_lookup_ord (p->zh->reg->zei,
 244                                  su_code, &db, &set, &use);
 245         yaz_log(log_level_rpn, "grep:  set=%d use=%d db=%s", set, use, db);
 246
 247         resultSetAddTerm(p->zh, p->termset, name[len], db,
 248                          set, use, term_tmp);
 249     }
 250 #endif
 251     (p->isam_p_indx)++;
 252 }
 253
 254 static int grep_handle(char *name, const char *info, void *p)
 255 {
 256     add_isam_p(name, info, (struct grep_info *) p);
 257     return 0;
 258 }
 259
 260 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 261                     const char *ct1, const char *ct2, int first)
 262 {
 263     const char *s1, *s0 = *src;
 264     const char **map;
 265
 266     /* skip white space */
 267     while (*s0)
 268     {
 269         if (ct1 && strchr(ct1, *s0))
 270             break;
 271         if (ct2 && strchr(ct2, *s0))
 272             break;
 273         s1 = s0;
 274         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 275         if (**map != *CHR_SPACE)
 276             break;
 277         s0 = s1;
 278     }
 279     *src = s0;
 280     return *s0;
 281 }
 282
 283
 284 static void esc_str(char *out_buf, int out_size,
 285                     const char *in_buf, int in_size)
 286 {
 287     int k;
 288
 289     assert(out_buf);
 290     assert(in_buf);
 291     assert(out_size > 20);
 292     *out_buf = '\0';
 293     for (k = 0; k<in_size; k++)
 294     {
 295         int c = in_buf[k] & 0xff;
 296         int pc;
 297         if (c < 32 || c > 126)
 298             pc = '?';
 299         else
 300             pc = c;
 301         sprintf(out_buf +strlen(out_buf), "%02X:%c  ", c, pc);
 302         if (strlen(out_buf) > out_size-20)
 303         {
 304             strcat(out_buf, "..");
 305             break;
 306         }
 307     }
 308 }
 309
 310 #define REGEX_CHARS " []()|.*+?!"
 311
 312 /* term_100: handle term, where trunc = none(no operators at all) */
 313 static int term_100(ZebraMaps zebra_maps, int reg_type,
 314                     const char **src, char *dst, int space_split,
 315                     char *dst_term)
 316 {
 317     const char *s0;
 318     const char **map;
 319     int i = 0;
 320     int j = 0;
 321
 322     const char *space_start = 0;
 323     const char *space_end = 0;
 324
 325     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 326         return 0;
 327     s0 = *src;
 328     while (*s0)
 329     {
 330         const char *s1 = s0;
 331         int q_map_match = 0;
 332         map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 333                                 &q_map_match);
 334         if (space_split)
 335         {
 336             if (**map == *CHR_SPACE)
 337                 break;
 338         }
 339         else  /* complete subfield only. */
 340         {
 341             if (**map == *CHR_SPACE)
 342             {   /* save space mapping for later  .. */
 343                 space_start = s1;
 344                 space_end = s0;
 345                 continue;
 346             }
 347             else if (space_start)
 348             {   /* reload last space */
 349                 while (space_start < space_end)
 350                 {
 351                     if (strchr(REGEX_CHARS, *space_start))
 352                         dst[i++] = '\\';
 353                     dst_term[j++] = *space_start;
 354                     dst[i++] = *space_start++;
 355                 }
 356                 /* and reset */
 357                 space_start = space_end = 0;
 358             }
 359         }
 360         /* add non-space char */
 361         memcpy(dst_term+j, s1, s0 - s1);
 362         j += (s0 - s1);
 363         if (!q_map_match)
 364         {
 365             while (s1 < s0)
 366             {
 367                 if (strchr(REGEX_CHARS, *s1))
 368                     dst[i++] = '\\';
 369                 dst[i++] = *s1++;
 370             }
 371         }
 372         else
 373         {
 374             char tmpbuf[80];
 375             esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 376
 377             strcpy(dst + i, map[0]);
 378             i += strlen(map[0]);
 379         }
 380     }
 381     dst[i] = '\0';
 382     dst_term[j] = '\0';
 383     *src = s0;
 384     return i;
 385 }
 386
 387 /* term_101: handle term, where trunc = Process # */
 388 static int term_101(ZebraMaps zebra_maps, int reg_type,
 389                     const char **src, char *dst, int space_split,
 390                     char *dst_term)
 391 {
 392     const char *s0;
 393     const char **map;
 394     int i = 0;
 395     int j = 0;
 396
 397     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 398         return 0;
 399     s0 = *src;
 400     while (*s0)
 401     {
 402         if (*s0 == '#')
 403         {
 404             dst[i++] = '.';
 405             dst[i++] = '*';
 406             dst_term[j++] = *s0++;
 407         }
 408         else
 409         {
 410             const char *s1 = s0;
 411             int q_map_match = 0;
 412             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 413                                     &q_map_match);
 414             if (space_split && **map == *CHR_SPACE)
 415                 break;
 416
 417             /* add non-space char */
 418             memcpy(dst_term+j, s1, s0 - s1);
 419             j += (s0 - s1);
 420             if (!q_map_match)
 421             {
 422                 while (s1 < s0)
 423                 {
 424                     if (strchr(REGEX_CHARS, *s1))
 425                         dst[i++] = '\\';
 426                     dst[i++] = *s1++;
 427                 }
 428             }
 429             else
 430             {
 431                 char tmpbuf[80];
 432                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 433
 434                 strcpy(dst + i, map[0]);
 435                 i += strlen(map[0]);
 436             }
 437         }
 438     }
 439     dst[i] = '\0';
 440     dst_term[j++] = '\0';
 441     *src = s0;
 442     return i;
 443 }
 444
 445 /* term_103: handle term, where trunc = re-2 (regular expressions) */
 446 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 447                     char *dst, int *errors, int space_split,
 448                     char *dst_term)
 449 {
 450     int i = 0;
 451     int j = 0;
 452     const char *s0;
 453     const char **map;
 454
 455     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 456         return 0;
 457     s0 = *src;
 458     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 459         isdigit(((const unsigned char *)s0)[1]))
 460     {
 461         *errors = s0[1] - '0';
 462         s0 += 3;
 463         if (*errors > 3)
 464             *errors = 3;
 465     }
 466     while (*s0)
 467     {
 468         if (strchr("^\\()[].*+?|-", *s0))
 469         {
 470             dst_term[j++] = *s0;
 471             dst[i++] = *s0++;
 472         }
 473         else
 474         {
 475             const char *s1 = s0;
 476             int q_map_match = 0;
 477             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 478                                     &q_map_match);
 479             if (space_split && **map == *CHR_SPACE)
 480                 break;
 481
 482             /* add non-space char */
 483             memcpy(dst_term+j, s1, s0 - s1);
 484             j += (s0 - s1);
 485             if (!q_map_match)
 486             {
 487                 while (s1 < s0)
 488                 {
 489                     if (strchr(REGEX_CHARS, *s1))
 490                         dst[i++] = '\\';
 491                     dst[i++] = *s1++;
 492                 }
 493             }
 494             else
 495             {
 496                 char tmpbuf[80];
 497                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 498
 499                 strcpy(dst + i, map[0]);
 500                 i += strlen(map[0]);
 501             }
 502         }
 503     }
 504     dst[i] = '\0';
 505     dst_term[j] = '\0';
 506     *src = s0;
 507
 508     return i;
 509 }
 510
 511 /* term_103: handle term, where trunc = re-1 (regular expressions) */
 512 static int term_102(ZebraMaps zebra_maps, int reg_type, const char **src,
 513                     char *dst, int space_split, char *dst_term)
 514 {
 515     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 516                     dst_term);
 517 }
 518
 519
 520 /* term_104: handle term, where trunc = Process # and ! */
 521 static int term_104(ZebraMaps zebra_maps, int reg_type,
 522                     const char **src, char *dst, int space_split,
 523                     char *dst_term)
 524 {
 525     const char *s0, *s1;
 526     const char **map;
 527     int i = 0;
 528     int j = 0;
 529
 530     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 531         return 0;
 532     s0 = *src;
 533     while (*s0)
 534     {
 535         if (*s0 == '?')
 536         {
 537             dst_term[j++] = *s0++;
 538             if (*s0 >= '0' && *s0 <= '9')
 539             {
 540                 int limit = 0;
 541                 while (*s0 >= '0' && *s0 <= '9')
 542                 {
 543                     limit = limit * 10 + (*s0 - '0');
 544                     dst_term[j++] = *s0++;
 545                 }
 546                 if (limit > 20)
 547                     limit = 20;
 548                 while (--limit >= 0)
 549                 {
 550                     dst[i++] = '.';
 551                     dst[i++] = '?';
 552                 }
 553             }
 554             else
 555             {
 556                 dst[i++] = '.';
 557                 dst[i++] = '*';
 558             }
 559         }
 560         else if (*s0 == '*')
 561         {
 562             dst[i++] = '.';
 563             dst[i++] = '*';
 564             dst_term[j++] = *s0++;
 565         }
 566         else if (*s0 == '#')
 567         {
 568             dst[i++] = '.';
 569             dst_term[j++] = *s0++;
 570         }
 571         else
 572         {
 573             s1 = s0;
 574             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 575             if (space_split && **map == *CHR_SPACE)
 576                 break;
 577             while (s1 < s0)
 578             {
 579                 if (strchr(REGEX_CHARS, *s1))
 580                     dst[i++] = '\\';
 581                 dst_term[j++] = *s1;
 582                 dst[i++] = *s1++;
 583             }
 584         }
 585     }
 586     dst[i] = '\0';
 587     dst_term[j++] = '\0';
 588     *src = s0;
 589     return i;
 590 }
 591
 592 /* term_105/106: handle term, where trunc = Process * and ! and right trunc */
 593 static int term_105(ZebraMaps zebra_maps, int reg_type,
 594                     const char **src, char *dst, int space_split,
 595                     char *dst_term, int right_truncate)
 596 {
 597     const char *s0, *s1;
 598     const char **map;
 599     int i = 0;
 600     int j = 0;
 601
 602     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 603         return 0;
 604     s0 = *src;
 605     while (*s0)
 606     {
 607         if (*s0 == '*')
 608         {
 609             dst[i++] = '.';
 610             dst[i++] = '*';
 611             dst_term[j++] = *s0++;
 612         }
 613         else if (*s0 == '!')
 614         {
 615             dst[i++] = '.';
 616             dst_term[j++] = *s0++;
 617         }
 618         else
 619         {
 620             s1 = s0;
 621             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 622             if (space_split && **map == *CHR_SPACE)
 623                 break;
 624             while (s1 < s0)
 625             {
 626                 if (strchr(REGEX_CHARS, *s1))
 627                     dst[i++] = '\\';
 628                 dst_term[j++] = *s1;
 629                 dst[i++] = *s1++;
 630             }
 631         }
 632     }
 633     if (right_truncate)
 634     {
 635         dst[i++] = '.';
 636         dst[i++] = '*';
 637     }
 638     dst[i] = '\0';
 639
 640     dst_term[j++] = '\0';
 641     *src = s0;
 642     return i;
 643 }
 644
 645
 646 /* gen_regular_rel - generate regular expression from relation
 647  *  val:     border value (inclusive)
 648  *  islt:    1 if <=; 0 if >=.
 649  */
 650 static void gen_regular_rel(char *dst, int val, int islt)
 651 {
 652     int dst_p;
 653     int w, d, i;
 654     int pos = 0;
 655     char numstr[20];
 656
 657     yaz_log(YLOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 658     if (val >= 0)
 659     {
 660         if (islt)
 661             strcpy(dst, "(-[0-9]+|(");
 662         else
 663             strcpy(dst, "((");
 664     }
 665     else
 666     {
 667         if (!islt)
 668         {
 669             strcpy(dst, "([0-9]+|-(");
 670             dst_p = strlen(dst);
 671             islt = 1;
 672         }
 673         else
 674         {
 675             strcpy(dst, "(-(");
 676             islt = 0;
 677         }
 678         val = -val;
 679     }
 680     dst_p = strlen(dst);
 681     sprintf(numstr, "%d", val);
 682     for (w = strlen(numstr); --w >= 0; pos++)
 683     {
 684         d = numstr[w];
 685         if (pos > 0)
 686         {
 687             if (islt)
 688             {
 689                 if (d == '0')
 690                     continue;
 691                 d--;
 692             }
 693             else
 694             {
 695                 if (d == '9')
 696                     continue;
 697                 d++;
 698             }
 699         }
 700
 701         strcpy(dst + dst_p, numstr);
 702         dst_p = strlen(dst) - pos - 1;
 703
 704         if (islt)
 705         {
 706             if (d != '0')
 707             {
 708                 dst[dst_p++] = '[';
 709                 dst[dst_p++] = '0';
 710                 dst[dst_p++] = '-';
 711                 dst[dst_p++] = d;
 712                 dst[dst_p++] = ']';
 713             }
 714             else
 715                 dst[dst_p++] = d;
 716         }
 717         else
 718         {
 719             if (d != '9')
 720             {
 721                 dst[dst_p++] = '[';
 722                 dst[dst_p++] = d;
 723                 dst[dst_p++] = '-';
 724                 dst[dst_p++] = '9';
 725                 dst[dst_p++] = ']';
 726             }
 727             else
 728                 dst[dst_p++] = d;
 729         }
 730         for (i = 0; i<pos; i++)
 731         {
 732             dst[dst_p++] = '[';
 733             dst[dst_p++] = '0';
 734             dst[dst_p++] = '-';
 735             dst[dst_p++] = '9';
 736             dst[dst_p++] = ']';
 737         }
 738         dst[dst_p++] = '|';
 739     }
 740     dst[dst_p] = '\0';
 741     if (islt)
 742     {
 743         /* match everything less than 10^(pos-1) */
 744         strcat(dst, "0*");
 745         for (i = 1; i<pos; i++)
 746             strcat(dst, "[0-9]?");
 747     }
 748     else
 749     {
 750         /* match everything greater than 10^pos */
 751         for (i = 0; i <= pos; i++)
 752             strcat(dst, "[0-9]");
 753         strcat(dst, "[0-9]*");
 754     }
 755     strcat(dst, "))");
 756 }
 757
 758 void string_rel_add_char(char **term_p, const char *src, int *indx)
 759 {
 760     if (src[*indx] == '\\')
 761         *(*term_p)++ = src[(*indx)++];
 762     *(*term_p)++ = src[(*indx)++];
 763 }
 764
 765 /*
 766  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 767  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 768  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 769  *              ([^-a].*|a[^-b].*|ab[c-].*)
 770  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 771  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 772  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 773  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 774  */
 775 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 776                            const char **term_sub, char *term_dict,
 777                            oid_value attributeSet,
 778                            int reg_type, int space_split, char *term_dst,
 779                            int *error_code)
 780 {
 781     AttrType relation;
 782     int relation_value;
 783     int i;
 784     char *term_tmp = term_dict + strlen(term_dict);
 785     char term_component[2*IT_MAX_WORD+20];
 786
 787     attr_init(&relation, zapt, 2);
 788     relation_value = attr_find(&relation, NULL);
 789
 790     *error_code = 0;
 791     yaz_log(YLOG_DEBUG, "string relation value=%d", relation_value);
 792     switch (relation_value)
 793     {
 794     case 1:
 795         if (!term_100(zh->reg->zebra_maps, reg_type,
 796                       term_sub, term_component,
 797                       space_split, term_dst))
 798             return 0;
 799         yaz_log(log_level_rpn, "Relation <");
 800
 801         *term_tmp++ = '(';
 802         for (i = 0; term_component[i]; )
 803         {
 804             int j = 0;
 805
 806             if (i)
 807                 *term_tmp++ = '|';
 808             while (j < i)
 809                 string_rel_add_char(&term_tmp, term_component, &j);
 810
 811             *term_tmp++ = '[';
 812
 813             *term_tmp++ = '^';
 814             string_rel_add_char(&term_tmp, term_component, &i);
 815             *term_tmp++ = '-';
 816
 817             *term_tmp++ = ']';
 818             *term_tmp++ = '.';
 819             *term_tmp++ = '*';
 820
 821             if ((term_tmp - term_dict) > IT_MAX_WORD)
 822                 break;
 823         }
 824         *term_tmp++ = ')';
 825         *term_tmp = '\0';
 826         break;
 827     case 2:
 828         if (!term_100(zh->reg->zebra_maps, reg_type,
 829                       term_sub, term_component,
 830                       space_split, term_dst))
 831             return 0;
 832         yaz_log(log_level_rpn, "Relation <=");
 833
 834         *term_tmp++ = '(';
 835         for (i = 0; term_component[i]; )
 836         {
 837             int j = 0;
 838
 839             while (j < i)
 840                 string_rel_add_char(&term_tmp, term_component, &j);
 841             *term_tmp++ = '[';
 842
 843             *term_tmp++ = '^';
 844             string_rel_add_char(&term_tmp, term_component, &i);
 845             *term_tmp++ = '-';
 846
 847             *term_tmp++ = ']';
 848             *term_tmp++ = '.';
 849             *term_tmp++ = '*';
 850
 851             *term_tmp++ = '|';
 852
 853             if ((term_tmp - term_dict) > IT_MAX_WORD)
 854                 break;
 855         }
 856         for (i = 0; term_component[i]; )
 857             string_rel_add_char(&term_tmp, term_component, &i);
 858         *term_tmp++ = ')';
 859         *term_tmp = '\0';
 860         break;
 861     case 5:
 862         if (!term_100 (zh->reg->zebra_maps, reg_type,
 863                        term_sub, term_component, space_split, term_dst))
 864             return 0;
 865         yaz_log(log_level_rpn, "Relation >");
 866
 867         *term_tmp++ = '(';
 868         for (i = 0; term_component[i];)
 869         {
 870             int j = 0;
 871
 872             while (j < i)
 873                 string_rel_add_char(&term_tmp, term_component, &j);
 874             *term_tmp++ = '[';
 875
 876             *term_tmp++ = '^';
 877             *term_tmp++ = '-';
 878             string_rel_add_char(&term_tmp, term_component, &i);
 879
 880             *term_tmp++ = ']';
 881             *term_tmp++ = '.';
 882             *term_tmp++ = '*';
 883
 884             *term_tmp++ = '|';
 885
 886             if ((term_tmp - term_dict) > IT_MAX_WORD)
 887                 break;
 888         }
 889         for (i = 0; term_component[i];)
 890             string_rel_add_char(&term_tmp, term_component, &i);
 891         *term_tmp++ = '.';
 892         *term_tmp++ = '+';
 893         *term_tmp++ = ')';
 894         *term_tmp = '\0';
 895         break;
 896     case 4:
 897         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 898                       term_component, space_split, term_dst))
 899             return 0;
 900         yaz_log(log_level_rpn, "Relation >=");
 901
 902         *term_tmp++ = '(';
 903         for (i = 0; term_component[i];)
 904         {
 905             int j = 0;
 906
 907             if (i)
 908                 *term_tmp++ = '|';
 909             while (j < i)
 910                 string_rel_add_char(&term_tmp, term_component, &j);
 911             *term_tmp++ = '[';
 912
 913             if (term_component[i+1])
 914             {
 915                 *term_tmp++ = '^';
 916                 *term_tmp++ = '-';
 917                 string_rel_add_char(&term_tmp, term_component, &i);
 918             }
 919             else
 920             {
 921                 string_rel_add_char(&term_tmp, term_component, &i);
 922                 *term_tmp++ = '-';
 923             }
 924             *term_tmp++ = ']';
 925             *term_tmp++ = '.';
 926             *term_tmp++ = '*';
 927
 928             if ((term_tmp - term_dict) > IT_MAX_WORD)
 929                 break;
 930         }
 931         *term_tmp++ = ')';
 932         *term_tmp = '\0';
 933         break;
 934     case 3:
 935     case 102:
 936     case -1:
 937         yaz_log(log_level_rpn, "Relation =");
 938         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 939                       term_component, space_split, term_dst))
 940             return 0;
 941         strcat(term_tmp, "(");
 942         strcat(term_tmp, term_component);
 943         strcat(term_tmp, ")");
 944         break;
 945     default:
 946         *error_code = 117;
 947         return 0;
 948     }
 949     return 1;
 950 }
 951
 952 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 953                              const char **term_sub,
 954                              oid_value attributeSet, NMEM stream,
 955                              struct grep_info *grep_info,
 956                              int reg_type, int complete_flag,
 957                              int num_bases, char **basenames,
 958                              char *term_dst, int xpath_use);
 959
 960 static ZEBRA_RES term_trunc(ZebraHandle zh,
 961                             Z_AttributesPlusTerm *zapt,
 962                             const char **term_sub,
 963                             oid_value attributeSet, NMEM stream,
 964                             struct grep_info *grep_info,
 965                             int reg_type, int complete_flag,
 966                             int num_bases, char **basenames,
 967                             char *term_dst,
 968                             const char *rank_type, int xpath_use,
 969                             NMEM rset_nmem,
 970                             RSET *rset,
 971                             struct rset_key_control *kc)
 972 {
 973     ZEBRA_RES res;
 974     *rset = 0;
 975     grep_info->isam_p_indx = 0;
 976     res = string_term(zh, zapt, term_sub, attributeSet, stream, grep_info,
 977                       reg_type, complete_flag, num_bases, basenames,
 978                       term_dst, xpath_use);
 979     if (res != ZEBRA_OK)
 980         return res;
 981     if (!*term_sub)  /* no more terms ? */
 982         return res;
 983     yaz_log(log_level_rpn, "term: %s", term_dst);
 984     *rset = rset_trunc(zh, grep_info->isam_p_buf,
 985                        grep_info->isam_p_indx, term_dst,
 986                        strlen(term_dst), rank_type, 1 /* preserve pos */,
 987                        zapt->term->which, rset_nmem,
 988                        kc, kc->scope);
 989     if (!*rset)
 990         return ZEBRA_FAIL;
 991     return ZEBRA_OK;
 992 }
 993
 994 static char *nmem_strdup_i(NMEM nmem, int v)
 995 {
 996     char val_str[64];
 997     sprintf(val_str, "%d", v);
 998     return nmem_strdup(nmem, val_str);
 999 }
1000
1001 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1002                              const char **term_sub,
1003                              oid_value attributeSet, NMEM stream,
1004                              struct grep_info *grep_info,
1005                              int reg_type, int complete_flag,
1006                              int num_bases, char **basenames,
1007                              char *term_dst, int xpath_use)
1008 {
1009     char term_dict[2*IT_MAX_WORD+4000];
1010     int j, r, base_no;
1011     AttrType truncation;
1012     int truncation_value;
1013     AttrType use;
1014     int use_value;
1015     const char *use_string = 0;
1016     oid_value curAttributeSet = attributeSet;
1017     const char *termp;
1018     struct rpn_char_map_info rcmi;
1019     int space_split = complete_flag ? 0 : 1;
1020
1021     int bases_ok = 0;     /* no of databases with OK attribute */
1022     int errCode = 0;      /* err code (if any is not OK) */
1023     char *errString = 0;  /* addinfo */
1024
1025     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1026     attr_init(&use, zapt, 1);
1027     use_value = attr_find_ex(&use, &curAttributeSet, &use_string);
1028     yaz_log(log_level_rpn, "string_term, use value %d", use_value);
1029     attr_init(&truncation, zapt, 5);
1030     truncation_value = attr_find(&truncation, NULL);
1031     yaz_log(log_level_rpn, "truncation value %d", truncation_value);
1032
1033     if (use_value == -1)    /* no attribute - assumy "any" */
1034         use_value = 1016;
1035     for (base_no = 0; base_no < num_bases; base_no++)
1036     {
1037         int ord = -1;
1038         int attr_ok = 0;
1039         int regex_range = 0;
1040         int init_pos = 0;
1041         attent attp;
1042         data1_local_attribute id_xpath_attr;
1043         data1_local_attribute *local_attr;
1044         int max_pos, prefix_len = 0;
1045         int relation_error;
1046
1047         termp = *term_sub;
1048
1049         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1050         {
1051             zh->errCode = YAZ_BIB1_DATABASE_UNAVAILABLE;
1052             zh->errString = basenames[base_no];
1053             return ZEBRA_FAIL;
1054         }
1055         if (xpath_use > 0 && use_value == -2)
1056         {
1057             /* xpath mode and we have a string attribute */
1058             attp.local_attributes = &id_xpath_attr;
1059             attp.attset_ordinal = VAL_IDXPATH;
1060             id_xpath_attr.next = 0;
1061
1062             use_value = xpath_use;  /* xpath_use as use-attribute now */
1063             id_xpath_attr.local = use_value;
1064         }
1065         else if (curAttributeSet == VAL_IDXPATH && use_value >= 0)
1066         {
1067             /* X-Path attribute, use numeric value directly */
1068             attp.local_attributes = &id_xpath_attr;
1069             attp.attset_ordinal = VAL_IDXPATH;
1070             id_xpath_attr.next = 0;
1071             id_xpath_attr.local = use_value;
1072         }
1073         else if (use_string &&
1074                  (ord = zebraExplain_lookup_attr_str(zh->reg->zei,
1075                                                      use_string)) >= 0)
1076         {
1077             /* we have a match for a raw string attribute */
1078             char ord_buf[32];
1079             int i, ord_len;
1080
1081             if (prefix_len)
1082                 term_dict[prefix_len++] = '|';
1083             else
1084                 term_dict[prefix_len++] = '(';
1085
1086             ord_len = key_SU_encode (ord, ord_buf);
1087             for (i = 0; i<ord_len; i++)
1088             {
1089                 term_dict[prefix_len++] = 1;
1090                 term_dict[prefix_len++] = ord_buf[i];
1091             }
1092             attp.local_attributes = 0;  /* no more attributes */
1093         }
1094         else
1095         {
1096             /* lookup in the .att files . Allow string as well */
1097             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1098                                       use_string)))
1099             {
1100                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1101                         curAttributeSet, use_value, r);
1102                 if (r == -1)
1103                 {
1104                     /* set was found, but value wasn't defined */
1105                     errCode = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
1106                     if (use_string)
1107                         errString = nmem_strdup(stream, use_string);
1108                     else
1109                         errString = nmem_strdup_i (stream, use_value);
1110                 }
1111                 else
1112                 {
1113                     int oid[OID_SIZE];
1114                     struct oident oident;
1115
1116                     oident.proto = PROTO_Z3950;
1117                     oident.oclass = CLASS_ATTSET;
1118                     oident.value = curAttributeSet;
1119                     oid_ent_to_oid (&oident, oid);
1120
1121                     errCode = YAZ_BIB1_UNSUPP_ATTRIBUTE_SET;
1122                     errString = nmem_strdup(stream, oident.desc);
1123                 }
1124                 continue;
1125             }
1126         }
1127         for (local_attr = attp.local_attributes; local_attr;
1128              local_attr = local_attr->next)
1129         {
1130             char ord_buf[32];
1131             int i, ord_len;
1132
1133             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1134                                               attp.attset_ordinal,
1135                                               local_attr->local);
1136             if (ord < 0)
1137                 continue;
1138             if (prefix_len)
1139                 term_dict[prefix_len++] = '|';
1140             else
1141                 term_dict[prefix_len++] = '(';
1142
1143             ord_len = key_SU_encode (ord, ord_buf);
1144             for (i = 0; i<ord_len; i++)
1145             {
1146                 term_dict[prefix_len++] = 1;
1147                 term_dict[prefix_len++] = ord_buf[i];
1148             }
1149         }
1150         bases_ok++;
1151         if (prefix_len)
1152             attr_ok = 1;
1153
1154         term_dict[prefix_len++] = ')';
1155         term_dict[prefix_len++] = 1;
1156         term_dict[prefix_len++] = reg_type;
1157         yaz_log(log_level_rpn, "reg_type = %d", term_dict[prefix_len-1]);
1158         term_dict[prefix_len] = '\0';
1159         j = prefix_len;
1160         switch (truncation_value)
1161         {
1162         case -1:         /* not specified */
1163         case 100:        /* do not truncate */
1164             if (!string_relation (zh, zapt, &termp, term_dict,
1165                                   attributeSet,
1166                                   reg_type, space_split, term_dst,
1167                                   &relation_error))
1168             {
1169                 if (relation_error)
1170                 {
1171                     zh->errCode = relation_error;
1172                     return ZEBRA_FAIL;
1173                 }
1174                 *term_sub = 0;
1175                 return ZEBRA_OK;
1176             }
1177             break;
1178         case 1:          /* right truncation */
1179             term_dict[j++] = '(';
1180             if (!term_100(zh->reg->zebra_maps, reg_type,
1181                           &termp, term_dict + j, space_split, term_dst))
1182             {
1183                 *term_sub = 0;
1184                 return ZEBRA_OK;
1185             }
1186             strcat(term_dict, ".*)");
1187             break;
1188         case 2:          /* keft truncation */
1189             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1190             if (!term_100(zh->reg->zebra_maps, reg_type,
1191                           &termp, term_dict + j, space_split, term_dst))
1192             {
1193                 *term_sub = 0;
1194                 return ZEBRA_OK;
1195             }
1196             strcat(term_dict, ")");
1197             break;
1198         case 3:          /* left&right truncation */
1199             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1200             if (!term_100(zh->reg->zebra_maps, reg_type,
1201                           &termp, term_dict + j, space_split, term_dst))
1202             {
1203                 *term_sub = 0;
1204                 return ZEBRA_OK;
1205             }
1206             strcat(term_dict, ".*)");
1207             break;
1208         case 101:        /* process # in term */
1209             term_dict[j++] = '(';
1210             if (!term_101(zh->reg->zebra_maps, reg_type,
1211                           &termp, term_dict + j, space_split, term_dst))
1212             {
1213                 *term_sub = 0;
1214                 return ZEBRA_OK;
1215             }
1216             strcat(term_dict, ")");
1217             break;
1218         case 102:        /* Regexp-1 */
1219             term_dict[j++] = '(';
1220             if (!term_102(zh->reg->zebra_maps, reg_type,
1221                           &termp, term_dict + j, space_split, term_dst))
1222             {
1223                 *term_sub = 0;
1224                 return ZEBRA_OK;
1225             }
1226             strcat(term_dict, ")");
1227             break;
1228         case 103:       /* Regexp-2 */
1229             regex_range = 1;
1230             term_dict[j++] = '(';
1231             init_pos = 2;
1232             if (!term_103(zh->reg->zebra_maps, reg_type,
1233                           &termp, term_dict + j, &regex_range,
1234                           space_split, term_dst))
1235             {
1236                 *term_sub = 0;
1237                 return ZEBRA_OK;
1238             }
1239             strcat(term_dict, ")");
1240             break;
1241         case 104:        /* process # and ! in term */
1242             term_dict[j++] = '(';
1243             if (!term_104(zh->reg->zebra_maps, reg_type,
1244                           &termp, term_dict + j, space_split, term_dst))
1245             {
1246                 *term_sub = 0;
1247                 return ZEBRA_OK;
1248             }
1249             strcat(term_dict, ")");
1250             break;
1251         case 105:        /* process * and ! in term */
1252             term_dict[j++] = '(';
1253             if (!term_105(zh->reg->zebra_maps, reg_type,
1254                           &termp, term_dict + j, space_split, term_dst, 1))
1255             {
1256                 *term_sub = 0;
1257                 return ZEBRA_OK;
1258             }
1259             strcat(term_dict, ")");
1260             break;
1261         case 106:        /* process * and ! in term */
1262             term_dict[j++] = '(';
1263             if (!term_105(zh->reg->zebra_maps, reg_type,
1264                           &termp, term_dict + j, space_split, term_dst, 0))
1265             {
1266                 *term_sub = 0;
1267                 return ZEBRA_OK;
1268             }
1269             strcat(term_dict, ")");
1270             break;
1271         default:
1272             zh->errCode = YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE;
1273             zh->errString = nmem_strdup_i(stream, truncation_value);
1274             return ZEBRA_FAIL;
1275         }
1276         if (attr_ok)
1277         {
1278             char buf[80];
1279             const char *input = term_dict + prefix_len;
1280             esc_str(buf, sizeof(buf), input, strlen(input));
1281         }
1282         if (attr_ok)
1283         {
1284             yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_dict+prefix_len);
1285             r = dict_lookup_grep(zh->reg->dict, term_dict, regex_range,
1286                                  grep_info, &max_pos, init_pos,
1287                                  grep_handle);
1288             if (r)
1289                 yaz_log(YLOG_WARN, "dict_lookup_grep fail %d", r);
1290         }
1291     }
1292     if (!bases_ok)
1293     {
1294         zh->errCode = errCode;
1295         zh->errString = errString;
1296         return ZEBRA_FAIL;
1297     }
1298     *term_sub = termp;
1299     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1300     return ZEBRA_OK;
1301 }
1302
1303
1304 /* convert APT search term to UTF8 */
1305 static ZEBRA_RES zapt_term_to_utf8(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1306                                    char *termz)
1307 {
1308     size_t sizez;
1309     Z_Term *term = zapt->term;
1310
1311     switch (term->which)
1312     {
1313     case Z_Term_general:
1314         if (zh->iconv_to_utf8 != 0)
1315         {
1316             char *inbuf = term->u.general->buf;
1317             size_t inleft = term->u.general->len;
1318             char *outbuf = termz;
1319             size_t outleft = IT_MAX_WORD-1;
1320             size_t ret;
1321
1322             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1323                         &outbuf, &outleft);
1324             if (ret == (size_t)(-1))
1325             {
1326                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1327                 zh->errCode =
1328                     YAZ_BIB1_QUERY_TERM_INCLUDES_CHARS_THAT_DO_NOT_TRANSLATE_INTO_;
1329                 return -1;
1330             }
1331             *outbuf = 0;
1332         }
1333         else
1334         {
1335             sizez = term->u.general->len;
1336             if (sizez > IT_MAX_WORD-1)
1337                 sizez = IT_MAX_WORD-1;
1338             memcpy (termz, term->u.general->buf, sizez);
1339             termz[sizez] = '\0';
1340         }
1341         break;
1342     case Z_Term_characterString:
1343         sizez = strlen(term->u.characterString);
1344         if (sizez > IT_MAX_WORD-1)
1345             sizez = IT_MAX_WORD-1;
1346         memcpy (termz, term->u.characterString, sizez);
1347         termz[sizez] = '\0';
1348         break;
1349     default:
1350         zh->errCode = YAZ_BIB1_UNSUPP_CODED_VALUE_FOR_TERM;
1351         return ZEBRA_FAIL;
1352     }
1353     return ZEBRA_OK;
1354 }
1355
1356 /* convert APT SCAN term to internal cmap */
1357 static ZEBRA_RES trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1358                                  char *termz, int reg_type)
1359 {
1360     char termz0[IT_MAX_WORD];
1361
1362     if (zapt_term_to_utf8(zh, zapt, termz0) == ZEBRA_FAIL)
1363         return ZEBRA_FAIL;    /* error */
1364     else
1365     {
1366         const char **map;
1367         const char *cp = (const char *) termz0;
1368         const char *cp_end = cp + strlen(cp);
1369         const char *src;
1370         int i = 0;
1371         const char *space_map = NULL;
1372         int len;
1373
1374         while ((len = (cp_end - cp)) > 0)
1375         {
1376             map = zebra_maps_input(zh->reg->zebra_maps, reg_type, &cp, len, 0);
1377             if (**map == *CHR_SPACE)
1378                 space_map = *map;
1379             else
1380             {
1381                 if (i && space_map)
1382                     for (src = space_map; *src; src++)
1383                         termz[i++] = *src;
1384                 space_map = NULL;
1385                 for (src = *map; *src; src++)
1386                     termz[i++] = *src;
1387             }
1388         }
1389         termz[i] = '\0';
1390     }
1391     return ZEBRA_OK;
1392 }
1393
1394 char *normalize_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1395                      const char *termz, NMEM stream, unsigned reg_id)
1396 {
1397     WRBUF wrbuf = 0;
1398     AttrType truncation;
1399     int truncation_value;
1400     char *ex_list = 0;
1401
1402     attr_init(&truncation, zapt, 5);
1403     truncation_value = attr_find(&truncation, NULL);
1404
1405     switch (truncation_value)
1406     {
1407     default:
1408         ex_list = "";
1409         break;
1410     case 101:
1411         ex_list = "#";
1412         break;
1413     case 102:
1414     case 103:
1415         ex_list = 0;
1416         break;
1417     case 104:
1418         ex_list = "!#";
1419         break;
1420     case 105:
1421         ex_list = "!*";
1422         break;
1423     }
1424     if (ex_list)
1425         wrbuf = zebra_replace(zh->reg->zebra_maps, reg_id, ex_list,
1426                               termz, strlen(termz));
1427     if (!wrbuf)
1428         return nmem_strdup(stream, termz);
1429     else
1430     {
1431         char *buf = (char*) nmem_malloc(stream, wrbuf_len(wrbuf)+1);
1432         memcpy (buf, wrbuf_buf(wrbuf), wrbuf_len(wrbuf));
1433         buf[wrbuf_len(wrbuf)] = '\0';
1434         return buf;
1435     }
1436 }
1437
1438 static void grep_info_delete(struct grep_info *grep_info)
1439 {
1440 #ifdef TERM_COUNT
1441     xfree(grep_info->term_no);
1442 #endif
1443     xfree(grep_info->isam_p_buf);
1444 }
1445
1446 static int grep_info_prepare(ZebraHandle zh,
1447                              Z_AttributesPlusTerm *zapt,
1448                              struct grep_info *grep_info,
1449                              int reg_type,
1450                              NMEM stream)
1451 {
1452     AttrType termset;
1453     int termset_value_numeric;
1454     const char *termset_value_string;
1455
1456 #ifdef TERM_COUNT
1457     grep_info->term_no = 0;
1458 #endif
1459     grep_info->isam_p_size = 0;
1460     grep_info->isam_p_buf = NULL;
1461     grep_info->zh = zh;
1462     grep_info->reg_type = reg_type;
1463     grep_info->termset = 0;
1464
1465     if (!zapt)
1466         return 0;
1467     attr_init(&termset, zapt, 8);
1468     termset_value_numeric =
1469         attr_find_ex(&termset, NULL, &termset_value_string);
1470     if (termset_value_numeric != -1)
1471     {
1472         char resname[32];
1473         const char *termset_name = 0;
1474         if (termset_value_numeric != -2)
1475         {
1476
1477             sprintf(resname, "%d", termset_value_numeric);
1478             termset_name = resname;
1479         }
1480         else
1481             termset_name = termset_value_string;
1482         yaz_log(log_level_rpn, "creating termset set %s", termset_name);
1483         grep_info->termset = resultSetAdd(zh, termset_name, 1);
1484         if (!grep_info->termset)
1485         {
1486             zh->errCode = YAZ_BIB1_ILLEGAL_RESULT_SET_NAME;
1487             zh->errString = nmem_strdup(stream, termset_name);
1488             return -1;
1489         }
1490     }
1491     return 0;
1492 }
1493
1494
1495 static ZEBRA_RES term_list_trunc(ZebraHandle zh,
1496                                  Z_AttributesPlusTerm *zapt,
1497                                  const char *termz_org,
1498                                  oid_value attributeSet,
1499                                  NMEM stream,
1500                                  int reg_type, int complete_flag,
1501                                  const char *rank_type, int xpath_use,
1502                                  int num_bases, char **basenames,
1503                                  NMEM rset_nmem,
1504                                  RSET **result_sets, int *num_result_sets,
1505                                  struct rset_key_control *kc)
1506 {
1507     char term_dst[IT_MAX_WORD+1];
1508     struct grep_info grep_info;
1509     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1510     const char *termp = termz;
1511     int alloc_sets = 0;
1512
1513     *num_result_sets = 0;
1514     *term_dst = 0;
1515     if (grep_info_prepare(zh, zapt, &grep_info, reg_type, stream))
1516         return ZEBRA_FAIL;
1517     while(1)
1518     {
1519         ZEBRA_RES res;
1520
1521         if (alloc_sets == *num_result_sets)
1522         {
1523             int add = 10;
1524             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1525                                               sizeof(*rnew));
1526             if (alloc_sets)
1527                 memcpy(rnew, *result_sets, alloc_sets * sizeof(*rnew));
1528             alloc_sets = alloc_sets + add;
1529             *result_sets = rnew;
1530         }
1531         res = term_trunc(zh, zapt, &termp, attributeSet,
1532                          stream, &grep_info,
1533                          reg_type, complete_flag,
1534                          num_bases, basenames,
1535                          term_dst, rank_type,
1536                          xpath_use, rset_nmem,
1537                          &(*result_sets)[*num_result_sets],
1538                          kc);
1539         if (res != ZEBRA_OK)
1540         {
1541             int i;
1542             for (i = 0; i < *num_result_sets; i++)
1543                 rset_delete((*result_sets)[i]);
1544             grep_info_delete (&grep_info);
1545             return res;
1546         }
1547         if ((*result_sets)[*num_result_sets] == 0)
1548             break;
1549         (*num_result_sets)++;
1550     }
1551     grep_info_delete(&grep_info);
1552     return ZEBRA_OK;
1553 }
1554
1555 static ZEBRA_RES rpn_search_APT_phrase(ZebraHandle zh,
1556                                        Z_AttributesPlusTerm *zapt,
1557                                        const char *termz_org,
1558                                        oid_value attributeSet,
1559                                        NMEM stream,
1560                                        int reg_type, int complete_flag,
1561                                        const char *rank_type, int xpath_use,
1562                                        int num_bases, char **basenames,
1563                                        NMEM rset_nmem,
1564                                        RSET *rset,
1565                                        struct rset_key_control *kc)
1566 {
1567     RSET *result_sets = 0;
1568     int num_result_sets = 0;
1569     ZEBRA_RES res =
1570         term_list_trunc(zh, zapt, termz_org, attributeSet,
1571                         stream, reg_type, complete_flag,
1572                         rank_type, xpath_use,
1573                         num_bases, basenames,
1574                         rset_nmem,
1575                         &result_sets, &num_result_sets, kc);
1576     if (res != ZEBRA_OK)
1577         return res;
1578     if (num_result_sets == 0)
1579         *rset = rsnull_create (rset_nmem, kc);
1580     else if (num_result_sets == 1)
1581         *rset = result_sets[0];
1582     else
1583         *rset = rsprox_create(rset_nmem, kc, kc->scope,
1584                               num_result_sets, result_sets,
1585                               1 /* ordered */, 0 /* exclusion */,
1586                               3 /* relation */, 1 /* distance */);
1587     if (!*rset)
1588         return ZEBRA_FAIL;
1589     return ZEBRA_OK;
1590 }
1591
1592 static ZEBRA_RES rpn_search_APT_or_list(ZebraHandle zh,
1593                                         Z_AttributesPlusTerm *zapt,
1594                                         const char *termz_org,
1595                                         oid_value attributeSet,
1596                                         NMEM stream,
1597                                         int reg_type, int complete_flag,
1598                                         const char *rank_type,
1599                                         int xpath_use,
1600                                         int num_bases, char **basenames,
1601                                         NMEM rset_nmem,
1602                                         RSET *rset,
1603                                         struct rset_key_control *kc)
1604 {
1605     RSET *result_sets = 0;
1606     int num_result_sets = 0;
1607     ZEBRA_RES res =
1608         term_list_trunc(zh, zapt, termz_org, attributeSet,
1609                         stream, reg_type, complete_flag,
1610                         rank_type, xpath_use,
1611                         num_bases, basenames,
1612                         rset_nmem,
1613                         &result_sets, &num_result_sets, kc);
1614     if (res != ZEBRA_OK)
1615         return res;
1616     if (num_result_sets == 0)
1617         *rset = rsnull_create (rset_nmem, kc);
1618     else if (num_result_sets == 1)
1619         *rset = result_sets[0];
1620     else
1621         *rset = rsmulti_or_create(rset_nmem, kc, kc->scope,
1622                                   num_result_sets, result_sets);
1623     if (!*rset)
1624         return ZEBRA_FAIL;
1625     return ZEBRA_OK;
1626 }
1627
1628 static ZEBRA_RES rpn_search_APT_and_list(ZebraHandle zh,
1629                                          Z_AttributesPlusTerm *zapt,
1630                                          const char *termz_org,
1631                                          oid_value attributeSet,
1632                                          NMEM stream,
1633                                          int reg_type, int complete_flag,
1634                                          const char *rank_type,
1635                                          int xpath_use,
1636                                          int num_bases, char **basenames,
1637                                          NMEM rset_nmem,
1638                                          RSET *rset,
1639                                          struct rset_key_control *kc)
1640 {
1641     RSET *result_sets = 0;
1642     int num_result_sets = 0;
1643     ZEBRA_RES res =
1644         term_list_trunc(zh, zapt, termz_org, attributeSet,
1645                         stream, reg_type, complete_flag,
1646                         rank_type, xpath_use,
1647                         num_bases, basenames,
1648                         rset_nmem,
1649                         &result_sets, &num_result_sets,
1650                         kc);
1651     if (res != ZEBRA_OK)
1652         return res;
1653     if (num_result_sets == 0)
1654         *rset = rsnull_create (rset_nmem, kc);
1655     else if (num_result_sets == 1)
1656         *rset = result_sets[0];
1657     else
1658         *rset = rsmulti_and_create(rset_nmem, kc, kc->scope,
1659                                    num_result_sets, result_sets);
1660     if (!*rset)
1661         return ZEBRA_FAIL;
1662     return ZEBRA_OK;
1663 }
1664
1665 static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1666                             const char **term_sub,
1667                             char *term_dict,
1668                             oid_value attributeSet,
1669                             struct grep_info *grep_info,
1670                             int *max_pos,
1671                             int reg_type,
1672                             char *term_dst,
1673                             int *error_code)
1674 {
1675     AttrType relation;
1676     int relation_value;
1677     int term_value;
1678     int r;
1679     char *term_tmp = term_dict + strlen(term_dict);
1680
1681     *error_code = 0;
1682     attr_init(&relation, zapt, 2);
1683     relation_value = attr_find(&relation, NULL);
1684
1685     yaz_log(log_level_rpn, "numeric relation value=%d", relation_value);
1686
1687     if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1688                   term_dst))
1689         return 0;
1690     term_value = atoi (term_tmp);
1691     switch (relation_value)
1692     {
1693     case 1:
1694         yaz_log(log_level_rpn, "Relation <");
1695         gen_regular_rel(term_tmp, term_value-1, 1);
1696         break;
1697     case 2:
1698         yaz_log(log_level_rpn, "Relation <=");
1699         gen_regular_rel(term_tmp, term_value, 1);
1700         break;
1701     case 4:
1702         yaz_log(log_level_rpn, "Relation >=");
1703         gen_regular_rel(term_tmp, term_value, 0);
1704         break;
1705     case 5:
1706         yaz_log(log_level_rpn, "Relation >");
1707         gen_regular_rel(term_tmp, term_value+1, 0);
1708         break;
1709     case -1:
1710     case 3:
1711         yaz_log(log_level_rpn, "Relation =");
1712         sprintf(term_tmp, "(0*%d)", term_value);
1713         break;
1714     default:
1715         *error_code = 117;
1716         return 0;
1717     }
1718     yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_tmp);
1719     r = dict_lookup_grep(zh->reg->dict, term_dict, 0, grep_info, max_pos,
1720                           0, grep_handle);
1721     if (r)
1722         yaz_log(YLOG_WARN, "dict_lookup_grep fail, rel = gt: %d", r);
1723     yaz_log(log_level_rpn, "%d positions", grep_info->isam_p_indx);
1724     return 1;
1725 }
1726
1727 static ZEBRA_RES numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1728                               const char **term_sub,
1729                               oid_value attributeSet,
1730                               struct grep_info *grep_info,
1731                               int reg_type, int complete_flag,
1732                               int num_bases, char **basenames,
1733                               char *term_dst, int xpath_use, NMEM stream)
1734 {
1735     char term_dict[2*IT_MAX_WORD+2];
1736     int r, base_no;
1737     AttrType use;
1738     int use_value;
1739     const char *use_string = 0;
1740     oid_value curAttributeSet = attributeSet;
1741     const char *termp;
1742     struct rpn_char_map_info rcmi;
1743
1744     int bases_ok = 0;     /* no of databases with OK attribute */
1745     int errCode = 0;      /* err code (if any is not OK) */
1746     char *errString = 0;  /* addinfo */
1747
1748     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1749     attr_init(&use, zapt, 1);
1750     use_value = attr_find_ex(&use, &curAttributeSet, &use_string);
1751
1752     if (use_value == -1)
1753         use_value = 1016;
1754
1755     for (base_no = 0; base_no < num_bases; base_no++)
1756     {
1757         attent attp;
1758         data1_local_attribute id_xpath_attr;
1759         data1_local_attribute *local_attr;
1760         int max_pos, prefix_len = 0;
1761         int relation_error = 0;
1762
1763         termp = *term_sub;
1764         if (use_value == -2)  /* string attribute (assume IDXPATH/any) */
1765         {
1766             use_value = xpath_use;
1767             attp.local_attributes = &id_xpath_attr;
1768             attp.attset_ordinal = VAL_IDXPATH;
1769             id_xpath_attr.next = 0;
1770             id_xpath_attr.local = use_value;
1771         }
1772         else if (curAttributeSet == VAL_IDXPATH)
1773         {
1774             attp.local_attributes = &id_xpath_attr;
1775             attp.attset_ordinal = VAL_IDXPATH;
1776             id_xpath_attr.next = 0;
1777             id_xpath_attr.local = use_value;
1778         }
1779         else
1780         {
1781             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1782                                             use_string)))
1783             {
1784                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1785                       curAttributeSet, use_value, r);
1786                 if (r == -1)
1787                 {
1788                     errCode = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
1789                     if (use_string)
1790                         errString = nmem_strdup(stream, use_string);
1791                     else
1792                         errString = nmem_strdup_i (stream, use_value);
1793                 }
1794                 else
1795                     errCode = YAZ_BIB1_UNSUPP_ATTRIBUTE_SET;
1796                 continue;
1797             }
1798         }
1799         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1800         {
1801             zh->errCode = YAZ_BIB1_DATABASE_UNAVAILABLE;
1802             zh->errString = basenames[base_no];
1803             return -1;
1804         }
1805         for (local_attr = attp.local_attributes; local_attr;
1806              local_attr = local_attr->next)
1807         {
1808             int ord;
1809             char ord_buf[32];
1810             int i, ord_len;
1811
1812             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1813                                               attp.attset_ordinal,
1814                                               local_attr->local);
1815             if (ord < 0)
1816                 continue;
1817             if (prefix_len)
1818                 term_dict[prefix_len++] = '|';
1819             else
1820                 term_dict[prefix_len++] = '(';
1821
1822             ord_len = key_SU_encode (ord, ord_buf);
1823             for (i = 0; i<ord_len; i++)
1824             {
1825                 term_dict[prefix_len++] = 1;
1826                 term_dict[prefix_len++] = ord_buf[i];
1827             }
1828         }
1829         if (!prefix_len)
1830         {
1831             errCode = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
1832             errString = nmem_strdup_i(stream, use_value);
1833             continue;
1834         }
1835         bases_ok++;
1836         term_dict[prefix_len++] = ')';
1837         term_dict[prefix_len++] = 1;
1838         term_dict[prefix_len++] = reg_type;
1839         yaz_log(YLOG_DEBUG, "reg_type = %d", term_dict[prefix_len-1]);
1840         term_dict[prefix_len] = '\0';
1841         if (!numeric_relation(zh, zapt, &termp, term_dict,
1842                               attributeSet, grep_info, &max_pos, reg_type,
1843                               term_dst, &relation_error))
1844         {
1845             if (relation_error)
1846             {
1847                 zh->errCode = relation_error;
1848                 zh->errString = 0;
1849                 return ZEBRA_FAIL;
1850             }
1851             *term_sub = 0;
1852             return ZEBRA_OK;
1853         }
1854     }
1855     if (!bases_ok)
1856     {
1857         zh->errCode = errCode;
1858         zh->errString = errString;
1859         return ZEBRA_FAIL;
1860     }
1861     *term_sub = termp;
1862     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1863     return ZEBRA_OK;
1864 }
1865
1866 static ZEBRA_RES rpn_search_APT_numeric(ZebraHandle zh,
1867                                         Z_AttributesPlusTerm *zapt,
1868                                         const char *termz,
1869                                         oid_value attributeSet,
1870                                         NMEM stream,
1871                                         int reg_type, int complete_flag,
1872                                         const char *rank_type, int xpath_use,
1873                                         int num_bases, char **basenames,
1874                                         NMEM rset_nmem,
1875                                         RSET *rset,
1876                                         struct rset_key_control *kc)
1877 {
1878     char term_dst[IT_MAX_WORD+1];
1879     const char *termp = termz;
1880     RSET *result_sets = 0;
1881     int num_result_sets = 0;
1882     ZEBRA_RES res;
1883     struct grep_info grep_info;
1884     int alloc_sets = 0;
1885
1886     yaz_log(log_level_rpn, "APT_numeric t='%s'", termz);
1887     if (grep_info_prepare(zh, zapt, &grep_info, reg_type, stream))
1888         return ZEBRA_FAIL;
1889     while (1)
1890     {
1891         if (alloc_sets == num_result_sets)
1892         {
1893             int add = 10;
1894             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1895                                               sizeof(*rnew));
1896             if (alloc_sets)
1897                 memcpy(rnew, result_sets, alloc_sets * sizeof(*rnew));
1898             alloc_sets = alloc_sets + add;
1899             result_sets = rnew;
1900         }
1901         yaz_log(YLOG_DEBUG, "APT_numeric termp=%s", termp);
1902         grep_info.isam_p_indx = 0;
1903         res = numeric_term(zh, zapt, &termp, attributeSet, &grep_info,
1904                            reg_type, complete_flag, num_bases, basenames,
1905                            term_dst, xpath_use,
1906                            stream);
1907         if (res == ZEBRA_FAIL || termp == 0)
1908             break;
1909         yaz_log(YLOG_DEBUG, "term: %s", term_dst);
1910         result_sets[num_result_sets] =
1911             rset_trunc(zh, grep_info.isam_p_buf,
1912                        grep_info.isam_p_indx, term_dst,
1913                        strlen(term_dst), rank_type,
1914                        0 /* preserve position */,
1915                        zapt->term->which, rset_nmem,
1916                        kc, kc->scope);
1917         if (!result_sets[num_result_sets])
1918             break;
1919         num_result_sets++;
1920     }
1921     grep_info_delete(&grep_info);
1922     if (termp)
1923     {
1924         int i;
1925         for (i = 0; i<num_result_sets; i++)
1926             rset_delete(result_sets[i]);
1927         return ZEBRA_FAIL;
1928     }
1929     if (num_result_sets == 0)
1930         *rset = rsnull_create(rset_nmem, kc);
1931     if (num_result_sets == 1)
1932         *rset = result_sets[0];
1933     else
1934         *rset = rsmulti_and_create(rset_nmem, kc, kc->scope,
1935                                    num_result_sets, result_sets);
1936     if (!*rset)
1937         return ZEBRA_FAIL;
1938     return ZEBRA_OK;
1939 }
1940
1941 static ZEBRA_RES rpn_search_APT_local(ZebraHandle zh,
1942                                       Z_AttributesPlusTerm *zapt,
1943                                       const char *termz,
1944                                       oid_value attributeSet,
1945                                       NMEM stream,
1946                                       const char *rank_type, NMEM rset_nmem,
1947                                       RSET *rset,
1948                                       struct rset_key_control *kc)
1949 {
1950     RSFD rsfd;
1951     struct it_key key;
1952     int sys;
1953     *rset = rstemp_create(rset_nmem, kc, kc->scope,
1954                           res_get (zh->res, "setTmpDir"),0 );
1955     rsfd = rset_open(*rset, RSETF_WRITE);
1956
1957     sys = atoi(termz);
1958     if (sys <= 0)
1959         sys = 1;
1960     key.mem[0] = sys;
1961     key.mem[1] = 1;
1962     key.len = 2;
1963     rset_write (rsfd, &key);
1964     rset_close (rsfd);
1965     return ZEBRA_OK;
1966 }
1967
1968 static ZEBRA_RES rpn_sort_spec(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1969                                oid_value attributeSet, NMEM stream,
1970                                Z_SortKeySpecList *sort_sequence,
1971                                const char *rank_type,
1972                                RSET *rset,
1973                                struct rset_key_control *kc)
1974 {
1975     int i;
1976     int sort_relation_value;
1977     AttrType sort_relation_type;
1978     int use_value;
1979     AttrType use_type;
1980     Z_SortKeySpec *sks;
1981     Z_SortKey *sk;
1982     Z_AttributeElement *ae;
1983     int oid[OID_SIZE];
1984     oident oe;
1985     char termz[20];
1986
1987     attr_init(&sort_relation_type, zapt, 7);
1988     sort_relation_value = attr_find(&sort_relation_type, &attributeSet);
1989
1990     attr_init(&use_type, zapt, 1);
1991     use_value = attr_find(&use_type, &attributeSet);
1992
1993     if (!sort_sequence->specs)
1994     {
1995         sort_sequence->num_specs = 10;
1996         sort_sequence->specs = (Z_SortKeySpec **)
1997             nmem_malloc(stream, sort_sequence->num_specs *
1998                          sizeof(*sort_sequence->specs));
1999         for (i = 0; i<sort_sequence->num_specs; i++)
2000             sort_sequence->specs[i] = 0;
2001     }
2002     if (zapt->term->which != Z_Term_general)
2003         i = 0;
2004     else
2005         i = atoi_n ((char *) zapt->term->u.general->buf,
2006                     zapt->term->u.general->len);
2007     if (i >= sort_sequence->num_specs)
2008         i = 0;
2009     sprintf(termz, "%d", i);
2010
2011     oe.proto = PROTO_Z3950;
2012     oe.oclass = CLASS_ATTSET;
2013     oe.value = attributeSet;
2014     if (!oid_ent_to_oid (&oe, oid))
2015         return ZEBRA_FAIL;
2016
2017     sks = (Z_SortKeySpec *) nmem_malloc(stream, sizeof(*sks));
2018     sks->sortElement = (Z_SortElement *)
2019         nmem_malloc(stream, sizeof(*sks->sortElement));
2020     sks->sortElement->which = Z_SortElement_generic;
2021     sk = sks->sortElement->u.generic = (Z_SortKey *)
2022         nmem_malloc(stream, sizeof(*sk));
2023     sk->which = Z_SortKey_sortAttributes;
2024     sk->u.sortAttributes = (Z_SortAttributes *)
2025         nmem_malloc(stream, sizeof(*sk->u.sortAttributes));
2026
2027     sk->u.sortAttributes->id = oid;
2028     sk->u.sortAttributes->list = (Z_AttributeList *)
2029         nmem_malloc(stream, sizeof(*sk->u.sortAttributes->list));
2030     sk->u.sortAttributes->list->num_attributes = 1;
2031     sk->u.sortAttributes->list->attributes = (Z_AttributeElement **)
2032         nmem_malloc(stream, sizeof(*sk->u.sortAttributes->list->attributes));
2033     ae = *sk->u.sortAttributes->list->attributes = (Z_AttributeElement *)
2034         nmem_malloc(stream, sizeof(**sk->u.sortAttributes->list->attributes));
2035     ae->attributeSet = 0;
2036     ae->attributeType = (int *)
2037         nmem_malloc(stream, sizeof(*ae->attributeType));
2038     *ae->attributeType = 1;
2039     ae->which = Z_AttributeValue_numeric;
2040     ae->value.numeric = (int *)
2041         nmem_malloc(stream, sizeof(*ae->value.numeric));
2042     *ae->value.numeric = use_value;
2043
2044     sks->sortRelation = (int *)
2045         nmem_malloc(stream, sizeof(*sks->sortRelation));
2046     if (sort_relation_value == 1)
2047         *sks->sortRelation = Z_SortKeySpec_ascending;
2048     else if (sort_relation_value == 2)
2049         *sks->sortRelation = Z_SortKeySpec_descending;
2050     else
2051         *sks->sortRelation = Z_SortKeySpec_ascending;
2052
2053     sks->caseSensitivity = (int *)
2054         nmem_malloc(stream, sizeof(*sks->caseSensitivity));
2055     *sks->caseSensitivity = 0;
2056
2057     sks->which = Z_SortKeySpec_null;
2058     sks->u.null = odr_nullval ();
2059     sort_sequence->specs[i] = sks;
2060     *rset = rsnull_create (NULL, kc);
2061     return ZEBRA_OK;
2062 }
2063
2064
2065 static int parse_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2066                        oid_value attributeSet,
2067                        struct xpath_location_step *xpath, int max, NMEM mem)
2068 {
2069     oid_value curAttributeSet = attributeSet;
2070     AttrType use;
2071     const char *use_string = 0;
2072
2073     attr_init(&use, zapt, 1);
2074     attr_find_ex(&use, &curAttributeSet, &use_string);
2075
2076     if (!use_string || *use_string != '/')
2077         return -1;
2078
2079     return zebra_parse_xpath_str(use_string, xpath, max, mem);
2080 }
2081
2082
2083
2084 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
2085                         int reg_type, const char *term, int use,
2086                         oid_value curAttributeSet, NMEM rset_nmem,
2087                         struct rset_key_control *kc)
2088 {
2089     RSET rset;
2090     struct grep_info grep_info;
2091     char term_dict[2048];
2092     char ord_buf[32];
2093     int prefix_len = 0;
2094     int ord = zebraExplain_lookup_attr_su(zh->reg->zei, curAttributeSet, use);
2095     int ord_len, i, r, max_pos;
2096     int term_type = Z_Term_characterString;
2097     const char *flags = "void";
2098
2099     if (grep_info_prepare(zh, 0 /* zapt */, &grep_info, '0', stream))
2100         return rsnull_create(rset_nmem, kc);
2101
2102     if (ord < 0)
2103         return rsnull_create(rset_nmem, kc);
2104     if (prefix_len)
2105         term_dict[prefix_len++] = '|';
2106     else
2107         term_dict[prefix_len++] = '(';
2108
2109     ord_len = key_SU_encode (ord, ord_buf);
2110     for (i = 0; i<ord_len; i++)
2111     {
2112         term_dict[prefix_len++] = 1;
2113         term_dict[prefix_len++] = ord_buf[i];
2114     }
2115     term_dict[prefix_len++] = ')';
2116     term_dict[prefix_len++] = 1;
2117     term_dict[prefix_len++] = reg_type;
2118
2119     strcpy(term_dict+prefix_len, term);
2120
2121     grep_info.isam_p_indx = 0;
2122     r = dict_lookup_grep(zh->reg->dict, term_dict, 0,
2123                           &grep_info, &max_pos, 0, grep_handle);
2124     yaz_log(YLOG_DEBUG, "%s %d positions", term,
2125              grep_info.isam_p_indx);
2126     rset = rset_trunc(zh, grep_info.isam_p_buf,
2127                       grep_info.isam_p_indx, term, strlen(term),
2128                       flags, 1, term_type,rset_nmem,
2129                       kc, kc->scope);
2130     grep_info_delete(&grep_info);
2131     return rset;
2132 }
2133
2134 static RSET rpn_search_xpath(ZebraHandle zh,
2135                              oid_value attributeSet,
2136                              int num_bases, char **basenames,
2137                              NMEM stream, const char *rank_type, RSET rset,
2138                              int xpath_len, struct xpath_location_step *xpath,
2139                              NMEM rset_nmem,
2140                              struct rset_key_control *kc)
2141 {
2142     oid_value curAttributeSet = attributeSet;
2143     int base_no;
2144     int i;
2145
2146     if (xpath_len < 0)
2147         return rset;
2148
2149     yaz_log(YLOG_DEBUG, "xpath len=%d", xpath_len);
2150     for (i = 0; i<xpath_len; i++)
2151     {
2152         yaz_log(log_level_rpn, "XPATH %d %s", i, xpath[i].part);
2153
2154     }
2155
2156     curAttributeSet = VAL_IDXPATH;
2157
2158     /*
2159       //a    ->    a/.*
2160       //a/b  ->    b/a/.*
2161       /a     ->    a/
2162       /a/b   ->    b/a/
2163
2164       /      ->    none
2165
2166    a[@attr = value]/b[@other = othervalue]
2167
2168  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
2169  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
2170  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
2171  /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y)
2172  /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y)
2173  /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x)
2174
2175     */
2176
2177     dict_grep_cmap (zh->reg->dict, 0, 0);
2178
2179     for (base_no = 0; base_no < num_bases; base_no++)
2180     {
2181         int level = xpath_len;
2182         int first_path = 1;
2183
2184         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2185         {
2186             zh->errCode = YAZ_BIB1_DATABASE_UNAVAILABLE;
2187             zh->errString = basenames[base_no];
2188             return rset;
2189         }
2190         while (--level >= 0)
2191         {
2192             char xpath_rev[128];
2193             int i, len;
2194             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
2195
2196             *xpath_rev = 0;
2197             len = 0;
2198             for (i = level; i >= 1; --i)
2199             {
2200                 const char *cp = xpath[i].part;
2201                 if (*cp)
2202                 {
2203                     for (;*cp; cp++)
2204                         if (*cp == '*')
2205                         {
2206                             memcpy (xpath_rev + len, "[^/]*", 5);
2207                             len += 5;
2208                         }
2209                         else if (*cp == ' ')
2210                         {
2211
2212                             xpath_rev[len++] = 1;
2213                             xpath_rev[len++] = ' ';
2214                         }
2215
2216                         else
2217                             xpath_rev[len++] = *cp;
2218                     xpath_rev[len++] = '/';
2219                 }
2220                 else if (i == 1)  /* // case */
2221                 {
2222                     xpath_rev[len++] = '.';
2223                     xpath_rev[len++] = '*';
2224                 }
2225             }
2226             xpath_rev[len] = 0;
2227
2228             if (xpath[level].predicate &&
2229                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
2230                 xpath[level].predicate->u.relation.name[0])
2231             {
2232                 WRBUF wbuf = wrbuf_alloc();
2233                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
2234                 if (xpath[level].predicate->u.relation.value)
2235                 {
2236                     const char *cp = xpath[level].predicate->u.relation.value;
2237                     wrbuf_putc(wbuf, '=');
2238
2239                     while (*cp)
2240                     {
2241                         if (strchr(REGEX_CHARS, *cp))
2242                             wrbuf_putc(wbuf, '\\');
2243                         wrbuf_putc(wbuf, *cp);
2244                         cp++;
2245                     }
2246                 }
2247                 wrbuf_puts(wbuf, "");
2248                 rset_attr = xpath_trunc(
2249                     zh, stream, '0', wrbuf_buf(wbuf), 3,
2250                     curAttributeSet, rset_nmem, kc);
2251                 wrbuf_free(wbuf, 1);
2252             }
2253             else
2254             {
2255                 if (!first_path)
2256                     continue;
2257             }
2258             yaz_log(log_level_rpn, "xpath_rev (%d) = %s", level, xpath_rev);
2259             if (strlen(xpath_rev))
2260             {
2261                 rset_start_tag = xpath_trunc(zh, stream, '0',
2262                         xpath_rev, 1, curAttributeSet, rset_nmem, kc);
2263
2264                 rset_end_tag = xpath_trunc(zh, stream, '0',
2265                         xpath_rev, 2, curAttributeSet, rset_nmem, kc);
2266
2267                 rset = rsbetween_create(rset_nmem, kc, kc->scope,
2268                                         rset_start_tag, rset,
2269                                         rset_end_tag, rset_attr);
2270             }
2271             first_path = 0;
2272         }
2273     }
2274
2275     return rset;
2276 }
2277
2278 static ZEBRA_RES rpn_search_APT(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2279                                 oid_value attributeSet, NMEM stream,
2280                                 Z_SortKeySpecList *sort_sequence,
2281                                 int num_bases, char **basenames,
2282                                 NMEM rset_nmem,
2283                                 RSET *rset,
2284                                 struct rset_key_control *kc)
2285 {
2286     ZEBRA_RES res = ZEBRA_OK;
2287     unsigned reg_id;
2288     char *search_type = NULL;
2289     char rank_type[128];
2290     int complete_flag;
2291     int sort_flag;
2292     char termz[IT_MAX_WORD+1];
2293     int xpath_len;
2294     int xpath_use = 0;
2295     struct xpath_location_step xpath[10];
2296
2297     if (!log_level_set)
2298     {
2299         log_level_rpn = yaz_log_module_level("rpn");
2300         log_level_set = 1;
2301     }
2302     zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2303                     rank_type, &complete_flag, &sort_flag);
2304
2305     yaz_log(YLOG_DEBUG, "reg_id=%c", reg_id);
2306     yaz_log(YLOG_DEBUG, "complete_flag=%d", complete_flag);
2307     yaz_log(YLOG_DEBUG, "search_type=%s", search_type);
2308     yaz_log(YLOG_DEBUG, "rank_type=%s", rank_type);
2309
2310     if (zapt_term_to_utf8(zh, zapt, termz) == ZEBRA_FAIL)
2311         return ZEBRA_FAIL;
2312
2313     if (sort_flag)
2314         return rpn_sort_spec(zh, zapt, attributeSet, stream, sort_sequence,
2315                              rank_type, rset, kc);
2316     xpath_len = parse_xpath(zh, zapt, attributeSet, xpath, 10, stream);
2317     if (xpath_len >= 0)
2318     {
2319         xpath_use = 1016;
2320         if (xpath[xpath_len-1].part[0] == '@')
2321             xpath_use = 1015;
2322     }
2323
2324     if (!strcmp(search_type, "phrase"))
2325     {
2326         res = rpn_search_APT_phrase(zh, zapt, termz, attributeSet, stream,
2327                                     reg_id, complete_flag, rank_type,
2328                                     xpath_use,
2329                                     num_bases, basenames, rset_nmem,
2330                                     rset, kc);
2331     }
2332     else if (!strcmp(search_type, "and-list"))
2333     {
2334         res = rpn_search_APT_and_list(zh, zapt, termz, attributeSet, stream,
2335                                       reg_id, complete_flag, rank_type,
2336                                       xpath_use,
2337                                       num_bases, basenames, rset_nmem,
2338                                       rset, kc);
2339     }
2340     else if (!strcmp(search_type, "or-list"))
2341     {
2342         res = rpn_search_APT_or_list(zh, zapt, termz, attributeSet, stream,
2343                                      reg_id, complete_flag, rank_type,
2344                                      xpath_use,
2345                                      num_bases, basenames, rset_nmem,
2346                                      rset, kc);
2347     }
2348     else if (!strcmp(search_type, "local"))
2349     {
2350         res = rpn_search_APT_local(zh, zapt, termz, attributeSet, stream,
2351                                    rank_type, rset_nmem, rset, kc);
2352     }
2353     else if (!strcmp(search_type, "numeric"))
2354     {
2355         res = rpn_search_APT_numeric(zh, zapt, termz, attributeSet, stream,
2356                                      reg_id, complete_flag, rank_type,
2357                                      xpath_use,
2358                                      num_bases, basenames, rset_nmem,
2359                                      rset, kc);
2360     }
2361     else
2362     {
2363         zh->errCode = YAZ_BIB1_UNSUPP_STRUCTURE_ATTRIBUTE;
2364         return ZEBRA_FAIL;
2365     }
2366     if (res != ZEBRA_OK)
2367         return res;
2368     if (!*rset)
2369         return ZEBRA_FAIL;
2370     *rset = rpn_search_xpath(zh, attributeSet, num_bases, basenames,
2371                              stream, rank_type, *rset,
2372                              xpath_len, xpath, rset_nmem, kc);
2373     if (!*rset)
2374         return ZEBRA_FAIL;
2375     return ZEBRA_OK;
2376 }
2377
2378 static ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2379                                       oid_value attributeSet,
2380                                       NMEM stream, NMEM rset_nmem,
2381                                       Z_SortKeySpecList *sort_sequence,
2382                                       int num_bases, char **basenames,
2383                                       RSET **result_sets, int *num_result_sets,
2384                                       Z_Operator *parent_op,
2385                                       struct rset_key_control *kc);
2386
2387 ZEBRA_RES rpn_search_top(ZebraHandle zh, Z_RPNStructure *zs,
2388                          oid_value attributeSet,
2389                          NMEM stream, NMEM rset_nmem,
2390                          Z_SortKeySpecList *sort_sequence,
2391                          int num_bases, char **basenames,
2392                          RSET *result_set)
2393 {
2394     RSET *result_sets = 0;
2395     int num_result_sets = 0;
2396     ZEBRA_RES res;
2397     struct rset_key_control *kc = zebra_key_control_create(zh);
2398
2399     res = rpn_search_structure(zh, zs, attributeSet,
2400                                stream, rset_nmem,
2401                                sort_sequence,
2402                                num_bases, basenames,
2403                                &result_sets, &num_result_sets,
2404                                0 /* no parent op */,
2405                                kc);
2406     if (res != ZEBRA_OK)
2407     {
2408         int i;
2409         for (i = 0; i<num_result_sets; i++)
2410             rset_delete(result_sets[i]);
2411         *result_set = 0;
2412         return res;
2413     }
2414     assert(num_result_sets == 1);
2415     assert(result_sets);
2416     assert(*result_sets);
2417     *result_set = *result_sets;
2418
2419     (*kc->dec)(kc);
2420     return ZEBRA_OK;
2421 }
2422
2423 ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2424                                oid_value attributeSet,
2425                                NMEM stream, NMEM rset_nmem,
2426                                Z_SortKeySpecList *sort_sequence,
2427                                int num_bases, char **basenames,
2428                                RSET **result_sets, int *num_result_sets,
2429                                Z_Operator *parent_op,
2430                                struct rset_key_control *kc)
2431 {
2432     *num_result_sets = 0;
2433     if (zs->which == Z_RPNStructure_complex)
2434     {
2435         ZEBRA_RES res;
2436         Z_Operator *zop = zs->u.complex->roperator;
2437         RSET *result_sets_l = 0;
2438         int num_result_sets_l = 0;
2439         RSET *result_sets_r = 0;
2440         int num_result_sets_r = 0;
2441
2442         res = rpn_search_structure(zh, zs->u.complex->s1,
2443                                    attributeSet, stream, rset_nmem,
2444                                    sort_sequence,
2445                                    num_bases, basenames,
2446                                    &result_sets_l, &num_result_sets_l,
2447                                    zop, kc);
2448         if (res != ZEBRA_OK)
2449         {
2450             int i;
2451             for (i = 0; i<num_result_sets_l; i++)
2452                 rset_delete(result_sets_l[i]);
2453             return res;
2454         }
2455         res = rpn_search_structure(zh, zs->u.complex->s2,
2456                                    attributeSet, stream, rset_nmem,
2457                                    sort_sequence,
2458                                    num_bases, basenames,
2459                                    &result_sets_r, &num_result_sets_r,
2460                                    zop, kc);
2461         if (res != ZEBRA_OK)
2462         {
2463             int i;
2464             for (i = 0; i<num_result_sets_l; i++)
2465                 rset_delete(result_sets_l[i]);
2466             for (i = 0; i<num_result_sets_r; i++)
2467                 rset_delete(result_sets_r[i]);
2468             return res;
2469         }
2470
2471         /* make a new list of result for all children */
2472         *num_result_sets = num_result_sets_l + num_result_sets_r;
2473         *result_sets = nmem_malloc(stream, *num_result_sets *
2474                                    sizeof(**result_sets));
2475         memcpy(*result_sets, result_sets_l,
2476                num_result_sets_l * sizeof(**result_sets));
2477         memcpy(*result_sets + num_result_sets_l, result_sets_r,
2478                num_result_sets_r * sizeof(**result_sets));
2479
2480         if (!parent_op || parent_op->which != zop->which
2481             || (zop->which != Z_Operator_and &&
2482                 zop->which != Z_Operator_or))
2483         {
2484             /* parent node different from this one (or non-present) */
2485             /* we must combine result sets now */
2486             RSET rset;
2487             switch (zop->which)
2488             {
2489             case Z_Operator_and:
2490                 rset = rsmulti_and_create(rset_nmem, kc,
2491                                           kc->scope,
2492                                           *num_result_sets, *result_sets);
2493                 break;
2494             case Z_Operator_or:
2495                 rset = rsmulti_or_create(rset_nmem, kc,
2496                                          kc->scope,
2497                                          *num_result_sets, *result_sets);
2498                 break;
2499             case Z_Operator_and_not:
2500                 rset = rsbool_create_not(rset_nmem, kc,
2501                                          kc->scope,
2502                                          (*result_sets)[0],
2503                                          (*result_sets)[1]);
2504                 break;
2505             case Z_Operator_prox:
2506                 if (zop->u.prox->which != Z_ProximityOperator_known)
2507                 {
2508                     zh->errCode = YAZ_BIB1_UNSUPP_PROX_UNIT_CODE;
2509                     return ZEBRA_FAIL;
2510                 }
2511                 if (*zop->u.prox->u.known != Z_ProxUnit_word)
2512                 {
2513                     char *val = (char *) nmem_malloc(stream, 16);
2514                     zh->errCode = YAZ_BIB1_UNSUPP_PROX_UNIT_CODE;
2515                     zh->errString = val;
2516                     sprintf(val, "%d", *zop->u.prox->u.known);
2517                     return ZEBRA_FAIL;
2518                 }
2519                 else
2520                 {
2521                     rset = rsprox_create(rset_nmem, kc,
2522                                          kc->scope,
2523                                          *num_result_sets, *result_sets,
2524                                          *zop->u.prox->ordered,
2525                                          (!zop->u.prox->exclusion ?
2526                                           0 : *zop->u.prox->exclusion),
2527                                          *zop->u.prox->relationType,
2528                                          *zop->u.prox->distance );
2529                 }
2530                 break;
2531             default:
2532                 zh->errCode = YAZ_BIB1_OPERATOR_UNSUPP;
2533                 return ZEBRA_FAIL;
2534             }
2535             *num_result_sets = 1;
2536             *result_sets = nmem_malloc(stream, *num_result_sets *
2537                                        sizeof(**result_sets));
2538             (*result_sets)[0] = rset;
2539         }
2540     }
2541     else if (zs->which == Z_RPNStructure_simple)
2542     {
2543         RSET rset;
2544         ZEBRA_RES res;
2545
2546         if (zs->u.simple->which == Z_Operand_APT)
2547         {
2548             yaz_log(YLOG_DEBUG, "rpn_search_APT");
2549             res = rpn_search_APT(zh, zs->u.simple->u.attributesPlusTerm,
2550                                  attributeSet, stream, sort_sequence,
2551                                  num_bases, basenames, rset_nmem, &rset,
2552                                  kc);
2553             if (res != ZEBRA_OK)
2554                 return res;
2555         }
2556         else if (zs->u.simple->which == Z_Operand_resultSetId)
2557         {
2558             yaz_log(YLOG_DEBUG, "rpn_search_ref");
2559             rset = resultSetRef(zh, zs->u.simple->u.resultSetId);
2560             if (!rset)
2561             {
2562                 zh->errCode = YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST;
2563                 zh->errString =
2564                     nmem_strdup(stream, zs->u.simple->u.resultSetId);
2565                 return ZEBRA_FAIL;
2566             }
2567             rset_dup(rset);
2568         }
2569         else
2570         {
2571             zh->errCode = YAZ_BIB1_UNSUPP_SEARCH;
2572             return ZEBRA_FAIL;
2573         }
2574         *num_result_sets = 1;
2575         *result_sets = nmem_malloc(stream, *num_result_sets *
2576                                    sizeof(**result_sets));
2577         (*result_sets)[0] = rset;
2578     }
2579     else
2580     {
2581         zh->errCode = YAZ_BIB1_UNSUPP_SEARCH;
2582         return ZEBRA_FAIL;
2583     }
2584     return ZEBRA_OK;
2585 }
2586
2587 struct scan_info_entry {
2588     char *term;
2589     ISAM_P isam_p;
2590 };
2591
2592 struct scan_info {
2593     struct scan_info_entry *list;
2594     ODR odr;
2595     int before, after;
2596     char prefix[20];
2597 };
2598
2599 static int scan_handle (char *name, const char *info, int pos, void *client)
2600 {
2601     int len_prefix, idx;
2602     struct scan_info *scan_info = (struct scan_info *) client;
2603
2604     len_prefix = strlen(scan_info->prefix);
2605     if (memcmp (name, scan_info->prefix, len_prefix))
2606         return 1;
2607     if (pos > 0)
2608         idx = scan_info->after - pos + scan_info->before;
2609     else
2610         idx = - pos - 1;
2611
2612     if (idx < 0)
2613         return 0;
2614     scan_info->list[idx].term = (char *)
2615         odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
2616     strcpy(scan_info->list[idx].term, name + len_prefix);
2617     assert (*info == sizeof(ISAM_P));
2618     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
2619     return 0;
2620 }
2621
2622 static void scan_term_untrans (ZebraHandle zh, NMEM stream, int reg_type,
2623                                char **dst, const char *src)
2624 {
2625     char term_src[IT_MAX_WORD];
2626     char term_dst[IT_MAX_WORD];
2627
2628     term_untrans (zh, reg_type, term_src, src);
2629
2630     if (zh->iconv_from_utf8 != 0)
2631     {
2632         int len;
2633         char *inbuf = term_src;
2634         size_t inleft = strlen(term_src);
2635         char *outbuf = term_dst;
2636         size_t outleft = sizeof(term_dst)-1;
2637         size_t ret;
2638
2639         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2640                          &outbuf, &outleft);
2641         if (ret == (size_t)(-1))
2642             len = 0;
2643         else
2644             len = outbuf - term_dst;
2645         *dst = nmem_malloc(stream, len + 1);
2646         if (len > 0)
2647             memcpy (*dst, term_dst, len);
2648         (*dst)[len] = '\0';
2649     }
2650     else
2651         *dst = nmem_strdup(stream, term_src);
2652 }
2653
2654 static void count_set (RSET r, int *count)
2655 {
2656     zint psysno = 0;
2657     int kno = 0;
2658     struct it_key key;
2659     RSFD rfd;
2660
2661     yaz_log(YLOG_DEBUG, "count_set");
2662
2663     *count = 0;
2664     rfd = rset_open (r, RSETF_READ);
2665     while (rset_read (rfd, &key,0 /* never mind terms */))
2666     {
2667         if (key.mem[0] != psysno)
2668         {
2669             psysno = key.mem[0];
2670             (*count)++;
2671         }
2672         kno++;
2673     }
2674     rset_close (rfd);
2675     yaz_log(YLOG_DEBUG, "%d keys, %d records", kno, *count);
2676 }
2677
2678 ZEBRA_RES rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2679                    oid_value attributeset,
2680                    int num_bases, char **basenames,
2681                    int *position, int *num_entries, ZebraScanEntry **list,
2682                    int *is_partial, RSET limit_set, int return_zero)
2683 {
2684     int i;
2685     int pos = *position;
2686     int num = *num_entries;
2687     int before;
2688     int after;
2689     int base_no;
2690     char termz[IT_MAX_WORD+20];
2691     AttrType use;
2692     int use_value;
2693     const char *use_string = 0;
2694     struct scan_info *scan_info_array;
2695     ZebraScanEntry *glist;
2696     int ords[32], ord_no = 0;
2697     int ptr[32];
2698
2699     int bases_ok = 0;     /* no of databases with OK attribute */
2700     int errCode = 0;      /* err code (if any is not OK) */
2701     char *errString = 0;  /* addinfo */
2702
2703     unsigned reg_id;
2704     char *search_type = NULL;
2705     char rank_type[128];
2706     int complete_flag;
2707     int sort_flag;
2708     NMEM rset_nmem = NULL;
2709     struct rset_key_control *kc = 0;
2710
2711     *list = 0;
2712     *is_partial = 0;
2713
2714     if (attributeset == VAL_NONE)
2715         attributeset = VAL_BIB1;
2716
2717     if (!limit_set)
2718     {
2719         AttrType termset;
2720         int termset_value_numeric;
2721         const char *termset_value_string;
2722         attr_init(&termset, zapt, 8);
2723         termset_value_numeric =
2724             attr_find_ex(&termset, NULL, &termset_value_string);
2725         if (termset_value_numeric != -1)
2726         {
2727             char resname[32];
2728             const char *termset_name = 0;
2729
2730             if (termset_value_numeric != -2)
2731             {
2732
2733                 sprintf(resname, "%d", termset_value_numeric);
2734                 termset_name = resname;
2735             }
2736             else
2737                 termset_name = termset_value_string;
2738
2739             limit_set = resultSetRef (zh, termset_name);
2740         }
2741     }
2742
2743     yaz_log(YLOG_DEBUG, "position = %d, num = %d set=%d",
2744             pos, num, attributeset);
2745
2746     attr_init(&use, zapt, 1);
2747     use_value = attr_find_ex(&use, &attributeset, &use_string);
2748
2749     if (zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2750                         rank_type, &complete_flag, &sort_flag))
2751     {
2752         *num_entries = 0;
2753         zh->errCode = YAZ_BIB1_UNSUPP_ATTRIBUTE_TYPE;
2754         return ZEBRA_FAIL;
2755     }
2756     yaz_log(YLOG_DEBUG, "use_value = %d", use_value);
2757
2758     if (use_value == -1)
2759         use_value = 1016;
2760     for (base_no = 0; base_no < num_bases && ord_no < 32; base_no++)
2761     {
2762         data1_local_attribute *local_attr;
2763         attent attp;
2764         int ord;
2765
2766         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2767         {
2768             zh->errString = basenames[base_no];
2769             zh->errCode = YAZ_BIB1_DATABASE_UNAVAILABLE;
2770             *num_entries = 0;
2771             return ZEBRA_FAIL;
2772         }
2773
2774         if (use_string &&
2775             (ord = zebraExplain_lookup_attr_str(zh->reg->zei,
2776                                                 use_string)) >= 0)
2777         {
2778             /* we have a match for a raw string attribute */
2779             if (ord > 0)
2780                 ords[ord_no++] = ord;
2781             attp.local_attributes = 0;  /* no more attributes */
2782         }
2783         else
2784         {
2785             int r;
2786
2787             if ((r = att_getentbyatt (zh, &attp, attributeset, use_value,
2788                                       use_string)))
2789             {
2790                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
2791                         attributeset, use_value);
2792                 if (r == -1)
2793                 {
2794                     errCode = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
2795                     if (use_string)
2796                         errString = odr_strdup(stream, use_string);
2797                     else
2798                     {
2799                         char val_str[32];
2800                         sprintf(val_str, "%d", use_value);
2801                         errString = odr_strdup(stream, val_str);
2802                     }
2803                 }
2804                 else
2805                     errCode = YAZ_BIB1_UNSUPP_ATTRIBUTE_SET;
2806                 continue;
2807             }
2808         }
2809         bases_ok++;
2810         for (local_attr = attp.local_attributes; local_attr && ord_no < 32;
2811              local_attr = local_attr->next)
2812         {
2813             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
2814                                               attp.attset_ordinal,
2815                                               local_attr->local);
2816             if (ord > 0)
2817                 ords[ord_no++] = ord;
2818         }
2819     }
2820     if (!bases_ok && errCode)
2821     {
2822         zh->errCode = errCode;
2823         zh->errString = errString;
2824         *num_entries = 0;
2825         return ZEBRA_FAIL;
2826     }
2827     if (ord_no == 0)
2828     {
2829         *num_entries = 0;
2830         return ZEBRA_OK;
2831     }
2832     /* prepare dictionary scanning */
2833     if (num < 1)
2834     {
2835         *num_entries = 0;
2836         return ZEBRA_OK;
2837     }
2838     before = pos-1;
2839     if (before < 0)
2840         before = 0;
2841     after = 1+num-pos;
2842     if (after < 0)
2843         after = 0;
2844     yaz_log(YLOG_DEBUG, "rpn_scan pos=%d num=%d before=%d "
2845             "after=%d before+after=%d",
2846             pos, num, before, after, before+after);
2847     scan_info_array = (struct scan_info *)
2848         odr_malloc(stream, ord_no * sizeof(*scan_info_array));
2849     for (i = 0; i < ord_no; i++)
2850     {
2851         int j, prefix_len = 0;
2852         int before_tmp = before, after_tmp = after;
2853         struct scan_info *scan_info = scan_info_array + i;
2854         struct rpn_char_map_info rcmi;
2855
2856         rpn_char_map_prepare (zh->reg, reg_id, &rcmi);
2857
2858         scan_info->before = before;
2859         scan_info->after = after;
2860         scan_info->odr = stream;
2861
2862         scan_info->list = (struct scan_info_entry *)
2863             odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
2864         for (j = 0; j<before+after; j++)
2865             scan_info->list[j].term = NULL;
2866
2867         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2868         termz[prefix_len++] = reg_id;
2869         termz[prefix_len] = 0;
2870         strcpy(scan_info->prefix, termz);
2871
2872         if (trans_scan_term(zh, zapt, termz+prefix_len, reg_id) == ZEBRA_FAIL)
2873             return ZEBRA_FAIL;
2874
2875         dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
2876                   scan_info, scan_handle);
2877     }
2878     glist = (ZebraScanEntry *)
2879         odr_malloc(stream, (before+after)*sizeof(*glist));
2880
2881     rset_nmem = nmem_create();
2882     kc = zebra_key_control_create(zh);
2883
2884     /* consider terms after main term */
2885     for (i = 0; i < ord_no; i++)
2886         ptr[i] = before;
2887
2888     *is_partial = 0;
2889     for (i = 0; i<after; i++)
2890     {
2891         int j, j0 = -1;
2892         const char *mterm = NULL;
2893         const char *tst;
2894         RSET rset = 0;
2895         int lo = i + pos-1; /* offset in result list */
2896
2897         /* find: j0 is the first of the minimal values */
2898         for (j = 0; j < ord_no; j++)
2899         {
2900             if (ptr[j] < before+after && ptr[j] >= 0 &&
2901                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2902                 (!mterm || strcmp (tst, mterm) < 0))
2903             {
2904                 j0 = j;
2905                 mterm = tst;
2906             }
2907         }
2908         if (j0 == -1)
2909             break;  /* no value found, stop */
2910
2911         /* get result set for first one , but only if it's within bounds */
2912         if (lo >= 0)
2913         {
2914             /* get result set for first term */
2915             scan_term_untrans(zh, stream->mem, reg_id,
2916                               &glist[lo].term, mterm);
2917             rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2918                               glist[lo].term, strlen(glist[lo].term),
2919                               NULL, 0, zapt->term->which, rset_nmem,
2920                               kc, kc->scope);
2921         }
2922         ptr[j0]++; /* move index for this set .. */
2923         /* get result set for remaining scan terms */
2924         for (j = j0+1; j<ord_no; j++)
2925         {
2926             if (ptr[j] < before+after && ptr[j] >= 0 &&
2927                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2928                 !strcmp (tst, mterm))
2929             {
2930                 if (lo >= 0)
2931                 {
2932                     RSET rsets[2];
2933
2934                     rsets[0] = rset;
2935                     rsets[1] =
2936                         rset_trunc(
2937                             zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
2938                             glist[lo].term,
2939                             strlen(glist[lo].term), NULL, 0,
2940                             zapt->term->which,rset_nmem,
2941                             kc, kc->scope);
2942                     rset = rsmulti_or_create(rset_nmem, kc,
2943                                              2, kc->scope, rsets);
2944                 }
2945                 ptr[j]++;
2946             }
2947         }
2948         if (lo >= 0)
2949         {
2950             /* merge with limit_set if given */
2951             if (limit_set)
2952             {
2953                 RSET rsets[2];
2954                 rsets[0] = rset;
2955                 rsets[1] = rset_dup(limit_set);
2956
2957                 rset = rsmulti_and_create(rset_nmem, kc,
2958                                           kc->scope, 2, rsets);
2959             }
2960             /* count it */
2961             count_set(rset, &glist[lo].occurrences);
2962             rset_delete(rset);
2963         }
2964     }
2965     if (i < after)
2966     {
2967         *num_entries -= (after-i);
2968         *is_partial = 1;
2969         if (*num_entries < 0)
2970         {
2971             (*kc->dec)(kc);
2972             nmem_destroy(rset_nmem);
2973             *num_entries = 0;
2974             return ZEBRA_OK;
2975         }
2976     }
2977     /* consider terms before main term */
2978     for (i = 0; i<ord_no; i++)
2979         ptr[i] = 0;
2980
2981     for (i = 0; i<before; i++)
2982     {
2983         int j, j0 = -1;
2984         const char *mterm = NULL;
2985         const char *tst;
2986         RSET rset;
2987         int lo = before-1-i; /* offset in result list */
2988
2989         for (j = 0; j <ord_no; j++)
2990         {
2991             if (ptr[j] < before && ptr[j] >= 0 &&
2992                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2993                 (!mterm || strcmp (tst, mterm) > 0))
2994             {
2995                 j0 = j;
2996                     mterm = tst;
2997             }
2998         }
2999         if (j0 == -1)
3000             break;
3001
3002         scan_term_untrans (zh, stream->mem, reg_id,
3003                            &glist[lo].term, mterm);
3004
3005         rset = rset_trunc
3006             (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
3007              glist[lo].term, strlen(glist[lo].term),
3008              NULL, 0, zapt->term->which,rset_nmem,
3009              kc, kc->scope);
3010
3011         ptr[j0]++;
3012
3013         for (j = j0+1; j<ord_no; j++)
3014         {
3015             if (ptr[j] < before && ptr[j] >= 0 &&
3016                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
3017                 !strcmp (tst, mterm))
3018             {
3019                 RSET rsets[2];
3020
3021                 rsets[0] = rset;
3022                 rsets[1] = rset_trunc(
3023                     zh,
3024                     &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
3025                     glist[lo].term,
3026                     strlen(glist[lo].term), NULL, 0,
3027                     zapt->term->which, rset_nmem,
3028                     kc, kc->scope);
3029                 rset = rsmulti_or_create(rset_nmem, kc,
3030                                          2, kc->scope, rsets);
3031
3032                 ptr[j]++;
3033             }
3034         }
3035         if (limit_set)
3036         {
3037             RSET rsets[2];
3038             rsets[0] = rset;
3039             rsets[1] = rset_dup(limit_set);
3040
3041             rset = rsmulti_and_create(rset_nmem, kc,
3042                                       kc->scope, 2, rsets);
3043         }
3044         count_set (rset, &glist[lo].occurrences);
3045         rset_delete (rset);
3046     }
3047     (*kc->dec)(kc);
3048     nmem_destroy(rset_nmem);
3049     i = before-i;
3050     if (i)
3051     {
3052         *is_partial = 1;
3053         *position -= i;
3054         *num_entries -= i;
3055         if (*num_entries <= 0)
3056         {
3057             *num_entries = 0;
3058             return ZEBRA_OK;
3059         }
3060     }
3061
3062     *list = glist + i;               /* list is set to first 'real' entry */
3063
3064     yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
3065             *position, *num_entries);
3066     return ZEBRA_OK;
3067 }
3068