index/zrpn.c

   1 /* $Id: zrpn.c,v 1.186 2005-05-03 09:11:34 adam Exp $
   2    Copyright (C) 1995-2005
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with Zebra; see the file LICENSE.zebra.  If not, write to the
  19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  20 02111-1307, USA.
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #ifdef WIN32
  26 #include <io.h>
  27 #else
  28 #include <unistd.h>
  29 #endif
  30 #include <ctype.h>
  31
  32 #include <yaz/diagbib1.h>
  33 #include "index.h"
  34 #include <zebra_xpath.h>
  35
  36 #include <charmap.h>
  37 #include <rset.h>
  38
  39 struct rpn_char_map_info
  40 {
  41     ZebraMaps zm;
  42     int reg_type;
  43 };
  44
  45 typedef struct
  46 {
  47     int type;
  48     int major;
  49     int minor;
  50     Z_AttributesPlusTerm *zapt;
  51 } AttrType;
  52
  53
  54 static int log_level_set = 0;
  55 static int log_level_rpn = 0;
  56
  57 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  58 {
  59     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  60     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  61 #if 0
  62     if (out && *out)
  63     {
  64         const char *outp = *out;
  65         yaz_log(YLOG_LOG, "---");
  66         while (*outp)
  67         {
  68             yaz_log(YLOG_LOG, "%02X", *outp);
  69             outp++;
  70         }
  71     }
  72 #endif
  73     return out;
  74 }
  75
  76 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  77                                   struct rpn_char_map_info *map_info)
  78 {
  79     map_info->zm = reg->zebra_maps;
  80     map_info->reg_type = reg_type;
  81     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  82 }
  83
  84 static int attr_find_ex(AttrType *src, oid_value *attributeSetP,
  85                          const char **string_value)
  86 {
  87     int num_attributes;
  88
  89     num_attributes = src->zapt->attributes->num_attributes;
  90     while (src->major < num_attributes)
  91     {
  92         Z_AttributeElement *element;
  93
  94         element = src->zapt->attributes->attributes[src->major];
  95         if (src->type == *element->attributeType)
  96         {
  97             switch (element->which)
  98             {
  99             case Z_AttributeValue_numeric:
 100                 ++(src->major);
 101                 if (element->attributeSet && attributeSetP)
 102                 {
 103                     oident *attrset;
 104
 105                     attrset = oid_getentbyoid(element->attributeSet);
 106                     *attributeSetP = attrset->value;
 107                 }
 108                 return *element->value.numeric;
 109                 break;
 110             case Z_AttributeValue_complex:
 111                 if (src->minor >= element->value.complex->num_list)
 112                     break;
 113                 if (element->attributeSet && attributeSetP)
 114                 {
 115                     oident *attrset;
 116
 117                     attrset = oid_getentbyoid(element->attributeSet);
 118                     *attributeSetP = attrset->value;
 119                 }
 120                 if (element->value.complex->list[src->minor]->which ==
 121                     Z_StringOrNumeric_numeric)
 122                 {
 123                     ++(src->minor);
 124                     return
 125                         *element->value.complex->list[src->minor-1]->u.numeric;
 126                 }
 127                 else if (element->value.complex->list[src->minor]->which ==
 128                          Z_StringOrNumeric_string)
 129                 {
 130                     if (!string_value)
 131                         break;
 132                     ++(src->minor);
 133                     *string_value =
 134                         element->value.complex->list[src->minor-1]->u.string;
 135                     return -2;
 136                 }
 137                 else
 138                     break;
 139             default:
 140                 assert(0);
 141             }
 142         }
 143         ++(src->major);
 144     }
 145     return -1;
 146 }
 147
 148 static int attr_find(AttrType *src, oid_value *attributeSetP)
 149 {
 150     return attr_find_ex(src, attributeSetP, 0);
 151 }
 152
 153 static void attr_init(AttrType *src, Z_AttributesPlusTerm *zapt,
 154                        int type)
 155 {
 156     src->zapt = zapt;
 157     src->type = type;
 158     src->major = 0;
 159     src->minor = 0;
 160 }
 161
 162 #define TERM_COUNT
 163
 164 struct grep_info {
 165 #ifdef TERM_COUNT
 166     int *term_no;
 167 #endif
 168     ISAM_P *isam_p_buf;
 169     int isam_p_size;
 170     int isam_p_indx;
 171     ZebraHandle zh;
 172     int reg_type;
 173     ZebraSet termset;
 174 };
 175
 176 static void term_untrans(ZebraHandle zh, int reg_type,
 177                            char *dst, const char *src)
 178 {
 179     int len = 0;
 180     while (*src)
 181     {
 182         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
 183                                            reg_type, &src);
 184         if (!cp && len < IT_MAX_WORD-1)
 185             dst[len++] = *src++;
 186         else
 187             while (*cp && len < IT_MAX_WORD-1)
 188                 dst[len++] = *cp++;
 189     }
 190     dst[len] = '\0';
 191 }
 192
 193 static void add_isam_p(const char *name, const char *info,
 194                        struct grep_info *p)
 195 {
 196     if (!log_level_set)
 197     {
 198         log_level_rpn = yaz_log_module_level("rpn");
 199         log_level_set = 1;
 200     }
 201     if (p->isam_p_indx == p->isam_p_size)
 202     {
 203         ISAM_P *new_isam_p_buf;
 204 #ifdef TERM_COUNT
 205         int *new_term_no;
 206 #endif
 207         p->isam_p_size = 2*p->isam_p_size + 100;
 208         new_isam_p_buf = (ISAM_P *) xmalloc(sizeof(*new_isam_p_buf) *
 209                                             p->isam_p_size);
 210         if (p->isam_p_buf)
 211         {
 212             memcpy(new_isam_p_buf, p->isam_p_buf,
 213                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 214             xfree(p->isam_p_buf);
 215         }
 216         p->isam_p_buf = new_isam_p_buf;
 217
 218 #ifdef TERM_COUNT
 219         new_term_no = (int *) xmalloc(sizeof(*new_term_no) * p->isam_p_size);
 220         if (p->term_no)
 221         {
 222             memcpy(new_term_no, p->isam_p_buf,
 223                     p->isam_p_indx * sizeof(*p->term_no));
 224             xfree(p->term_no);
 225         }
 226         p->term_no = new_term_no;
 227 #endif
 228     }
 229     assert(*info == sizeof(*p->isam_p_buf));
 230     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 231
 232 #if 1
 233     if (p->termset)
 234     {
 235         const char *db;
 236         int set, use;
 237         char term_tmp[IT_MAX_WORD];
 238         int su_code = 0;
 239         int len = key_SU_decode (&su_code, name);
 240
 241         term_untrans  (p->zh, p->reg_type, term_tmp, name+len+1);
 242         yaz_log(log_level_rpn, "grep: %d %c %s", su_code, name[len], term_tmp);
 243         zebraExplain_lookup_ord (p->zh->reg->zei,
 244                                  su_code, &db, &set, &use);
 245         yaz_log(log_level_rpn, "grep:  set=%d use=%d db=%s", set, use, db);
 246
 247         resultSetAddTerm(p->zh, p->termset, name[len], db,
 248                          set, use, term_tmp);
 249     }
 250 #endif
 251     (p->isam_p_indx)++;
 252 }
 253
 254 static int grep_handle(char *name, const char *info, void *p)
 255 {
 256     add_isam_p(name, info, (struct grep_info *) p);
 257     return 0;
 258 }
 259
 260 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 261                     const char *ct1, const char *ct2, int first)
 262 {
 263     const char *s1, *s0 = *src;
 264     const char **map;
 265
 266     /* skip white space */
 267     while (*s0)
 268     {
 269         if (ct1 && strchr(ct1, *s0))
 270             break;
 271         if (ct2 && strchr(ct2, *s0))
 272             break;
 273         s1 = s0;
 274         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 275         if (**map != *CHR_SPACE)
 276             break;
 277         s0 = s1;
 278     }
 279     *src = s0;
 280     return *s0;
 281 }
 282
 283
 284 static void esc_str(char *out_buf, int out_size,
 285                     const char *in_buf, int in_size)
 286 {
 287     int k;
 288
 289     assert(out_buf);
 290     assert(in_buf);
 291     assert(out_size > 20);
 292     *out_buf = '\0';
 293     for (k = 0; k<in_size; k++)
 294     {
 295         int c = in_buf[k] & 0xff;
 296         int pc;
 297         if (c < 32 || c > 126)
 298             pc = '?';
 299         else
 300             pc = c;
 301         sprintf(out_buf +strlen(out_buf), "%02X:%c  ", c, pc);
 302         if (strlen(out_buf) > out_size-20)
 303         {
 304             strcat(out_buf, "..");
 305             break;
 306         }
 307     }
 308 }
 309
 310 #define REGEX_CHARS " []()|.*+?!"
 311
 312 /* term_100: handle term, where trunc = none(no operators at all) */
 313 static int term_100(ZebraMaps zebra_maps, int reg_type,
 314                     const char **src, char *dst, int space_split,
 315                     char *dst_term)
 316 {
 317     const char *s0;
 318     const char **map;
 319     int i = 0;
 320     int j = 0;
 321
 322     const char *space_start = 0;
 323     const char *space_end = 0;
 324
 325     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 326         return 0;
 327     s0 = *src;
 328     while (*s0)
 329     {
 330         const char *s1 = s0;
 331         int q_map_match = 0;
 332         map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 333                                 &q_map_match);
 334         if (space_split)
 335         {
 336             if (**map == *CHR_SPACE)
 337                 break;
 338         }
 339         else  /* complete subfield only. */
 340         {
 341             if (**map == *CHR_SPACE)
 342             {   /* save space mapping for later  .. */
 343                 space_start = s1;
 344                 space_end = s0;
 345                 continue;
 346             }
 347             else if (space_start)
 348             {   /* reload last space */
 349                 while (space_start < space_end)
 350                 {
 351                     if (strchr(REGEX_CHARS, *space_start))
 352                         dst[i++] = '\\';
 353                     dst_term[j++] = *space_start;
 354                     dst[i++] = *space_start++;
 355                 }
 356                 /* and reset */
 357                 space_start = space_end = 0;
 358             }
 359         }
 360         /* add non-space char */
 361         memcpy(dst_term+j, s1, s0 - s1);
 362         j += (s0 - s1);
 363         if (!q_map_match)
 364         {
 365             while (s1 < s0)
 366             {
 367                 if (strchr(REGEX_CHARS, *s1))
 368                     dst[i++] = '\\';
 369                 dst[i++] = *s1++;
 370             }
 371         }
 372         else
 373         {
 374             char tmpbuf[80];
 375             esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 376
 377             strcpy(dst + i, map[0]);
 378             i += strlen(map[0]);
 379         }
 380     }
 381     dst[i] = '\0';
 382     dst_term[j] = '\0';
 383     *src = s0;
 384     return i;
 385 }
 386
 387 /* term_101: handle term, where trunc = Process # */
 388 static int term_101(ZebraMaps zebra_maps, int reg_type,
 389                     const char **src, char *dst, int space_split,
 390                     char *dst_term)
 391 {
 392     const char *s0;
 393     const char **map;
 394     int i = 0;
 395     int j = 0;
 396
 397     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 398         return 0;
 399     s0 = *src;
 400     while (*s0)
 401     {
 402         if (*s0 == '#')
 403         {
 404             dst[i++] = '.';
 405             dst[i++] = '*';
 406             dst_term[j++] = *s0++;
 407         }
 408         else
 409         {
 410             const char *s1 = s0;
 411             int q_map_match = 0;
 412             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 413                                     &q_map_match);
 414             if (space_split && **map == *CHR_SPACE)
 415                 break;
 416
 417             /* add non-space char */
 418             memcpy(dst_term+j, s1, s0 - s1);
 419             j += (s0 - s1);
 420             if (!q_map_match)
 421             {
 422                 while (s1 < s0)
 423                 {
 424                     if (strchr(REGEX_CHARS, *s1))
 425                         dst[i++] = '\\';
 426                     dst[i++] = *s1++;
 427                 }
 428             }
 429             else
 430             {
 431                 char tmpbuf[80];
 432                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 433
 434                 strcpy(dst + i, map[0]);
 435                 i += strlen(map[0]);
 436             }
 437         }
 438     }
 439     dst[i] = '\0';
 440     dst_term[j++] = '\0';
 441     *src = s0;
 442     return i;
 443 }
 444
 445 /* term_103: handle term, where trunc = re-2 (regular expressions) */
 446 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 447                     char *dst, int *errors, int space_split,
 448                     char *dst_term)
 449 {
 450     int i = 0;
 451     int j = 0;
 452     const char *s0;
 453     const char **map;
 454
 455     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 456         return 0;
 457     s0 = *src;
 458     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 459         isdigit(((const unsigned char *)s0)[1]))
 460     {
 461         *errors = s0[1] - '0';
 462         s0 += 3;
 463         if (*errors > 3)
 464             *errors = 3;
 465     }
 466     while (*s0)
 467     {
 468         if (strchr("^\\()[].*+?|-", *s0))
 469         {
 470             dst_term[j++] = *s0;
 471             dst[i++] = *s0++;
 472         }
 473         else
 474         {
 475             const char *s1 = s0;
 476             int q_map_match = 0;
 477             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 478                                     &q_map_match);
 479             if (space_split && **map == *CHR_SPACE)
 480                 break;
 481
 482             /* add non-space char */
 483             memcpy(dst_term+j, s1, s0 - s1);
 484             j += (s0 - s1);
 485             if (!q_map_match)
 486             {
 487                 while (s1 < s0)
 488                 {
 489                     if (strchr(REGEX_CHARS, *s1))
 490                         dst[i++] = '\\';
 491                     dst[i++] = *s1++;
 492                 }
 493             }
 494             else
 495             {
 496                 char tmpbuf[80];
 497                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 498
 499                 strcpy(dst + i, map[0]);
 500                 i += strlen(map[0]);
 501             }
 502         }
 503     }
 504     dst[i] = '\0';
 505     dst_term[j] = '\0';
 506     *src = s0;
 507
 508     return i;
 509 }
 510
 511 /* term_103: handle term, where trunc = re-1 (regular expressions) */
 512 static int term_102(ZebraMaps zebra_maps, int reg_type, const char **src,
 513                     char *dst, int space_split, char *dst_term)
 514 {
 515     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 516                     dst_term);
 517 }
 518
 519
 520 /* term_104: handle term, where trunc = Process # and ! */
 521 static int term_104(ZebraMaps zebra_maps, int reg_type,
 522                     const char **src, char *dst, int space_split,
 523                     char *dst_term)
 524 {
 525     const char *s0, *s1;
 526     const char **map;
 527     int i = 0;
 528     int j = 0;
 529
 530     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 531         return 0;
 532     s0 = *src;
 533     while (*s0)
 534     {
 535         if (*s0 == '?')
 536         {
 537             dst_term[j++] = *s0++;
 538             if (*s0 >= '0' && *s0 <= '9')
 539             {
 540                 int limit = 0;
 541                 while (*s0 >= '0' && *s0 <= '9')
 542                 {
 543                     limit = limit * 10 + (*s0 - '0');
 544                     dst_term[j++] = *s0++;
 545                 }
 546                 if (limit > 20)
 547                     limit = 20;
 548                 while (--limit >= 0)
 549                 {
 550                     dst[i++] = '.';
 551                     dst[i++] = '?';
 552                 }
 553             }
 554             else
 555             {
 556                 dst[i++] = '.';
 557                 dst[i++] = '*';
 558             }
 559         }
 560         else if (*s0 == '*')
 561         {
 562             dst[i++] = '.';
 563             dst[i++] = '*';
 564             dst_term[j++] = *s0++;
 565         }
 566         else if (*s0 == '#')
 567         {
 568             dst[i++] = '.';
 569             dst_term[j++] = *s0++;
 570         }
 571         else
 572         {
 573             s1 = s0;
 574             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 575             if (space_split && **map == *CHR_SPACE)
 576                 break;
 577             while (s1 < s0)
 578             {
 579                 if (strchr(REGEX_CHARS, *s1))
 580                     dst[i++] = '\\';
 581                 dst_term[j++] = *s1;
 582                 dst[i++] = *s1++;
 583             }
 584         }
 585     }
 586     dst[i] = '\0';
 587     dst_term[j++] = '\0';
 588     *src = s0;
 589     return i;
 590 }
 591
 592 /* term_105/106: handle term, where trunc = Process * and ! and right trunc */
 593 static int term_105(ZebraMaps zebra_maps, int reg_type,
 594                     const char **src, char *dst, int space_split,
 595                     char *dst_term, int right_truncate)
 596 {
 597     const char *s0, *s1;
 598     const char **map;
 599     int i = 0;
 600     int j = 0;
 601
 602     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 603         return 0;
 604     s0 = *src;
 605     while (*s0)
 606     {
 607         if (*s0 == '*')
 608         {
 609             dst[i++] = '.';
 610             dst[i++] = '*';
 611             dst_term[j++] = *s0++;
 612         }
 613         else if (*s0 == '!')
 614         {
 615             dst[i++] = '.';
 616             dst_term[j++] = *s0++;
 617         }
 618         else
 619         {
 620             s1 = s0;
 621             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 622             if (space_split && **map == *CHR_SPACE)
 623                 break;
 624             while (s1 < s0)
 625             {
 626                 if (strchr(REGEX_CHARS, *s1))
 627                     dst[i++] = '\\';
 628                 dst_term[j++] = *s1;
 629                 dst[i++] = *s1++;
 630             }
 631         }
 632     }
 633     if (right_truncate)
 634     {
 635         dst[i++] = '.';
 636         dst[i++] = '*';
 637     }
 638     dst[i] = '\0';
 639
 640     dst_term[j++] = '\0';
 641     *src = s0;
 642     return i;
 643 }
 644
 645
 646 /* gen_regular_rel - generate regular expression from relation
 647  *  val:     border value (inclusive)
 648  *  islt:    1 if <=; 0 if >=.
 649  */
 650 static void gen_regular_rel(char *dst, int val, int islt)
 651 {
 652     int dst_p;
 653     int w, d, i;
 654     int pos = 0;
 655     char numstr[20];
 656
 657     yaz_log(YLOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 658     if (val >= 0)
 659     {
 660         if (islt)
 661             strcpy(dst, "(-[0-9]+|(");
 662         else
 663             strcpy(dst, "((");
 664     }
 665     else
 666     {
 667         if (!islt)
 668         {
 669             strcpy(dst, "([0-9]+|-(");
 670             dst_p = strlen(dst);
 671             islt = 1;
 672         }
 673         else
 674         {
 675             strcpy(dst, "(-(");
 676             islt = 0;
 677         }
 678         val = -val;
 679     }
 680     dst_p = strlen(dst);
 681     sprintf(numstr, "%d", val);
 682     for (w = strlen(numstr); --w >= 0; pos++)
 683     {
 684         d = numstr[w];
 685         if (pos > 0)
 686         {
 687             if (islt)
 688             {
 689                 if (d == '0')
 690                     continue;
 691                 d--;
 692             }
 693             else
 694             {
 695                 if (d == '9')
 696                     continue;
 697                 d++;
 698             }
 699         }
 700
 701         strcpy(dst + dst_p, numstr);
 702         dst_p = strlen(dst) - pos - 1;
 703
 704         if (islt)
 705         {
 706             if (d != '0')
 707             {
 708                 dst[dst_p++] = '[';
 709                 dst[dst_p++] = '0';
 710                 dst[dst_p++] = '-';
 711                 dst[dst_p++] = d;
 712                 dst[dst_p++] = ']';
 713             }
 714             else
 715                 dst[dst_p++] = d;
 716         }
 717         else
 718         {
 719             if (d != '9')
 720             {
 721                 dst[dst_p++] = '[';
 722                 dst[dst_p++] = d;
 723                 dst[dst_p++] = '-';
 724                 dst[dst_p++] = '9';
 725                 dst[dst_p++] = ']';
 726             }
 727             else
 728                 dst[dst_p++] = d;
 729         }
 730         for (i = 0; i<pos; i++)
 731         {
 732             dst[dst_p++] = '[';
 733             dst[dst_p++] = '0';
 734             dst[dst_p++] = '-';
 735             dst[dst_p++] = '9';
 736             dst[dst_p++] = ']';
 737         }
 738         dst[dst_p++] = '|';
 739     }
 740     dst[dst_p] = '\0';
 741     if (islt)
 742     {
 743         /* match everything less than 10^(pos-1) */
 744         strcat(dst, "0*");
 745         for (i = 1; i<pos; i++)
 746             strcat(dst, "[0-9]?");
 747     }
 748     else
 749     {
 750         /* match everything greater than 10^pos */
 751         for (i = 0; i <= pos; i++)
 752             strcat(dst, "[0-9]");
 753         strcat(dst, "[0-9]*");
 754     }
 755     strcat(dst, "))");
 756 }
 757
 758 void string_rel_add_char(char **term_p, const char *src, int *indx)
 759 {
 760     if (src[*indx] == '\\')
 761         *(*term_p)++ = src[(*indx)++];
 762     *(*term_p)++ = src[(*indx)++];
 763 }
 764
 765 /*
 766  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 767  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 768  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 769  *              ([^-a].*|a[^-b].*|ab[c-].*)
 770  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 771  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 772  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 773  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 774  */
 775 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 776                            const char **term_sub, char *term_dict,
 777                            oid_value attributeSet,
 778                            int reg_type, int space_split, char *term_dst,
 779                            int *error_code)
 780 {
 781     AttrType relation;
 782     int relation_value;
 783     int i;
 784     char *term_tmp = term_dict + strlen(term_dict);
 785     char term_component[2*IT_MAX_WORD+20];
 786
 787     attr_init(&relation, zapt, 2);
 788     relation_value = attr_find(&relation, NULL);
 789
 790     *error_code = 0;
 791     yaz_log(YLOG_DEBUG, "string relation value=%d", relation_value);
 792     switch (relation_value)
 793     {
 794     case 1:
 795         if (!term_100(zh->reg->zebra_maps, reg_type,
 796                       term_sub, term_component,
 797                       space_split, term_dst))
 798             return 0;
 799         yaz_log(log_level_rpn, "Relation <");
 800
 801         *term_tmp++ = '(';
 802         for (i = 0; term_component[i]; )
 803         {
 804             int j = 0;
 805
 806             if (i)
 807                 *term_tmp++ = '|';
 808             while (j < i)
 809                 string_rel_add_char(&term_tmp, term_component, &j);
 810
 811             *term_tmp++ = '[';
 812
 813             *term_tmp++ = '^';
 814             string_rel_add_char(&term_tmp, term_component, &i);
 815             *term_tmp++ = '-';
 816
 817             *term_tmp++ = ']';
 818             *term_tmp++ = '.';
 819             *term_tmp++ = '*';
 820
 821             if ((term_tmp - term_dict) > IT_MAX_WORD)
 822                 break;
 823         }
 824         *term_tmp++ = ')';
 825         *term_tmp = '\0';
 826         break;
 827     case 2:
 828         if (!term_100(zh->reg->zebra_maps, reg_type,
 829                       term_sub, term_component,
 830                       space_split, term_dst))
 831             return 0;
 832         yaz_log(log_level_rpn, "Relation <=");
 833
 834         *term_tmp++ = '(';
 835         for (i = 0; term_component[i]; )
 836         {
 837             int j = 0;
 838
 839             while (j < i)
 840                 string_rel_add_char(&term_tmp, term_component, &j);
 841             *term_tmp++ = '[';
 842
 843             *term_tmp++ = '^';
 844             string_rel_add_char(&term_tmp, term_component, &i);
 845             *term_tmp++ = '-';
 846
 847             *term_tmp++ = ']';
 848             *term_tmp++ = '.';
 849             *term_tmp++ = '*';
 850
 851             *term_tmp++ = '|';
 852
 853             if ((term_tmp - term_dict) > IT_MAX_WORD)
 854                 break;
 855         }
 856         for (i = 0; term_component[i]; )
 857             string_rel_add_char(&term_tmp, term_component, &i);
 858         *term_tmp++ = ')';
 859         *term_tmp = '\0';
 860         break;
 861     case 5:
 862         if (!term_100 (zh->reg->zebra_maps, reg_type,
 863                        term_sub, term_component, space_split, term_dst))
 864             return 0;
 865         yaz_log(log_level_rpn, "Relation >");
 866
 867         *term_tmp++ = '(';
 868         for (i = 0; term_component[i];)
 869         {
 870             int j = 0;
 871
 872             while (j < i)
 873                 string_rel_add_char(&term_tmp, term_component, &j);
 874             *term_tmp++ = '[';
 875
 876             *term_tmp++ = '^';
 877             *term_tmp++ = '-';
 878             string_rel_add_char(&term_tmp, term_component, &i);
 879
 880             *term_tmp++ = ']';
 881             *term_tmp++ = '.';
 882             *term_tmp++ = '*';
 883
 884             *term_tmp++ = '|';
 885
 886             if ((term_tmp - term_dict) > IT_MAX_WORD)
 887                 break;
 888         }
 889         for (i = 0; term_component[i];)
 890             string_rel_add_char(&term_tmp, term_component, &i);
 891         *term_tmp++ = '.';
 892         *term_tmp++ = '+';
 893         *term_tmp++ = ')';
 894         *term_tmp = '\0';
 895         break;
 896     case 4:
 897         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 898                       term_component, space_split, term_dst))
 899             return 0;
 900         yaz_log(log_level_rpn, "Relation >=");
 901
 902         *term_tmp++ = '(';
 903         for (i = 0; term_component[i];)
 904         {
 905             int j = 0;
 906
 907             if (i)
 908                 *term_tmp++ = '|';
 909             while (j < i)
 910                 string_rel_add_char(&term_tmp, term_component, &j);
 911             *term_tmp++ = '[';
 912
 913             if (term_component[i+1])
 914             {
 915                 *term_tmp++ = '^';
 916                 *term_tmp++ = '-';
 917                 string_rel_add_char(&term_tmp, term_component, &i);
 918             }
 919             else
 920             {
 921                 string_rel_add_char(&term_tmp, term_component, &i);
 922                 *term_tmp++ = '-';
 923             }
 924             *term_tmp++ = ']';
 925             *term_tmp++ = '.';
 926             *term_tmp++ = '*';
 927
 928             if ((term_tmp - term_dict) > IT_MAX_WORD)
 929                 break;
 930         }
 931         *term_tmp++ = ')';
 932         *term_tmp = '\0';
 933         break;
 934     case 3:
 935     case 102:
 936     case -1:
 937         yaz_log(log_level_rpn, "Relation =");
 938         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 939                       term_component, space_split, term_dst))
 940             return 0;
 941         strcat(term_tmp, "(");
 942         strcat(term_tmp, term_component);
 943         strcat(term_tmp, ")");
 944         break;
 945     default:
 946         *error_code = 117;
 947         return 0;
 948     }
 949     return 1;
 950 }
 951
 952 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 953                              const char **term_sub,
 954                              oid_value attributeSet, NMEM stream,
 955                              struct grep_info *grep_info,
 956                              int reg_type, int complete_flag,
 957                              int num_bases, char **basenames,
 958                              char *term_dst, int xpath_use);
 959
 960 static ZEBRA_RES term_trunc(ZebraHandle zh,
 961                             Z_AttributesPlusTerm *zapt,
 962                             const char **term_sub,
 963                             oid_value attributeSet, NMEM stream,
 964                             struct grep_info *grep_info,
 965                             int reg_type, int complete_flag,
 966                             int num_bases, char **basenames,
 967                             char *term_dst,
 968                             const char *rank_type, int xpath_use,
 969                             NMEM rset_nmem,
 970                             RSET *rset,
 971                             struct rset_key_control *kc)
 972 {
 973     ZEBRA_RES res;
 974     *rset = 0;
 975     grep_info->isam_p_indx = 0;
 976     res = string_term(zh, zapt, term_sub, attributeSet, stream, grep_info,
 977                       reg_type, complete_flag, num_bases, basenames,
 978                       term_dst, xpath_use);
 979     if (res != ZEBRA_OK)
 980         return res;
 981     if (!*term_sub)  /* no more terms ? */
 982         return res;
 983     yaz_log(log_level_rpn, "term: %s", term_dst);
 984     *rset = rset_trunc(zh, grep_info->isam_p_buf,
 985                        grep_info->isam_p_indx, term_dst,
 986                        strlen(term_dst), rank_type, 1 /* preserve pos */,
 987                        zapt->term->which, rset_nmem,
 988                        kc, kc->scope);
 989     if (!*rset)
 990         return ZEBRA_FAIL;
 991     return ZEBRA_OK;
 992 }
 993
 994 static char *nmem_strdup_i(NMEM nmem, int v)
 995 {
 996     char val_str[64];
 997     sprintf(val_str, "%d", v);
 998     return nmem_strdup(nmem, val_str);
 999 }
1000
1001 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1002                              const char **term_sub,
1003                              oid_value attributeSet, NMEM stream,
1004                              struct grep_info *grep_info,
1005                              int reg_type, int complete_flag,
1006                              int num_bases, char **basenames,
1007                              char *term_dst, int xpath_use)
1008 {
1009     char term_dict[2*IT_MAX_WORD+4000];
1010     int j, r, base_no;
1011     AttrType truncation;
1012     int truncation_value;
1013     AttrType use;
1014     int use_value;
1015     const char *use_string = 0;
1016     oid_value curAttributeSet = attributeSet;
1017     const char *termp;
1018     struct rpn_char_map_info rcmi;
1019     int space_split = complete_flag ? 0 : 1;
1020
1021     int bases_ok = 0;     /* no of databases with OK attribute */
1022     int errCode = 0;      /* err code (if any is not OK) */
1023     char *errString = 0;  /* addinfo */
1024
1025     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1026     attr_init(&use, zapt, 1);
1027     use_value = attr_find_ex(&use, &curAttributeSet, &use_string);
1028     yaz_log(log_level_rpn, "string_term, use value %d", use_value);
1029     attr_init(&truncation, zapt, 5);
1030     truncation_value = attr_find(&truncation, NULL);
1031     yaz_log(log_level_rpn, "truncation value %d", truncation_value);
1032
1033     if (use_value == -1)    /* no attribute - assumy "any" */
1034         use_value = 1016;
1035     for (base_no = 0; base_no < num_bases; base_no++)
1036     {
1037         int ord = -1;
1038         int attr_ok = 0;
1039         int regex_range = 0;
1040         int init_pos = 0;
1041         attent attp;
1042         data1_local_attribute id_xpath_attr;
1043         data1_local_attribute *local_attr;
1044         int max_pos, prefix_len = 0;
1045         int relation_error;
1046
1047         termp = *term_sub;
1048
1049         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1050         {
1051             zh->errCode = YAZ_BIB1_DATABASE_UNAVAILABLE;
1052             zh->errString = basenames[base_no];
1053             return ZEBRA_FAIL;
1054         }
1055         if (xpath_use > 0 && use_value == -2)
1056         {
1057             /* xpath mode and we have a string attribute */
1058             attp.local_attributes = &id_xpath_attr;
1059             attp.attset_ordinal = VAL_IDXPATH;
1060             id_xpath_attr.next = 0;
1061
1062             use_value = xpath_use;  /* xpath_use as use-attribute now */
1063             id_xpath_attr.local = use_value;
1064         }
1065         else if (curAttributeSet == VAL_IDXPATH && use_value >= 0)
1066         {
1067             /* X-Path attribute, use numeric value directly */
1068             attp.local_attributes = &id_xpath_attr;
1069             attp.attset_ordinal = VAL_IDXPATH;
1070             id_xpath_attr.next = 0;
1071             id_xpath_attr.local = use_value;
1072         }
1073         else if (use_string &&
1074                  (ord = zebraExplain_lookup_attr_str(zh->reg->zei,
1075                                                      use_string)) >= 0)
1076         {
1077             /* we have a match for a raw string attribute */
1078             char ord_buf[32];
1079             int i, ord_len;
1080
1081             if (prefix_len)
1082                 term_dict[prefix_len++] = '|';
1083             else
1084                 term_dict[prefix_len++] = '(';
1085
1086             ord_len = key_SU_encode (ord, ord_buf);
1087             for (i = 0; i<ord_len; i++)
1088             {
1089                 term_dict[prefix_len++] = 1;
1090                 term_dict[prefix_len++] = ord_buf[i];
1091             }
1092             attp.local_attributes = 0;  /* no more attributes */
1093         }
1094         else
1095         {
1096             /* lookup in the .att files . Allow string as well */
1097             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1098                                       use_string)))
1099             {
1100                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1101                         curAttributeSet, use_value, r);
1102                 if (r == -1)
1103                 {
1104                     /* set was found, but value wasn't defined */
1105                     errCode = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
1106                     if (use_string)
1107                         errString = nmem_strdup(stream, use_string);
1108                     else
1109                         errString = nmem_strdup_i (stream, use_value);
1110                 }
1111                 else
1112                 {
1113                     int oid[OID_SIZE];
1114                     struct oident oident;
1115
1116                     oident.proto = PROTO_Z3950;
1117                     oident.oclass = CLASS_ATTSET;
1118                     oident.value = curAttributeSet;
1119                     oid_ent_to_oid (&oident, oid);
1120
1121                     errCode = YAZ_BIB1_UNSUPP_ATTRIBUTE_SET;
1122                     errString = nmem_strdup(stream, oident.desc);
1123                 }
1124                 continue;
1125             }
1126         }
1127         for (local_attr = attp.local_attributes; local_attr;
1128              local_attr = local_attr->next)
1129         {
1130             char ord_buf[32];
1131             int i, ord_len;
1132
1133             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1134                                               attp.attset_ordinal,
1135                                               local_attr->local);
1136             if (ord < 0)
1137                 continue;
1138             if (prefix_len)
1139                 term_dict[prefix_len++] = '|';
1140             else
1141                 term_dict[prefix_len++] = '(';
1142
1143             ord_len = key_SU_encode (ord, ord_buf);
1144             for (i = 0; i<ord_len; i++)
1145             {
1146                 term_dict[prefix_len++] = 1;
1147                 term_dict[prefix_len++] = ord_buf[i];
1148             }
1149         }
1150         bases_ok++;
1151         if (prefix_len)
1152             attr_ok = 1;
1153
1154         term_dict[prefix_len++] = ')';
1155         term_dict[prefix_len++] = 1;
1156         term_dict[prefix_len++] = reg_type;
1157         yaz_log(log_level_rpn, "reg_type = %d", term_dict[prefix_len-1]);
1158         term_dict[prefix_len] = '\0';
1159         j = prefix_len;
1160         switch (truncation_value)
1161         {
1162         case -1:         /* not specified */
1163         case 100:        /* do not truncate */
1164             if (!string_relation (zh, zapt, &termp, term_dict,
1165                                   attributeSet,
1166                                   reg_type, space_split, term_dst,
1167                                   &relation_error))
1168             {
1169                 if (relation_error)
1170                 {
1171                     zh->errCode = relation_error;
1172                     return ZEBRA_FAIL;
1173                 }
1174                 *term_sub = 0;
1175                 return ZEBRA_OK;
1176             }
1177             break;
1178         case 1:          /* right truncation */
1179             term_dict[j++] = '(';
1180             if (!term_100(zh->reg->zebra_maps, reg_type,
1181                           &termp, term_dict + j, space_split, term_dst))
1182             {
1183                 *term_sub = 0;
1184                 return ZEBRA_OK;
1185             }
1186             strcat(term_dict, ".*)");
1187             break;
1188         case 2:          /* keft truncation */
1189             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1190             if (!term_100(zh->reg->zebra_maps, reg_type,
1191                           &termp, term_dict + j, space_split, term_dst))
1192             {
1193                 *term_sub = 0;
1194                 return ZEBRA_OK;
1195             }
1196             strcat(term_dict, ")");
1197             break;
1198         case 3:          /* left&right truncation */
1199             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1200             if (!term_100(zh->reg->zebra_maps, reg_type,
1201                           &termp, term_dict + j, space_split, term_dst))
1202             {
1203                 *term_sub = 0;
1204                 return ZEBRA_OK;
1205             }
1206             strcat(term_dict, ".*)");
1207             break;
1208         case 101:        /* process # in term */
1209             term_dict[j++] = '(';
1210             if (!term_101(zh->reg->zebra_maps, reg_type,
1211                           &termp, term_dict + j, space_split, term_dst))
1212             {
1213                 *term_sub = 0;
1214                 return ZEBRA_OK;
1215             }
1216             strcat(term_dict, ")");
1217             break;
1218         case 102:        /* Regexp-1 */
1219             term_dict[j++] = '(';
1220             if (!term_102(zh->reg->zebra_maps, reg_type,
1221                           &termp, term_dict + j, space_split, term_dst))
1222             {
1223                 *term_sub = 0;
1224                 return ZEBRA_OK;
1225             }
1226             strcat(term_dict, ")");
1227             break;
1228         case 103:       /* Regexp-2 */
1229             regex_range = 1;
1230             term_dict[j++] = '(';
1231             init_pos = 2;
1232             if (!term_103(zh->reg->zebra_maps, reg_type,
1233                           &termp, term_dict + j, &regex_range,
1234                           space_split, term_dst))
1235             {
1236                 *term_sub = 0;
1237                 return ZEBRA_OK;
1238             }
1239             strcat(term_dict, ")");
1240         case 104:        /* process # and ! in term */
1241             term_dict[j++] = '(';
1242             if (!term_104(zh->reg->zebra_maps, reg_type,
1243                           &termp, term_dict + j, space_split, term_dst))
1244             {
1245                 *term_sub = 0;
1246                 return ZEBRA_OK;
1247             }
1248             strcat(term_dict, ")");
1249             break;
1250         case 105:        /* process * and ! in term */
1251             term_dict[j++] = '(';
1252             if (!term_105(zh->reg->zebra_maps, reg_type,
1253                           &termp, term_dict + j, space_split, term_dst, 1))
1254             {
1255                 *term_sub = 0;
1256                 return ZEBRA_OK;
1257             }
1258             strcat(term_dict, ")");
1259             break;
1260         case 106:        /* process * and ! in term */
1261             term_dict[j++] = '(';
1262             if (!term_105(zh->reg->zebra_maps, reg_type,
1263                           &termp, term_dict + j, space_split, term_dst, 0))
1264             {
1265                 *term_sub = 0;
1266                 return ZEBRA_OK;
1267             }
1268             strcat(term_dict, ")");
1269             break;
1270         default:
1271             zh->errCode = YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE;
1272             zh->errString = nmem_strdup_i(stream, truncation_value);
1273             return ZEBRA_FAIL;
1274         }
1275         if (attr_ok)
1276         {
1277             char buf[80];
1278             const char *input = term_dict + prefix_len;
1279             esc_str(buf, sizeof(buf), input, strlen(input));
1280         }
1281         if (attr_ok)
1282         {
1283             yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_dict+prefix_len);
1284             r = dict_lookup_grep(zh->reg->dict, term_dict, regex_range,
1285                                  grep_info, &max_pos, init_pos,
1286                                  grep_handle);
1287             if (r)
1288                 yaz_log(YLOG_WARN, "dict_lookup_grep fail %d", r);
1289         }
1290     }
1291     if (!bases_ok)
1292     {
1293         zh->errCode = errCode;
1294         zh->errString = errString;
1295         return ZEBRA_FAIL;
1296     }
1297     *term_sub = termp;
1298     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1299     return ZEBRA_OK;
1300 }
1301
1302
1303 /* convert APT search term to UTF8 */
1304 static ZEBRA_RES zapt_term_to_utf8(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1305                                    char *termz)
1306 {
1307     size_t sizez;
1308     Z_Term *term = zapt->term;
1309
1310     switch (term->which)
1311     {
1312     case Z_Term_general:
1313         if (zh->iconv_to_utf8 != 0)
1314         {
1315             char *inbuf = term->u.general->buf;
1316             size_t inleft = term->u.general->len;
1317             char *outbuf = termz;
1318             size_t outleft = IT_MAX_WORD-1;
1319             size_t ret;
1320
1321             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1322                         &outbuf, &outleft);
1323             if (ret == (size_t)(-1))
1324             {
1325                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1326                 zh->errCode =
1327                     YAZ_BIB1_QUERY_TERM_INCLUDES_CHARS_THAT_DO_NOT_TRANSLATE_INTO_;
1328                 return -1;
1329             }
1330             *outbuf = 0;
1331         }
1332         else
1333         {
1334             sizez = term->u.general->len;
1335             if (sizez > IT_MAX_WORD-1)
1336                 sizez = IT_MAX_WORD-1;
1337             memcpy (termz, term->u.general->buf, sizez);
1338             termz[sizez] = '\0';
1339         }
1340         break;
1341     case Z_Term_characterString:
1342         sizez = strlen(term->u.characterString);
1343         if (sizez > IT_MAX_WORD-1)
1344             sizez = IT_MAX_WORD-1;
1345         memcpy (termz, term->u.characterString, sizez);
1346         termz[sizez] = '\0';
1347         break;
1348     default:
1349         zh->errCode = YAZ_BIB1_UNSUPP_CODED_VALUE_FOR_TERM;
1350         return ZEBRA_FAIL;
1351     }
1352     return ZEBRA_OK;
1353 }
1354
1355 /* convert APT SCAN term to internal cmap */
1356 static ZEBRA_RES trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1357                                  char *termz, int reg_type)
1358 {
1359     char termz0[IT_MAX_WORD];
1360
1361     if (zapt_term_to_utf8(zh, zapt, termz0) == ZEBRA_FAIL)
1362         return ZEBRA_FAIL;    /* error */
1363     else
1364     {
1365         const char **map;
1366         const char *cp = (const char *) termz0;
1367         const char *cp_end = cp + strlen(cp);
1368         const char *src;
1369         int i = 0;
1370         const char *space_map = NULL;
1371         int len;
1372
1373         while ((len = (cp_end - cp)) > 0)
1374         {
1375             map = zebra_maps_input(zh->reg->zebra_maps, reg_type, &cp, len, 0);
1376             if (**map == *CHR_SPACE)
1377                 space_map = *map;
1378             else
1379             {
1380                 if (i && space_map)
1381                     for (src = space_map; *src; src++)
1382                         termz[i++] = *src;
1383                 space_map = NULL;
1384                 for (src = *map; *src; src++)
1385                     termz[i++] = *src;
1386             }
1387         }
1388         termz[i] = '\0';
1389     }
1390     return ZEBRA_OK;
1391 }
1392
1393 char *normalize_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1394                      const char *termz, NMEM stream, unsigned reg_id)
1395 {
1396     WRBUF wrbuf = 0;
1397     AttrType truncation;
1398     int truncation_value;
1399     char *ex_list = 0;
1400
1401     attr_init(&truncation, zapt, 5);
1402     truncation_value = attr_find(&truncation, NULL);
1403
1404     switch (truncation_value)
1405     {
1406     default:
1407         ex_list = "";
1408         break;
1409     case 101:
1410         ex_list = "#";
1411         break;
1412     case 102:
1413     case 103:
1414         ex_list = 0;
1415         break;
1416     case 104:
1417         ex_list = "!#";
1418         break;
1419     case 105:
1420         ex_list = "!*";
1421         break;
1422     }
1423     if (ex_list)
1424         wrbuf = zebra_replace(zh->reg->zebra_maps, reg_id, ex_list,
1425                               termz, strlen(termz));
1426     if (!wrbuf)
1427         return nmem_strdup(stream, termz);
1428     else
1429     {
1430         char *buf = (char*) nmem_malloc(stream, wrbuf_len(wrbuf)+1);
1431         memcpy (buf, wrbuf_buf(wrbuf), wrbuf_len(wrbuf));
1432         buf[wrbuf_len(wrbuf)] = '\0';
1433         return buf;
1434     }
1435 }
1436
1437 static void grep_info_delete(struct grep_info *grep_info)
1438 {
1439 #ifdef TERM_COUNT
1440     xfree(grep_info->term_no);
1441 #endif
1442     xfree(grep_info->isam_p_buf);
1443 }
1444
1445 static int grep_info_prepare(ZebraHandle zh,
1446                              Z_AttributesPlusTerm *zapt,
1447                              struct grep_info *grep_info,
1448                              int reg_type,
1449                              NMEM stream)
1450 {
1451     AttrType termset;
1452     int termset_value_numeric;
1453     const char *termset_value_string;
1454
1455 #ifdef TERM_COUNT
1456     grep_info->term_no = 0;
1457 #endif
1458     grep_info->isam_p_size = 0;
1459     grep_info->isam_p_buf = NULL;
1460     grep_info->zh = zh;
1461     grep_info->reg_type = reg_type;
1462     grep_info->termset = 0;
1463
1464     if (!zapt)
1465         return 0;
1466     attr_init(&termset, zapt, 8);
1467     termset_value_numeric =
1468         attr_find_ex(&termset, NULL, &termset_value_string);
1469     if (termset_value_numeric != -1)
1470     {
1471         char resname[32];
1472         const char *termset_name = 0;
1473         if (termset_value_numeric != -2)
1474         {
1475
1476             sprintf(resname, "%d", termset_value_numeric);
1477             termset_name = resname;
1478         }
1479         else
1480             termset_name = termset_value_string;
1481         yaz_log(log_level_rpn, "creating termset set %s", termset_name);
1482         grep_info->termset = resultSetAdd(zh, termset_name, 1);
1483         if (!grep_info->termset)
1484         {
1485             zh->errCode = YAZ_BIB1_ILLEGAL_RESULT_SET_NAME;
1486             zh->errString = nmem_strdup(stream, termset_name);
1487             return -1;
1488         }
1489     }
1490     return 0;
1491 }
1492
1493
1494 static ZEBRA_RES term_list_trunc(ZebraHandle zh,
1495                                  Z_AttributesPlusTerm *zapt,
1496                                  const char *termz_org,
1497                                  oid_value attributeSet,
1498                                  NMEM stream,
1499                                  int reg_type, int complete_flag,
1500                                  const char *rank_type, int xpath_use,
1501                                  int num_bases, char **basenames,
1502                                  NMEM rset_nmem,
1503                                  RSET **result_sets, int *num_result_sets,
1504                                  struct rset_key_control *kc)
1505 {
1506     char term_dst[IT_MAX_WORD+1];
1507     struct grep_info grep_info;
1508     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1509     const char *termp = termz;
1510     int alloc_sets = 0;
1511
1512     *num_result_sets = 0;
1513     *term_dst = 0;
1514     if (grep_info_prepare(zh, zapt, &grep_info, reg_type, stream))
1515         return ZEBRA_FAIL;
1516     while(1)
1517     {
1518         ZEBRA_RES res;
1519
1520         if (alloc_sets == *num_result_sets)
1521         {
1522             int add = 10;
1523             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1524                                               sizeof(*rnew));
1525             if (alloc_sets)
1526                 memcpy(rnew, *result_sets, alloc_sets * sizeof(*rnew));
1527             alloc_sets = alloc_sets + add;
1528             *result_sets = rnew;
1529         }
1530         res = term_trunc(zh, zapt, &termp, attributeSet,
1531                          stream, &grep_info,
1532                          reg_type, complete_flag,
1533                          num_bases, basenames,
1534                          term_dst, rank_type,
1535                          xpath_use, rset_nmem,
1536                          &(*result_sets)[*num_result_sets],
1537                          kc);
1538         if (res != ZEBRA_OK)
1539         {
1540             int i;
1541             for (i = 0; i < *num_result_sets; i++)
1542                 rset_delete((*result_sets)[i]);
1543             grep_info_delete (&grep_info);
1544             return res;
1545         }
1546         if ((*result_sets)[*num_result_sets] == 0)
1547             break;
1548         (*num_result_sets)++;
1549     }
1550     grep_info_delete(&grep_info);
1551     return ZEBRA_OK;
1552 }
1553
1554 static ZEBRA_RES rpn_search_APT_phrase(ZebraHandle zh,
1555                                        Z_AttributesPlusTerm *zapt,
1556                                        const char *termz_org,
1557                                        oid_value attributeSet,
1558                                        NMEM stream,
1559                                        int reg_type, int complete_flag,
1560                                        const char *rank_type, int xpath_use,
1561                                        int num_bases, char **basenames,
1562                                        NMEM rset_nmem,
1563                                        RSET *rset,
1564                                        struct rset_key_control *kc)
1565 {
1566     RSET *result_sets = 0;
1567     int num_result_sets = 0;
1568     ZEBRA_RES res =
1569         term_list_trunc(zh, zapt, termz_org, attributeSet,
1570                         stream, reg_type, complete_flag,
1571                         rank_type, xpath_use,
1572                         num_bases, basenames,
1573                         rset_nmem,
1574                         &result_sets, &num_result_sets, kc);
1575     if (res != ZEBRA_OK)
1576         return res;
1577     if (num_result_sets == 0)
1578         *rset = rsnull_create (rset_nmem, kc);
1579     else if (num_result_sets == 1)
1580         *rset = result_sets[0];
1581     else
1582         *rset = rsprox_create(rset_nmem, kc, kc->scope,
1583                               num_result_sets, result_sets,
1584                               1 /* ordered */, 0 /* exclusion */,
1585                               3 /* relation */, 1 /* distance */);
1586     if (!*rset)
1587         return ZEBRA_FAIL;
1588     return ZEBRA_OK;
1589 }
1590
1591 static ZEBRA_RES rpn_search_APT_or_list(ZebraHandle zh,
1592                                         Z_AttributesPlusTerm *zapt,
1593                                         const char *termz_org,
1594                                         oid_value attributeSet,
1595                                         NMEM stream,
1596                                         int reg_type, int complete_flag,
1597                                         const char *rank_type,
1598                                         int xpath_use,
1599                                         int num_bases, char **basenames,
1600                                         NMEM rset_nmem,
1601                                         RSET *rset,
1602                                         struct rset_key_control *kc)
1603 {
1604     RSET *result_sets = 0;
1605     int num_result_sets = 0;
1606     ZEBRA_RES res =
1607         term_list_trunc(zh, zapt, termz_org, attributeSet,
1608                         stream, reg_type, complete_flag,
1609                         rank_type, xpath_use,
1610                         num_bases, basenames,
1611                         rset_nmem,
1612                         &result_sets, &num_result_sets, kc);
1613     if (res != ZEBRA_OK)
1614         return res;
1615     if (num_result_sets == 0)
1616         *rset = rsnull_create (rset_nmem, kc);
1617     else if (num_result_sets == 1)
1618         *rset = result_sets[0];
1619     else
1620         *rset = rsmulti_or_create(rset_nmem, kc, kc->scope,
1621                                   num_result_sets, result_sets);
1622     if (!*rset)
1623         return ZEBRA_FAIL;
1624     return ZEBRA_OK;
1625 }
1626
1627 static ZEBRA_RES rpn_search_APT_and_list(ZebraHandle zh,
1628                                          Z_AttributesPlusTerm *zapt,
1629                                          const char *termz_org,
1630                                          oid_value attributeSet,
1631                                          NMEM stream,
1632                                          int reg_type, int complete_flag,
1633                                          const char *rank_type,
1634                                          int xpath_use,
1635                                          int num_bases, char **basenames,
1636                                          NMEM rset_nmem,
1637                                          RSET *rset,
1638                                          struct rset_key_control *kc)
1639 {
1640     RSET *result_sets = 0;
1641     int num_result_sets = 0;
1642     ZEBRA_RES res =
1643         term_list_trunc(zh, zapt, termz_org, attributeSet,
1644                         stream, reg_type, complete_flag,
1645                         rank_type, xpath_use,
1646                         num_bases, basenames,
1647                         rset_nmem,
1648                         &result_sets, &num_result_sets,
1649                         kc);
1650     if (res != ZEBRA_OK)
1651         return res;
1652     if (num_result_sets == 0)
1653         *rset = rsnull_create (rset_nmem, kc);
1654     else if (num_result_sets == 1)
1655         *rset = result_sets[0];
1656     else
1657         *rset = rsmulti_and_create(rset_nmem, kc, kc->scope,
1658                                    num_result_sets, result_sets);
1659     if (!*rset)
1660         return ZEBRA_FAIL;
1661     return ZEBRA_OK;
1662 }
1663
1664 static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1665                             const char **term_sub,
1666                             char *term_dict,
1667                             oid_value attributeSet,
1668                             struct grep_info *grep_info,
1669                             int *max_pos,
1670                             int reg_type,
1671                             char *term_dst,
1672                             int *error_code)
1673 {
1674     AttrType relation;
1675     int relation_value;
1676     int term_value;
1677     int r;
1678     char *term_tmp = term_dict + strlen(term_dict);
1679
1680     *error_code = 0;
1681     attr_init(&relation, zapt, 2);
1682     relation_value = attr_find(&relation, NULL);
1683
1684     yaz_log(log_level_rpn, "numeric relation value=%d", relation_value);
1685
1686     if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1687                   term_dst))
1688         return 0;
1689     term_value = atoi (term_tmp);
1690     switch (relation_value)
1691     {
1692     case 1:
1693         yaz_log(log_level_rpn, "Relation <");
1694         gen_regular_rel(term_tmp, term_value-1, 1);
1695         break;
1696     case 2:
1697         yaz_log(log_level_rpn, "Relation <=");
1698         gen_regular_rel(term_tmp, term_value, 1);
1699         break;
1700     case 4:
1701         yaz_log(log_level_rpn, "Relation >=");
1702         gen_regular_rel(term_tmp, term_value, 0);
1703         break;
1704     case 5:
1705         yaz_log(log_level_rpn, "Relation >");
1706         gen_regular_rel(term_tmp, term_value+1, 0);
1707         break;
1708     case -1:
1709     case 3:
1710         yaz_log(log_level_rpn, "Relation =");
1711         sprintf(term_tmp, "(0*%d)", term_value);
1712         break;
1713     default:
1714         *error_code = 117;
1715         return 0;
1716     }
1717     yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_tmp);
1718     r = dict_lookup_grep(zh->reg->dict, term_dict, 0, grep_info, max_pos,
1719                           0, grep_handle);
1720     if (r)
1721         yaz_log(YLOG_WARN, "dict_lookup_grep fail, rel = gt: %d", r);
1722     yaz_log(log_level_rpn, "%d positions", grep_info->isam_p_indx);
1723     return 1;
1724 }
1725
1726 static ZEBRA_RES numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1727                               const char **term_sub,
1728                               oid_value attributeSet,
1729                               struct grep_info *grep_info,
1730                               int reg_type, int complete_flag,
1731                               int num_bases, char **basenames,
1732                               char *term_dst, int xpath_use, NMEM stream)
1733 {
1734     char term_dict[2*IT_MAX_WORD+2];
1735     int r, base_no;
1736     AttrType use;
1737     int use_value;
1738     const char *use_string = 0;
1739     oid_value curAttributeSet = attributeSet;
1740     const char *termp;
1741     struct rpn_char_map_info rcmi;
1742
1743     int bases_ok = 0;     /* no of databases with OK attribute */
1744     int errCode = 0;      /* err code (if any is not OK) */
1745     char *errString = 0;  /* addinfo */
1746
1747     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1748     attr_init(&use, zapt, 1);
1749     use_value = attr_find_ex(&use, &curAttributeSet, &use_string);
1750
1751     if (use_value == -1)
1752         use_value = 1016;
1753
1754     for (base_no = 0; base_no < num_bases; base_no++)
1755     {
1756         attent attp;
1757         data1_local_attribute id_xpath_attr;
1758         data1_local_attribute *local_attr;
1759         int max_pos, prefix_len = 0;
1760         int relation_error = 0;
1761
1762         termp = *term_sub;
1763         if (use_value == -2)  /* string attribute (assume IDXPATH/any) */
1764         {
1765             use_value = xpath_use;
1766             attp.local_attributes = &id_xpath_attr;
1767             attp.attset_ordinal = VAL_IDXPATH;
1768             id_xpath_attr.next = 0;
1769             id_xpath_attr.local = use_value;
1770         }
1771         else if (curAttributeSet == VAL_IDXPATH)
1772         {
1773             attp.local_attributes = &id_xpath_attr;
1774             attp.attset_ordinal = VAL_IDXPATH;
1775             id_xpath_attr.next = 0;
1776             id_xpath_attr.local = use_value;
1777         }
1778         else
1779         {
1780             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1781                                             use_string)))
1782             {
1783                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1784                       curAttributeSet, use_value, r);
1785                 if (r == -1)
1786                 {
1787                     errCode = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
1788                     if (use_string)
1789                         errString = nmem_strdup(stream, use_string);
1790                     else
1791                         errString = nmem_strdup_i (stream, use_value);
1792                 }
1793                 else
1794                     errCode = YAZ_BIB1_UNSUPP_ATTRIBUTE_SET;
1795                 continue;
1796             }
1797         }
1798         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1799         {
1800             zh->errCode = YAZ_BIB1_DATABASE_UNAVAILABLE;
1801             zh->errString = basenames[base_no];
1802             return -1;
1803         }
1804         for (local_attr = attp.local_attributes; local_attr;
1805              local_attr = local_attr->next)
1806         {
1807             int ord;
1808             char ord_buf[32];
1809             int i, ord_len;
1810
1811             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1812                                               attp.attset_ordinal,
1813                                               local_attr->local);
1814             if (ord < 0)
1815                 continue;
1816             if (prefix_len)
1817                 term_dict[prefix_len++] = '|';
1818             else
1819                 term_dict[prefix_len++] = '(';
1820
1821             ord_len = key_SU_encode (ord, ord_buf);
1822             for (i = 0; i<ord_len; i++)
1823             {
1824                 term_dict[prefix_len++] = 1;
1825                 term_dict[prefix_len++] = ord_buf[i];
1826             }
1827         }
1828         if (!prefix_len)
1829         {
1830             errCode = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
1831             errString = nmem_strdup_i(stream, use_value);
1832             continue;
1833         }
1834         bases_ok++;
1835         term_dict[prefix_len++] = ')';
1836         term_dict[prefix_len++] = 1;
1837         term_dict[prefix_len++] = reg_type;
1838         yaz_log(YLOG_DEBUG, "reg_type = %d", term_dict[prefix_len-1]);
1839         term_dict[prefix_len] = '\0';
1840         if (!numeric_relation(zh, zapt, &termp, term_dict,
1841                               attributeSet, grep_info, &max_pos, reg_type,
1842                               term_dst, &relation_error))
1843         {
1844             if (relation_error)
1845             {
1846                 zh->errCode = relation_error;
1847                 zh->errString = 0;
1848                 return ZEBRA_FAIL;
1849             }
1850             *term_sub = 0;
1851             return ZEBRA_OK;
1852         }
1853     }
1854     if (!bases_ok)
1855     {
1856         zh->errCode = errCode;
1857         zh->errString = errString;
1858         return ZEBRA_FAIL;
1859     }
1860     *term_sub = termp;
1861     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1862     return ZEBRA_OK;
1863 }
1864
1865 static ZEBRA_RES rpn_search_APT_numeric(ZebraHandle zh,
1866                                         Z_AttributesPlusTerm *zapt,
1867                                         const char *termz,
1868                                         oid_value attributeSet,
1869                                         NMEM stream,
1870                                         int reg_type, int complete_flag,
1871                                         const char *rank_type, int xpath_use,
1872                                         int num_bases, char **basenames,
1873                                         NMEM rset_nmem,
1874                                         RSET *rset,
1875                                         struct rset_key_control *kc)
1876 {
1877     char term_dst[IT_MAX_WORD+1];
1878     const char *termp = termz;
1879     RSET *result_sets = 0;
1880     int num_result_sets = 0;
1881     ZEBRA_RES res;
1882     struct grep_info grep_info;
1883     int alloc_sets = 0;
1884
1885     yaz_log(log_level_rpn, "APT_numeric t='%s'", termz);
1886     if (grep_info_prepare(zh, zapt, &grep_info, reg_type, stream))
1887         return ZEBRA_FAIL;
1888     while (1)
1889     {
1890         if (alloc_sets == num_result_sets)
1891         {
1892             int add = 10;
1893             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1894                                               sizeof(*rnew));
1895             if (alloc_sets)
1896                 memcpy(rnew, result_sets, alloc_sets * sizeof(*rnew));
1897             alloc_sets = alloc_sets + add;
1898             result_sets = rnew;
1899         }
1900         yaz_log(YLOG_DEBUG, "APT_numeric termp=%s", termp);
1901         grep_info.isam_p_indx = 0;
1902         res = numeric_term(zh, zapt, &termp, attributeSet, &grep_info,
1903                            reg_type, complete_flag, num_bases, basenames,
1904                            term_dst, xpath_use,
1905                            stream);
1906         if (res == ZEBRA_FAIL || termp == 0)
1907             break;
1908         yaz_log(YLOG_DEBUG, "term: %s", term_dst);
1909         result_sets[num_result_sets] =
1910             rset_trunc(zh, grep_info.isam_p_buf,
1911                        grep_info.isam_p_indx, term_dst,
1912                        strlen(term_dst), rank_type,
1913                        0 /* preserve position */,
1914                        zapt->term->which, rset_nmem,
1915                        kc, kc->scope);
1916         if (!result_sets[num_result_sets])
1917             break;
1918         num_result_sets++;
1919     }
1920     grep_info_delete(&grep_info);
1921     if (termp)
1922     {
1923         int i;
1924         for (i = 0; i<num_result_sets; i++)
1925             rset_delete(result_sets[i]);
1926         return ZEBRA_FAIL;
1927     }
1928     if (num_result_sets == 0)
1929         *rset = rsnull_create(rset_nmem, kc);
1930     if (num_result_sets == 1)
1931         *rset = result_sets[0];
1932     else
1933         *rset = rsmulti_and_create(rset_nmem, kc, kc->scope,
1934                                    num_result_sets, result_sets);
1935     if (!*rset)
1936         return ZEBRA_FAIL;
1937     return ZEBRA_OK;
1938 }
1939
1940 static ZEBRA_RES rpn_search_APT_local(ZebraHandle zh,
1941                                       Z_AttributesPlusTerm *zapt,
1942                                       const char *termz,
1943                                       oid_value attributeSet,
1944                                       NMEM stream,
1945                                       const char *rank_type, NMEM rset_nmem,
1946                                       RSET *rset,
1947                                       struct rset_key_control *kc)
1948 {
1949     RSFD rsfd;
1950     struct it_key key;
1951     int sys;
1952     *rset = rstemp_create(rset_nmem, kc, kc->scope,
1953                           res_get (zh->res, "setTmpDir"),0 );
1954     rsfd = rset_open(*rset, RSETF_WRITE);
1955
1956     sys = atoi(termz);
1957     if (sys <= 0)
1958         sys = 1;
1959     key.mem[0] = sys;
1960     key.mem[1] = 1;
1961     key.len = 2;
1962     rset_write (rsfd, &key);
1963     rset_close (rsfd);
1964     return ZEBRA_OK;
1965 }
1966
1967 static ZEBRA_RES rpn_sort_spec(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1968                                oid_value attributeSet, NMEM stream,
1969                                Z_SortKeySpecList *sort_sequence,
1970                                const char *rank_type,
1971                                RSET *rset,
1972                                struct rset_key_control *kc)
1973 {
1974     int i;
1975     int sort_relation_value;
1976     AttrType sort_relation_type;
1977     int use_value;
1978     AttrType use_type;
1979     Z_SortKeySpec *sks;
1980     Z_SortKey *sk;
1981     Z_AttributeElement *ae;
1982     int oid[OID_SIZE];
1983     oident oe;
1984     char termz[20];
1985
1986     attr_init(&sort_relation_type, zapt, 7);
1987     sort_relation_value = attr_find(&sort_relation_type, &attributeSet);
1988
1989     attr_init(&use_type, zapt, 1);
1990     use_value = attr_find(&use_type, &attributeSet);
1991
1992     if (!sort_sequence->specs)
1993     {
1994         sort_sequence->num_specs = 10;
1995         sort_sequence->specs = (Z_SortKeySpec **)
1996             nmem_malloc(stream, sort_sequence->num_specs *
1997                          sizeof(*sort_sequence->specs));
1998         for (i = 0; i<sort_sequence->num_specs; i++)
1999             sort_sequence->specs[i] = 0;
2000     }
2001     if (zapt->term->which != Z_Term_general)
2002         i = 0;
2003     else
2004         i = atoi_n ((char *) zapt->term->u.general->buf,
2005                     zapt->term->u.general->len);
2006     if (i >= sort_sequence->num_specs)
2007         i = 0;
2008     sprintf(termz, "%d", i);
2009
2010     oe.proto = PROTO_Z3950;
2011     oe.oclass = CLASS_ATTSET;
2012     oe.value = attributeSet;
2013     if (!oid_ent_to_oid (&oe, oid))
2014         return ZEBRA_FAIL;
2015
2016     sks = (Z_SortKeySpec *) nmem_malloc(stream, sizeof(*sks));
2017     sks->sortElement = (Z_SortElement *)
2018         nmem_malloc(stream, sizeof(*sks->sortElement));
2019     sks->sortElement->which = Z_SortElement_generic;
2020     sk = sks->sortElement->u.generic = (Z_SortKey *)
2021         nmem_malloc(stream, sizeof(*sk));
2022     sk->which = Z_SortKey_sortAttributes;
2023     sk->u.sortAttributes = (Z_SortAttributes *)
2024         nmem_malloc(stream, sizeof(*sk->u.sortAttributes));
2025
2026     sk->u.sortAttributes->id = oid;
2027     sk->u.sortAttributes->list = (Z_AttributeList *)
2028         nmem_malloc(stream, sizeof(*sk->u.sortAttributes->list));
2029     sk->u.sortAttributes->list->num_attributes = 1;
2030     sk->u.sortAttributes->list->attributes = (Z_AttributeElement **)
2031         nmem_malloc(stream, sizeof(*sk->u.sortAttributes->list->attributes));
2032     ae = *sk->u.sortAttributes->list->attributes = (Z_AttributeElement *)
2033         nmem_malloc(stream, sizeof(**sk->u.sortAttributes->list->attributes));
2034     ae->attributeSet = 0;
2035     ae->attributeType = (int *)
2036         nmem_malloc(stream, sizeof(*ae->attributeType));
2037     *ae->attributeType = 1;
2038     ae->which = Z_AttributeValue_numeric;
2039     ae->value.numeric = (int *)
2040         nmem_malloc(stream, sizeof(*ae->value.numeric));
2041     *ae->value.numeric = use_value;
2042
2043     sks->sortRelation = (int *)
2044         nmem_malloc(stream, sizeof(*sks->sortRelation));
2045     if (sort_relation_value == 1)
2046         *sks->sortRelation = Z_SortKeySpec_ascending;
2047     else if (sort_relation_value == 2)
2048         *sks->sortRelation = Z_SortKeySpec_descending;
2049     else
2050         *sks->sortRelation = Z_SortKeySpec_ascending;
2051
2052     sks->caseSensitivity = (int *)
2053         nmem_malloc(stream, sizeof(*sks->caseSensitivity));
2054     *sks->caseSensitivity = 0;
2055
2056     sks->which = Z_SortKeySpec_null;
2057     sks->u.null = odr_nullval ();
2058     sort_sequence->specs[i] = sks;
2059     *rset = rsnull_create (NULL, kc);
2060     return ZEBRA_OK;
2061 }
2062
2063
2064 static int parse_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2065                        oid_value attributeSet,
2066                        struct xpath_location_step *xpath, int max, NMEM mem)
2067 {
2068     oid_value curAttributeSet = attributeSet;
2069     AttrType use;
2070     const char *use_string = 0;
2071
2072     attr_init(&use, zapt, 1);
2073     attr_find_ex(&use, &curAttributeSet, &use_string);
2074
2075     if (!use_string || *use_string != '/')
2076         return -1;
2077
2078     return zebra_parse_xpath_str(use_string, xpath, max, mem);
2079 }
2080
2081
2082
2083 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
2084                         int reg_type, const char *term, int use,
2085                         oid_value curAttributeSet, NMEM rset_nmem,
2086                         struct rset_key_control *kc)
2087 {
2088     RSET rset;
2089     struct grep_info grep_info;
2090     char term_dict[2048];
2091     char ord_buf[32];
2092     int prefix_len = 0;
2093     int ord = zebraExplain_lookup_attr_su(zh->reg->zei, curAttributeSet, use);
2094     int ord_len, i, r, max_pos;
2095     int term_type = Z_Term_characterString;
2096     const char *flags = "void";
2097
2098     if (grep_info_prepare(zh, 0 /* zapt */, &grep_info, '0', stream))
2099         return rsnull_create(rset_nmem, kc);
2100
2101     if (ord < 0)
2102         return rsnull_create(rset_nmem, kc);
2103     if (prefix_len)
2104         term_dict[prefix_len++] = '|';
2105     else
2106         term_dict[prefix_len++] = '(';
2107
2108     ord_len = key_SU_encode (ord, ord_buf);
2109     for (i = 0; i<ord_len; i++)
2110     {
2111         term_dict[prefix_len++] = 1;
2112         term_dict[prefix_len++] = ord_buf[i];
2113     }
2114     term_dict[prefix_len++] = ')';
2115     term_dict[prefix_len++] = 1;
2116     term_dict[prefix_len++] = reg_type;
2117
2118     strcpy(term_dict+prefix_len, term);
2119
2120     grep_info.isam_p_indx = 0;
2121     r = dict_lookup_grep(zh->reg->dict, term_dict, 0,
2122                           &grep_info, &max_pos, 0, grep_handle);
2123     yaz_log(YLOG_DEBUG, "%s %d positions", term,
2124              grep_info.isam_p_indx);
2125     rset = rset_trunc(zh, grep_info.isam_p_buf,
2126                       grep_info.isam_p_indx, term, strlen(term),
2127                       flags, 1, term_type,rset_nmem,
2128                       kc, kc->scope);
2129     grep_info_delete(&grep_info);
2130     return rset;
2131 }
2132
2133 static RSET rpn_search_xpath(ZebraHandle zh,
2134                              oid_value attributeSet,
2135                              int num_bases, char **basenames,
2136                              NMEM stream, const char *rank_type, RSET rset,
2137                              int xpath_len, struct xpath_location_step *xpath,
2138                              NMEM rset_nmem,
2139                              struct rset_key_control *kc)
2140 {
2141     oid_value curAttributeSet = attributeSet;
2142     int base_no;
2143     int i;
2144
2145     if (xpath_len < 0)
2146         return rset;
2147
2148     yaz_log(YLOG_DEBUG, "xpath len=%d", xpath_len);
2149     for (i = 0; i<xpath_len; i++)
2150     {
2151         yaz_log(log_level_rpn, "XPATH %d %s", i, xpath[i].part);
2152
2153     }
2154
2155     curAttributeSet = VAL_IDXPATH;
2156
2157     /*
2158       //a    ->    a/.*
2159       //a/b  ->    b/a/.*
2160       /a     ->    a/
2161       /a/b   ->    b/a/
2162
2163       /      ->    none
2164
2165    a[@attr = value]/b[@other = othervalue]
2166
2167  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
2168  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
2169  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
2170  /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y)
2171  /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y)
2172  /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x)
2173
2174     */
2175
2176     dict_grep_cmap (zh->reg->dict, 0, 0);
2177
2178     for (base_no = 0; base_no < num_bases; base_no++)
2179     {
2180         int level = xpath_len;
2181         int first_path = 1;
2182
2183         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2184         {
2185             zh->errCode = YAZ_BIB1_DATABASE_UNAVAILABLE;
2186             zh->errString = basenames[base_no];
2187             return rset;
2188         }
2189         while (--level >= 0)
2190         {
2191             char xpath_rev[128];
2192             int i, len;
2193             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
2194
2195             *xpath_rev = 0;
2196             len = 0;
2197             for (i = level; i >= 1; --i)
2198             {
2199                 const char *cp = xpath[i].part;
2200                 if (*cp)
2201                 {
2202                     for (;*cp; cp++)
2203                         if (*cp == '*')
2204                         {
2205                             memcpy (xpath_rev + len, "[^/]*", 5);
2206                             len += 5;
2207                         }
2208                         else if (*cp == ' ')
2209                         {
2210
2211                             xpath_rev[len++] = 1;
2212                             xpath_rev[len++] = ' ';
2213                         }
2214
2215                         else
2216                             xpath_rev[len++] = *cp;
2217                     xpath_rev[len++] = '/';
2218                 }
2219                 else if (i == 1)  /* // case */
2220                 {
2221                     xpath_rev[len++] = '.';
2222                     xpath_rev[len++] = '*';
2223                 }
2224             }
2225             xpath_rev[len] = 0;
2226
2227             if (xpath[level].predicate &&
2228                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
2229                 xpath[level].predicate->u.relation.name[0])
2230             {
2231                 WRBUF wbuf = wrbuf_alloc();
2232                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
2233                 if (xpath[level].predicate->u.relation.value)
2234                 {
2235                     const char *cp = xpath[level].predicate->u.relation.value;
2236                     wrbuf_putc(wbuf, '=');
2237
2238                     while (*cp)
2239                     {
2240                         if (strchr(REGEX_CHARS, *cp))
2241                             wrbuf_putc(wbuf, '\\');
2242                         wrbuf_putc(wbuf, *cp);
2243                         cp++;
2244                     }
2245                 }
2246                 wrbuf_puts(wbuf, "");
2247                 rset_attr = xpath_trunc(
2248                     zh, stream, '0', wrbuf_buf(wbuf), 3,
2249                     curAttributeSet, rset_nmem, kc);
2250                 wrbuf_free(wbuf, 1);
2251             }
2252             else
2253             {
2254                 if (!first_path)
2255                     continue;
2256             }
2257             yaz_log(log_level_rpn, "xpath_rev (%d) = %s", level, xpath_rev);
2258             if (strlen(xpath_rev))
2259             {
2260                 rset_start_tag = xpath_trunc(zh, stream, '0',
2261                         xpath_rev, 1, curAttributeSet, rset_nmem, kc);
2262
2263                 rset_end_tag = xpath_trunc(zh, stream, '0',
2264                         xpath_rev, 2, curAttributeSet, rset_nmem, kc);
2265
2266                 rset = rsbetween_create(rset_nmem, kc, kc->scope,
2267                                         rset_start_tag, rset,
2268                                         rset_end_tag, rset_attr);
2269             }
2270             first_path = 0;
2271         }
2272     }
2273
2274     return rset;
2275 }
2276
2277 static ZEBRA_RES rpn_search_APT(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2278                                 oid_value attributeSet, NMEM stream,
2279                                 Z_SortKeySpecList *sort_sequence,
2280                                 int num_bases, char **basenames,
2281                                 NMEM rset_nmem,
2282                                 RSET *rset,
2283                                 struct rset_key_control *kc)
2284 {
2285     ZEBRA_RES res = ZEBRA_OK;
2286     unsigned reg_id;
2287     char *search_type = NULL;
2288     char rank_type[128];
2289     int complete_flag;
2290     int sort_flag;
2291     char termz[IT_MAX_WORD+1];
2292     int xpath_len;
2293     int xpath_use = 0;
2294     struct xpath_location_step xpath[10];
2295
2296     if (!log_level_set)
2297     {
2298         log_level_rpn = yaz_log_module_level("rpn");
2299         log_level_set = 1;
2300     }
2301     zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2302                     rank_type, &complete_flag, &sort_flag);
2303
2304     yaz_log(YLOG_DEBUG, "reg_id=%c", reg_id);
2305     yaz_log(YLOG_DEBUG, "complete_flag=%d", complete_flag);
2306     yaz_log(YLOG_DEBUG, "search_type=%s", search_type);
2307     yaz_log(YLOG_DEBUG, "rank_type=%s", rank_type);
2308
2309     if (zapt_term_to_utf8(zh, zapt, termz) == ZEBRA_FAIL)
2310         return ZEBRA_FAIL;
2311
2312     if (sort_flag)
2313         return rpn_sort_spec(zh, zapt, attributeSet, stream, sort_sequence,
2314                              rank_type, rset, kc);
2315     xpath_len = parse_xpath(zh, zapt, attributeSet, xpath, 10, stream);
2316     if (xpath_len >= 0)
2317     {
2318         xpath_use = 1016;
2319         if (xpath[xpath_len-1].part[0] == '@')
2320             xpath_use = 1015;
2321     }
2322
2323     if (!strcmp(search_type, "phrase"))
2324     {
2325         res = rpn_search_APT_phrase(zh, zapt, termz, attributeSet, stream,
2326                                     reg_id, complete_flag, rank_type,
2327                                     xpath_use,
2328                                     num_bases, basenames, rset_nmem,
2329                                     rset, kc);
2330     }
2331     else if (!strcmp(search_type, "and-list"))
2332     {
2333         res = rpn_search_APT_and_list(zh, zapt, termz, attributeSet, stream,
2334                                       reg_id, complete_flag, rank_type,
2335                                       xpath_use,
2336                                       num_bases, basenames, rset_nmem,
2337                                       rset, kc);
2338     }
2339     else if (!strcmp(search_type, "or-list"))
2340     {
2341         res = rpn_search_APT_or_list(zh, zapt, termz, attributeSet, stream,
2342                                      reg_id, complete_flag, rank_type,
2343                                      xpath_use,
2344                                      num_bases, basenames, rset_nmem,
2345                                      rset, kc);
2346     }
2347     else if (!strcmp(search_type, "local"))
2348     {
2349         res = rpn_search_APT_local(zh, zapt, termz, attributeSet, stream,
2350                                    rank_type, rset_nmem, rset, kc);
2351     }
2352     else if (!strcmp(search_type, "numeric"))
2353     {
2354         res = rpn_search_APT_numeric(zh, zapt, termz, attributeSet, stream,
2355                                      reg_id, complete_flag, rank_type,
2356                                      xpath_use,
2357                                      num_bases, basenames, rset_nmem,
2358                                      rset, kc);
2359     }
2360     else
2361     {
2362         zh->errCode = YAZ_BIB1_UNSUPP_STRUCTURE_ATTRIBUTE;
2363         return ZEBRA_FAIL;
2364     }
2365     if (res != ZEBRA_OK)
2366         return res;
2367     if (!*rset)
2368         return ZEBRA_FAIL;
2369     *rset = rpn_search_xpath(zh, attributeSet, num_bases, basenames,
2370                              stream, rank_type, *rset,
2371                              xpath_len, xpath, rset_nmem, kc);
2372     if (!*rset)
2373         return ZEBRA_FAIL;
2374     return ZEBRA_OK;
2375 }
2376
2377 static ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2378                                       oid_value attributeSet,
2379                                       NMEM stream, NMEM rset_nmem,
2380                                       Z_SortKeySpecList *sort_sequence,
2381                                       int num_bases, char **basenames,
2382                                       RSET **result_sets, int *num_result_sets,
2383                                       Z_Operator *parent_op,
2384                                       struct rset_key_control *kc);
2385
2386 ZEBRA_RES rpn_search_top(ZebraHandle zh, Z_RPNStructure *zs,
2387                          oid_value attributeSet,
2388                          NMEM stream, NMEM rset_nmem,
2389                          Z_SortKeySpecList *sort_sequence,
2390                          int num_bases, char **basenames,
2391                          RSET *result_set)
2392 {
2393     RSET *result_sets = 0;
2394     int num_result_sets = 0;
2395     ZEBRA_RES res;
2396     struct rset_key_control *kc = zebra_key_control_create(zh);
2397
2398     res = rpn_search_structure(zh, zs, attributeSet,
2399                                stream, rset_nmem,
2400                                sort_sequence,
2401                                num_bases, basenames,
2402                                &result_sets, &num_result_sets,
2403                                0 /* no parent op */,
2404                                kc);
2405     if (res != ZEBRA_OK)
2406     {
2407         int i;
2408         for (i = 0; i<num_result_sets; i++)
2409             rset_delete(result_sets[i]);
2410         *result_set = 0;
2411         return res;
2412     }
2413     assert(num_result_sets == 1);
2414     assert(result_sets);
2415     assert(*result_sets);
2416     *result_set = *result_sets;
2417
2418     (*kc->dec)(kc);
2419     return ZEBRA_OK;
2420 }
2421
2422 ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2423                                oid_value attributeSet,
2424                                NMEM stream, NMEM rset_nmem,
2425                                Z_SortKeySpecList *sort_sequence,
2426                                int num_bases, char **basenames,
2427                                RSET **result_sets, int *num_result_sets,
2428                                Z_Operator *parent_op,
2429                                struct rset_key_control *kc)
2430 {
2431     *num_result_sets = 0;
2432     if (zs->which == Z_RPNStructure_complex)
2433     {
2434         ZEBRA_RES res;
2435         Z_Operator *zop = zs->u.complex->roperator;
2436         RSET *result_sets_l = 0;
2437         int num_result_sets_l = 0;
2438         RSET *result_sets_r = 0;
2439         int num_result_sets_r = 0;
2440
2441         res = rpn_search_structure(zh, zs->u.complex->s1,
2442                                    attributeSet, stream, rset_nmem,
2443                                    sort_sequence,
2444                                    num_bases, basenames,
2445                                    &result_sets_l, &num_result_sets_l,
2446                                    zop, kc);
2447         if (res != ZEBRA_OK)
2448         {
2449             int i;
2450             for (i = 0; i<num_result_sets_l; i++)
2451                 rset_delete(result_sets_l[i]);
2452             return res;
2453         }
2454         res = rpn_search_structure(zh, zs->u.complex->s2,
2455                                    attributeSet, stream, rset_nmem,
2456                                    sort_sequence,
2457                                    num_bases, basenames,
2458                                    &result_sets_r, &num_result_sets_r,
2459                                    zop, kc);
2460         if (res != ZEBRA_OK)
2461         {
2462             int i;
2463             for (i = 0; i<num_result_sets_l; i++)
2464                 rset_delete(result_sets_l[i]);
2465             for (i = 0; i<num_result_sets_r; i++)
2466                 rset_delete(result_sets_r[i]);
2467             return res;
2468         }
2469
2470         /* make a new list of result for all children */
2471         *num_result_sets = num_result_sets_l + num_result_sets_r;
2472         *result_sets = nmem_malloc(stream, *num_result_sets *
2473                                    sizeof(**result_sets));
2474         memcpy(*result_sets, result_sets_l,
2475                num_result_sets_l * sizeof(**result_sets));
2476         memcpy(*result_sets + num_result_sets_l, result_sets_r,
2477                num_result_sets_r * sizeof(**result_sets));
2478
2479         if (!parent_op || parent_op->which != zop->which
2480             || (zop->which != Z_Operator_and &&
2481                 zop->which != Z_Operator_or))
2482         {
2483             /* parent node different from this one (or non-present) */
2484             /* we must combine result sets now */
2485             RSET rset;
2486             switch (zop->which)
2487             {
2488             case Z_Operator_and:
2489                 rset = rsmulti_and_create(rset_nmem, kc,
2490                                           kc->scope,
2491                                           *num_result_sets, *result_sets);
2492                 break;
2493             case Z_Operator_or:
2494                 rset = rsmulti_or_create(rset_nmem, kc,
2495                                          kc->scope,
2496                                          *num_result_sets, *result_sets);
2497                 break;
2498             case Z_Operator_and_not:
2499                 rset = rsbool_create_not(rset_nmem, kc,
2500                                          kc->scope,
2501                                          (*result_sets)[0],
2502                                          (*result_sets)[1]);
2503                 break;
2504             case Z_Operator_prox:
2505                 if (zop->u.prox->which != Z_ProximityOperator_known)
2506                 {
2507                     zh->errCode = YAZ_BIB1_UNSUPP_PROX_UNIT_CODE;
2508                     return ZEBRA_FAIL;
2509                 }
2510                 if (*zop->u.prox->u.known != Z_ProxUnit_word)
2511                 {
2512                     char *val = (char *) nmem_malloc(stream, 16);
2513                     zh->errCode = YAZ_BIB1_UNSUPP_PROX_UNIT_CODE;
2514                     zh->errString = val;
2515                     sprintf(val, "%d", *zop->u.prox->u.known);
2516                     return ZEBRA_FAIL;
2517                 }
2518                 else
2519                 {
2520                     rset = rsprox_create(rset_nmem, kc,
2521                                          kc->scope,
2522                                          *num_result_sets, *result_sets,
2523                                          *zop->u.prox->ordered,
2524                                          (!zop->u.prox->exclusion ?
2525                                           0 : *zop->u.prox->exclusion),
2526                                          *zop->u.prox->relationType,
2527                                          *zop->u.prox->distance );
2528                 }
2529                 break;
2530             default:
2531                 zh->errCode = YAZ_BIB1_OPERATOR_UNSUPP;
2532                 return ZEBRA_FAIL;
2533             }
2534             *num_result_sets = 1;
2535             *result_sets = nmem_malloc(stream, *num_result_sets *
2536                                        sizeof(**result_sets));
2537             (*result_sets)[0] = rset;
2538         }
2539     }
2540     else if (zs->which == Z_RPNStructure_simple)
2541     {
2542         RSET rset;
2543         ZEBRA_RES res;
2544
2545         if (zs->u.simple->which == Z_Operand_APT)
2546         {
2547             yaz_log(YLOG_DEBUG, "rpn_search_APT");
2548             res = rpn_search_APT(zh, zs->u.simple->u.attributesPlusTerm,
2549                                  attributeSet, stream, sort_sequence,
2550                                  num_bases, basenames, rset_nmem, &rset,
2551                                  kc);
2552             if (res != ZEBRA_OK)
2553                 return res;
2554         }
2555         else if (zs->u.simple->which == Z_Operand_resultSetId)
2556         {
2557             yaz_log(YLOG_DEBUG, "rpn_search_ref");
2558             rset = resultSetRef(zh, zs->u.simple->u.resultSetId);
2559             if (!rset)
2560             {
2561                 zh->errCode = YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST;
2562                 zh->errString =
2563                     nmem_strdup(stream, zs->u.simple->u.resultSetId);
2564                 return ZEBRA_FAIL;
2565             }
2566             rset_dup(rset);
2567         }
2568         else
2569         {
2570             zh->errCode = YAZ_BIB1_UNSUPP_SEARCH;
2571             return ZEBRA_FAIL;
2572         }
2573         *num_result_sets = 1;
2574         *result_sets = nmem_malloc(stream, *num_result_sets *
2575                                    sizeof(**result_sets));
2576         (*result_sets)[0] = rset;
2577     }
2578     else
2579     {
2580         zh->errCode = YAZ_BIB1_UNSUPP_SEARCH;
2581         return ZEBRA_FAIL;
2582     }
2583     return ZEBRA_OK;
2584 }
2585
2586 struct scan_info_entry {
2587     char *term;
2588     ISAM_P isam_p;
2589 };
2590
2591 struct scan_info {
2592     struct scan_info_entry *list;
2593     ODR odr;
2594     int before, after;
2595     char prefix[20];
2596 };
2597
2598 static int scan_handle (char *name, const char *info, int pos, void *client)
2599 {
2600     int len_prefix, idx;
2601     struct scan_info *scan_info = (struct scan_info *) client;
2602
2603     len_prefix = strlen(scan_info->prefix);
2604     if (memcmp (name, scan_info->prefix, len_prefix))
2605         return 1;
2606     if (pos > 0)
2607         idx = scan_info->after - pos + scan_info->before;
2608     else
2609         idx = - pos - 1;
2610
2611     if (idx < 0)
2612         return 0;
2613     scan_info->list[idx].term = (char *)
2614         odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
2615     strcpy(scan_info->list[idx].term, name + len_prefix);
2616     assert (*info == sizeof(ISAM_P));
2617     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
2618     return 0;
2619 }
2620
2621 static void scan_term_untrans (ZebraHandle zh, NMEM stream, int reg_type,
2622                                char **dst, const char *src)
2623 {
2624     char term_src[IT_MAX_WORD];
2625     char term_dst[IT_MAX_WORD];
2626
2627     term_untrans (zh, reg_type, term_src, src);
2628
2629     if (zh->iconv_from_utf8 != 0)
2630     {
2631         int len;
2632         char *inbuf = term_src;
2633         size_t inleft = strlen(term_src);
2634         char *outbuf = term_dst;
2635         size_t outleft = sizeof(term_dst)-1;
2636         size_t ret;
2637
2638         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2639                          &outbuf, &outleft);
2640         if (ret == (size_t)(-1))
2641             len = 0;
2642         else
2643             len = outbuf - term_dst;
2644         *dst = nmem_malloc(stream, len + 1);
2645         if (len > 0)
2646             memcpy (*dst, term_dst, len);
2647         (*dst)[len] = '\0';
2648     }
2649     else
2650         *dst = nmem_strdup(stream, term_src);
2651 }
2652
2653 static void count_set (RSET r, int *count)
2654 {
2655     zint psysno = 0;
2656     int kno = 0;
2657     struct it_key key;
2658     RSFD rfd;
2659
2660     yaz_log(YLOG_DEBUG, "count_set");
2661
2662     *count = 0;
2663     rfd = rset_open (r, RSETF_READ);
2664     while (rset_read (rfd, &key,0 /* never mind terms */))
2665     {
2666         if (key.mem[0] != psysno)
2667         {
2668             psysno = key.mem[0];
2669             (*count)++;
2670         }
2671         kno++;
2672     }
2673     rset_close (rfd);
2674     yaz_log(YLOG_DEBUG, "%d keys, %d records", kno, *count);
2675 }
2676
2677 ZEBRA_RES rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2678                    oid_value attributeset,
2679                    int num_bases, char **basenames,
2680                    int *position, int *num_entries, ZebraScanEntry **list,
2681                    int *is_partial, RSET limit_set, int return_zero)
2682 {
2683     int i;
2684     int pos = *position;
2685     int num = *num_entries;
2686     int before;
2687     int after;
2688     int base_no;
2689     char termz[IT_MAX_WORD+20];
2690     AttrType use;
2691     int use_value;
2692     const char *use_string = 0;
2693     struct scan_info *scan_info_array;
2694     ZebraScanEntry *glist;
2695     int ords[32], ord_no = 0;
2696     int ptr[32];
2697
2698     int bases_ok = 0;     /* no of databases with OK attribute */
2699     int errCode = 0;      /* err code (if any is not OK) */
2700     char *errString = 0;  /* addinfo */
2701
2702     unsigned reg_id;
2703     char *search_type = NULL;
2704     char rank_type[128];
2705     int complete_flag;
2706     int sort_flag;
2707     NMEM rset_nmem = NULL;
2708     struct rset_key_control *kc = 0;
2709
2710     *list = 0;
2711     *is_partial = 0;
2712
2713     if (attributeset == VAL_NONE)
2714         attributeset = VAL_BIB1;
2715
2716     if (!limit_set)
2717     {
2718         AttrType termset;
2719         int termset_value_numeric;
2720         const char *termset_value_string;
2721         attr_init(&termset, zapt, 8);
2722         termset_value_numeric =
2723             attr_find_ex(&termset, NULL, &termset_value_string);
2724         if (termset_value_numeric != -1)
2725         {
2726             char resname[32];
2727             const char *termset_name = 0;
2728
2729             if (termset_value_numeric != -2)
2730             {
2731
2732                 sprintf(resname, "%d", termset_value_numeric);
2733                 termset_name = resname;
2734             }
2735             else
2736                 termset_name = termset_value_string;
2737
2738             limit_set = resultSetRef (zh, termset_name);
2739         }
2740     }
2741
2742     yaz_log(YLOG_DEBUG, "position = %d, num = %d set=%d",
2743             pos, num, attributeset);
2744
2745     attr_init(&use, zapt, 1);
2746     use_value = attr_find_ex(&use, &attributeset, &use_string);
2747
2748     if (zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2749                         rank_type, &complete_flag, &sort_flag))
2750     {
2751         *num_entries = 0;
2752         zh->errCode = YAZ_BIB1_UNSUPP_ATTRIBUTE_TYPE;
2753         return ZEBRA_FAIL;
2754     }
2755     yaz_log(YLOG_DEBUG, "use_value = %d", use_value);
2756
2757     if (use_value == -1)
2758         use_value = 1016;
2759     for (base_no = 0; base_no < num_bases && ord_no < 32; base_no++)
2760     {
2761         data1_local_attribute *local_attr;
2762         attent attp;
2763         int ord;
2764
2765         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2766         {
2767             zh->errString = basenames[base_no];
2768             zh->errCode = YAZ_BIB1_DATABASE_UNAVAILABLE;
2769             *num_entries = 0;
2770             return ZEBRA_FAIL;
2771         }
2772
2773         if (use_string &&
2774             (ord = zebraExplain_lookup_attr_str(zh->reg->zei,
2775                                                 use_string)) >= 0)
2776         {
2777             /* we have a match for a raw string attribute */
2778             if (ord > 0)
2779                 ords[ord_no++] = ord;
2780             attp.local_attributes = 0;  /* no more attributes */
2781         }
2782         else
2783         {
2784             int r;
2785
2786             if ((r = att_getentbyatt (zh, &attp, attributeset, use_value,
2787                                       use_string)))
2788             {
2789                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
2790                         attributeset, use_value);
2791                 if (r == -1)
2792                 {
2793                     errCode = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
2794                     if (use_string)
2795                         errString = odr_strdup(stream, use_string);
2796                     else
2797                     {
2798                         char val_str[32];
2799                         sprintf(val_str, "%d", use_value);
2800                         errString = odr_strdup(stream, val_str);
2801                     }
2802                 }
2803                 else
2804                     errCode = YAZ_BIB1_UNSUPP_ATTRIBUTE_SET;
2805                 continue;
2806             }
2807         }
2808         bases_ok++;
2809         for (local_attr = attp.local_attributes; local_attr && ord_no < 32;
2810              local_attr = local_attr->next)
2811         {
2812             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
2813                                               attp.attset_ordinal,
2814                                               local_attr->local);
2815             if (ord > 0)
2816                 ords[ord_no++] = ord;
2817         }
2818     }
2819     if (!bases_ok && errCode)
2820     {
2821         zh->errCode = errCode;
2822         zh->errString = errString;
2823         *num_entries = 0;
2824         return ZEBRA_FAIL;
2825     }
2826     if (ord_no == 0)
2827     {
2828         *num_entries = 0;
2829         return ZEBRA_OK;
2830     }
2831     /* prepare dictionary scanning */
2832     if (num < 1)
2833     {
2834         *num_entries = 0;
2835         return ZEBRA_OK;
2836     }
2837     before = pos-1;
2838     if (before < 0)
2839         before = 0;
2840     after = 1+num-pos;
2841     if (after < 0)
2842         after = 0;
2843     yaz_log(YLOG_DEBUG, "rpn_scan pos=%d num=%d before=%d "
2844             "after=%d before+after=%d",
2845             pos, num, before, after, before+after);
2846     scan_info_array = (struct scan_info *)
2847         odr_malloc(stream, ord_no * sizeof(*scan_info_array));
2848     for (i = 0; i < ord_no; i++)
2849     {
2850         int j, prefix_len = 0;
2851         int before_tmp = before, after_tmp = after;
2852         struct scan_info *scan_info = scan_info_array + i;
2853         struct rpn_char_map_info rcmi;
2854
2855         rpn_char_map_prepare (zh->reg, reg_id, &rcmi);
2856
2857         scan_info->before = before;
2858         scan_info->after = after;
2859         scan_info->odr = stream;
2860
2861         scan_info->list = (struct scan_info_entry *)
2862             odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
2863         for (j = 0; j<before+after; j++)
2864             scan_info->list[j].term = NULL;
2865
2866         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2867         termz[prefix_len++] = reg_id;
2868         termz[prefix_len] = 0;
2869         strcpy(scan_info->prefix, termz);
2870
2871         if (trans_scan_term(zh, zapt, termz+prefix_len, reg_id) == ZEBRA_FAIL)
2872             return ZEBRA_FAIL;
2873
2874         dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
2875                   scan_info, scan_handle);
2876     }
2877     glist = (ZebraScanEntry *)
2878         odr_malloc(stream, (before+after)*sizeof(*glist));
2879
2880     rset_nmem = nmem_create();
2881     kc = zebra_key_control_create(zh);
2882
2883     /* consider terms after main term */
2884     for (i = 0; i < ord_no; i++)
2885         ptr[i] = before;
2886
2887     *is_partial = 0;
2888     for (i = 0; i<after; i++)
2889     {
2890         int j, j0 = -1;
2891         const char *mterm = NULL;
2892         const char *tst;
2893         RSET rset = 0;
2894         int lo = i + pos-1; /* offset in result list */
2895
2896         /* find: j0 is the first of the minimal values */
2897         for (j = 0; j < ord_no; j++)
2898         {
2899             if (ptr[j] < before+after && ptr[j] >= 0 &&
2900                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2901                 (!mterm || strcmp (tst, mterm) < 0))
2902             {
2903                 j0 = j;
2904                 mterm = tst;
2905             }
2906         }
2907         if (j0 == -1)
2908             break;  /* no value found, stop */
2909
2910         /* get result set for first one , but only if it's within bounds */
2911         if (lo >= 0)
2912         {
2913             /* get result set for first term */
2914             scan_term_untrans(zh, stream->mem, reg_id,
2915                               &glist[lo].term, mterm);
2916             rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2917                               glist[lo].term, strlen(glist[lo].term),
2918                               NULL, 0, zapt->term->which, rset_nmem,
2919                               kc, kc->scope);
2920         }
2921         ptr[j0]++; /* move index for this set .. */
2922         /* get result set for remaining scan terms */
2923         for (j = j0+1; j<ord_no; j++)
2924         {
2925             if (ptr[j] < before+after && ptr[j] >= 0 &&
2926                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2927                 !strcmp (tst, mterm))
2928             {
2929                 if (lo >= 0)
2930                 {
2931                     RSET rsets[2];
2932
2933                     rsets[0] = rset;
2934                     rsets[1] =
2935                         rset_trunc(
2936                             zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
2937                             glist[lo].term,
2938                             strlen(glist[lo].term), NULL, 0,
2939                             zapt->term->which,rset_nmem,
2940                             kc, kc->scope);
2941                     rset = rsmulti_or_create(rset_nmem, kc,
2942                                              2, kc->scope, rsets);
2943                 }
2944                 ptr[j]++;
2945             }
2946         }
2947         if (lo >= 0)
2948         {
2949             /* merge with limit_set if given */
2950             if (limit_set)
2951             {
2952                 RSET rsets[2];
2953                 rsets[0] = rset;
2954                 rsets[1] = rset_dup(limit_set);
2955
2956                 rset = rsmulti_and_create(rset_nmem, kc,
2957                                           kc->scope, 2, rsets);
2958             }
2959             /* count it */
2960             count_set(rset, &glist[lo].occurrences);
2961             rset_delete(rset);
2962         }
2963     }
2964     if (i < after)
2965     {
2966         *num_entries -= (after-i);
2967         *is_partial = 1;
2968         if (*num_entries < 0)
2969         {
2970             (*kc->dec)(kc);
2971             nmem_destroy(rset_nmem);
2972             *num_entries = 0;
2973             return ZEBRA_OK;
2974         }
2975     }
2976     /* consider terms before main term */
2977     for (i = 0; i<ord_no; i++)
2978         ptr[i] = 0;
2979
2980     for (i = 0; i<before; i++)
2981     {
2982         int j, j0 = -1;
2983         const char *mterm = NULL;
2984         const char *tst;
2985         RSET rset;
2986         int lo = before-1-i; /* offset in result list */
2987
2988         for (j = 0; j <ord_no; j++)
2989         {
2990             if (ptr[j] < before && ptr[j] >= 0 &&
2991                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2992                 (!mterm || strcmp (tst, mterm) > 0))
2993             {
2994                 j0 = j;
2995                     mterm = tst;
2996             }
2997         }
2998         if (j0 == -1)
2999             break;
3000
3001         scan_term_untrans (zh, stream->mem, reg_id,
3002                            &glist[lo].term, mterm);
3003
3004         rset = rset_trunc
3005             (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
3006              glist[lo].term, strlen(glist[lo].term),
3007              NULL, 0, zapt->term->which,rset_nmem,
3008              kc, kc->scope);
3009
3010         ptr[j0]++;
3011
3012         for (j = j0+1; j<ord_no; j++)
3013         {
3014             if (ptr[j] < before && ptr[j] >= 0 &&
3015                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
3016                 !strcmp (tst, mterm))
3017             {
3018                 RSET rsets[2];
3019
3020                 rsets[0] = rset;
3021                 rsets[1] = rset_trunc(
3022                     zh,
3023                     &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
3024                     glist[lo].term,
3025                     strlen(glist[lo].term), NULL, 0,
3026                     zapt->term->which, rset_nmem,
3027                     kc, kc->scope);
3028                 rset = rsmulti_or_create(rset_nmem, kc,
3029                                          2, kc->scope, rsets);
3030
3031                 ptr[j]++;
3032             }
3033         }
3034         if (limit_set)
3035         {
3036             RSET rsets[2];
3037             rsets[0] = rset;
3038             rsets[1] = rset_dup(limit_set);
3039
3040             rset = rsmulti_and_create(rset_nmem, kc,
3041                                       kc->scope, 2, rsets);
3042         }
3043         count_set (rset, &glist[lo].occurrences);
3044         rset_delete (rset);
3045     }
3046     (*kc->dec)(kc);
3047     nmem_destroy(rset_nmem);
3048     i = before-i;
3049     if (i)
3050     {
3051         *is_partial = 1;
3052         *position -= i;
3053         *num_entries -= i;
3054         if (*num_entries <= 0)
3055         {
3056             *num_entries = 0;
3057             return ZEBRA_OK;
3058         }
3059     }
3060
3061     *list = glist + i;               /* list is set to first 'real' entry */
3062
3063     yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
3064             *position, *num_entries);
3065     return ZEBRA_OK;
3066 }
3067