index/zrpn.c

   1 /* $Id: zrpn.c,v 1.189 2005-05-04 10:48:39 adam Exp $
   2    Copyright (C) 1995-2005
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with Zebra; see the file LICENSE.zebra.  If not, write to the
  19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  20 02111-1307, USA.
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #ifdef WIN32
  26 #include <io.h>
  27 #else
  28 #include <unistd.h>
  29 #endif
  30 #include <ctype.h>
  31
  32 #include <yaz/diagbib1.h>
  33 #include "index.h"
  34 #include <zebra_xpath.h>
  35
  36 #include <charmap.h>
  37 #include <rset.h>
  38
  39 struct rpn_char_map_info
  40 {
  41     ZebraMaps zm;
  42     int reg_type;
  43 };
  44
  45 typedef struct
  46 {
  47     int type;
  48     int major;
  49     int minor;
  50     Z_AttributesPlusTerm *zapt;
  51 } AttrType;
  52
  53
  54 static int log_level_set = 0;
  55 static int log_level_rpn = 0;
  56
  57 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  58 {
  59     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  60     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  61 #if 0
  62     if (out && *out)
  63     {
  64         const char *outp = *out;
  65         yaz_log(YLOG_LOG, "---");
  66         while (*outp)
  67         {
  68             yaz_log(YLOG_LOG, "%02X", *outp);
  69             outp++;
  70         }
  71     }
  72 #endif
  73     return out;
  74 }
  75
  76 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  77                                   struct rpn_char_map_info *map_info)
  78 {
  79     map_info->zm = reg->zebra_maps;
  80     map_info->reg_type = reg_type;
  81     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  82 }
  83
  84 static int attr_find_ex(AttrType *src, oid_value *attributeSetP,
  85                          const char **string_value)
  86 {
  87     int num_attributes;
  88
  89     num_attributes = src->zapt->attributes->num_attributes;
  90     while (src->major < num_attributes)
  91     {
  92         Z_AttributeElement *element;
  93
  94         element = src->zapt->attributes->attributes[src->major];
  95         if (src->type == *element->attributeType)
  96         {
  97             switch (element->which)
  98             {
  99             case Z_AttributeValue_numeric:
 100                 ++(src->major);
 101                 if (element->attributeSet && attributeSetP)
 102                 {
 103                     oident *attrset;
 104
 105                     attrset = oid_getentbyoid(element->attributeSet);
 106                     *attributeSetP = attrset->value;
 107                 }
 108                 return *element->value.numeric;
 109                 break;
 110             case Z_AttributeValue_complex:
 111                 if (src->minor >= element->value.complex->num_list)
 112                     break;
 113                 if (element->attributeSet && attributeSetP)
 114                 {
 115                     oident *attrset;
 116
 117                     attrset = oid_getentbyoid(element->attributeSet);
 118                     *attributeSetP = attrset->value;
 119                 }
 120                 if (element->value.complex->list[src->minor]->which ==
 121                     Z_StringOrNumeric_numeric)
 122                 {
 123                     ++(src->minor);
 124                     return
 125                         *element->value.complex->list[src->minor-1]->u.numeric;
 126                 }
 127                 else if (element->value.complex->list[src->minor]->which ==
 128                          Z_StringOrNumeric_string)
 129                 {
 130                     if (!string_value)
 131                         break;
 132                     ++(src->minor);
 133                     *string_value =
 134                         element->value.complex->list[src->minor-1]->u.string;
 135                     return -2;
 136                 }
 137                 else
 138                     break;
 139             default:
 140                 assert(0);
 141             }
 142         }
 143         ++(src->major);
 144     }
 145     return -1;
 146 }
 147
 148 static int attr_find(AttrType *src, oid_value *attributeSetP)
 149 {
 150     return attr_find_ex(src, attributeSetP, 0);
 151 }
 152
 153 static void attr_init(AttrType *src, Z_AttributesPlusTerm *zapt,
 154                        int type)
 155 {
 156     src->zapt = zapt;
 157     src->type = type;
 158     src->major = 0;
 159     src->minor = 0;
 160 }
 161
 162 #define TERM_COUNT
 163
 164 struct grep_info {
 165 #ifdef TERM_COUNT
 166     int *term_no;
 167 #endif
 168     ISAM_P *isam_p_buf;
 169     int isam_p_size;
 170     int isam_p_indx;
 171     ZebraHandle zh;
 172     int reg_type;
 173     ZebraSet termset;
 174 };
 175
 176 static void term_untrans(ZebraHandle zh, int reg_type,
 177                            char *dst, const char *src)
 178 {
 179     int len = 0;
 180     while (*src)
 181     {
 182         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
 183                                            reg_type, &src);
 184         if (!cp && len < IT_MAX_WORD-1)
 185             dst[len++] = *src++;
 186         else
 187             while (*cp && len < IT_MAX_WORD-1)
 188                 dst[len++] = *cp++;
 189     }
 190     dst[len] = '\0';
 191 }
 192
 193 static void add_isam_p(const char *name, const char *info,
 194                        struct grep_info *p)
 195 {
 196     if (!log_level_set)
 197     {
 198         log_level_rpn = yaz_log_module_level("rpn");
 199         log_level_set = 1;
 200     }
 201     if (p->isam_p_indx == p->isam_p_size)
 202     {
 203         ISAM_P *new_isam_p_buf;
 204 #ifdef TERM_COUNT
 205         int *new_term_no;
 206 #endif
 207         p->isam_p_size = 2*p->isam_p_size + 100;
 208         new_isam_p_buf = (ISAM_P *) xmalloc(sizeof(*new_isam_p_buf) *
 209                                             p->isam_p_size);
 210         if (p->isam_p_buf)
 211         {
 212             memcpy(new_isam_p_buf, p->isam_p_buf,
 213                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 214             xfree(p->isam_p_buf);
 215         }
 216         p->isam_p_buf = new_isam_p_buf;
 217
 218 #ifdef TERM_COUNT
 219         new_term_no = (int *) xmalloc(sizeof(*new_term_no) * p->isam_p_size);
 220         if (p->term_no)
 221         {
 222             memcpy(new_term_no, p->isam_p_buf,
 223                     p->isam_p_indx * sizeof(*p->term_no));
 224             xfree(p->term_no);
 225         }
 226         p->term_no = new_term_no;
 227 #endif
 228     }
 229     assert(*info == sizeof(*p->isam_p_buf));
 230     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 231
 232 #if 1
 233     if (p->termset)
 234     {
 235         const char *db;
 236         int set, use;
 237         char term_tmp[IT_MAX_WORD];
 238         int su_code = 0;
 239         int len = key_SU_decode (&su_code, name);
 240
 241         term_untrans  (p->zh, p->reg_type, term_tmp, name+len+1);
 242         yaz_log(log_level_rpn, "grep: %d %c %s", su_code, name[len], term_tmp);
 243         zebraExplain_lookup_ord (p->zh->reg->zei,
 244                                  su_code, &db, &set, &use);
 245         yaz_log(log_level_rpn, "grep:  set=%d use=%d db=%s", set, use, db);
 246
 247         resultSetAddTerm(p->zh, p->termset, name[len], db,
 248                          set, use, term_tmp);
 249     }
 250 #endif
 251     (p->isam_p_indx)++;
 252 }
 253
 254 static int grep_handle(char *name, const char *info, void *p)
 255 {
 256     add_isam_p(name, info, (struct grep_info *) p);
 257     return 0;
 258 }
 259
 260 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 261                     const char *ct1, const char *ct2, int first)
 262 {
 263     const char *s1, *s0 = *src;
 264     const char **map;
 265
 266     /* skip white space */
 267     while (*s0)
 268     {
 269         if (ct1 && strchr(ct1, *s0))
 270             break;
 271         if (ct2 && strchr(ct2, *s0))
 272             break;
 273         s1 = s0;
 274         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 275         if (**map != *CHR_SPACE)
 276             break;
 277         s0 = s1;
 278     }
 279     *src = s0;
 280     return *s0;
 281 }
 282
 283
 284 static void esc_str(char *out_buf, int out_size,
 285                     const char *in_buf, int in_size)
 286 {
 287     int k;
 288
 289     assert(out_buf);
 290     assert(in_buf);
 291     assert(out_size > 20);
 292     *out_buf = '\0';
 293     for (k = 0; k<in_size; k++)
 294     {
 295         int c = in_buf[k] & 0xff;
 296         int pc;
 297         if (c < 32 || c > 126)
 298             pc = '?';
 299         else
 300             pc = c;
 301         sprintf(out_buf +strlen(out_buf), "%02X:%c  ", c, pc);
 302         if (strlen(out_buf) > out_size-20)
 303         {
 304             strcat(out_buf, "..");
 305             break;
 306         }
 307     }
 308 }
 309
 310 #define REGEX_CHARS " []()|.*+?!"
 311
 312 /* term_100: handle term, where trunc = none(no operators at all) */
 313 static int term_100(ZebraMaps zebra_maps, int reg_type,
 314                     const char **src, char *dst, int space_split,
 315                     char *dst_term)
 316 {
 317     const char *s0;
 318     const char **map;
 319     int i = 0;
 320     int j = 0;
 321
 322     const char *space_start = 0;
 323     const char *space_end = 0;
 324
 325     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 326         return 0;
 327     s0 = *src;
 328     while (*s0)
 329     {
 330         const char *s1 = s0;
 331         int q_map_match = 0;
 332         map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 333                                 &q_map_match);
 334         if (space_split)
 335         {
 336             if (**map == *CHR_SPACE)
 337                 break;
 338         }
 339         else  /* complete subfield only. */
 340         {
 341             if (**map == *CHR_SPACE)
 342             {   /* save space mapping for later  .. */
 343                 space_start = s1;
 344                 space_end = s0;
 345                 continue;
 346             }
 347             else if (space_start)
 348             {   /* reload last space */
 349                 while (space_start < space_end)
 350                 {
 351                     if (strchr(REGEX_CHARS, *space_start))
 352                         dst[i++] = '\\';
 353                     dst_term[j++] = *space_start;
 354                     dst[i++] = *space_start++;
 355                 }
 356                 /* and reset */
 357                 space_start = space_end = 0;
 358             }
 359         }
 360         /* add non-space char */
 361         memcpy(dst_term+j, s1, s0 - s1);
 362         j += (s0 - s1);
 363         if (!q_map_match)
 364         {
 365             while (s1 < s0)
 366             {
 367                 if (strchr(REGEX_CHARS, *s1))
 368                     dst[i++] = '\\';
 369                 dst[i++] = *s1++;
 370             }
 371         }
 372         else
 373         {
 374             char tmpbuf[80];
 375             esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 376
 377             strcpy(dst + i, map[0]);
 378             i += strlen(map[0]);
 379         }
 380     }
 381     dst[i] = '\0';
 382     dst_term[j] = '\0';
 383     *src = s0;
 384     return i;
 385 }
 386
 387 /* term_101: handle term, where trunc = Process # */
 388 static int term_101(ZebraMaps zebra_maps, int reg_type,
 389                     const char **src, char *dst, int space_split,
 390                     char *dst_term)
 391 {
 392     const char *s0;
 393     const char **map;
 394     int i = 0;
 395     int j = 0;
 396
 397     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 398         return 0;
 399     s0 = *src;
 400     while (*s0)
 401     {
 402         if (*s0 == '#')
 403         {
 404             dst[i++] = '.';
 405             dst[i++] = '*';
 406             dst_term[j++] = *s0++;
 407         }
 408         else
 409         {
 410             const char *s1 = s0;
 411             int q_map_match = 0;
 412             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 413                                     &q_map_match);
 414             if (space_split && **map == *CHR_SPACE)
 415                 break;
 416
 417             /* add non-space char */
 418             memcpy(dst_term+j, s1, s0 - s1);
 419             j += (s0 - s1);
 420             if (!q_map_match)
 421             {
 422                 while (s1 < s0)
 423                 {
 424                     if (strchr(REGEX_CHARS, *s1))
 425                         dst[i++] = '\\';
 426                     dst[i++] = *s1++;
 427                 }
 428             }
 429             else
 430             {
 431                 char tmpbuf[80];
 432                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 433
 434                 strcpy(dst + i, map[0]);
 435                 i += strlen(map[0]);
 436             }
 437         }
 438     }
 439     dst[i] = '\0';
 440     dst_term[j++] = '\0';
 441     *src = s0;
 442     return i;
 443 }
 444
 445 /* term_103: handle term, where trunc = re-2 (regular expressions) */
 446 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 447                     char *dst, int *errors, int space_split,
 448                     char *dst_term)
 449 {
 450     int i = 0;
 451     int j = 0;
 452     const char *s0;
 453     const char **map;
 454
 455     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 456         return 0;
 457     s0 = *src;
 458     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 459         isdigit(((const unsigned char *)s0)[1]))
 460     {
 461         *errors = s0[1] - '0';
 462         s0 += 3;
 463         if (*errors > 3)
 464             *errors = 3;
 465     }
 466     while (*s0)
 467     {
 468         if (strchr("^\\()[].*+?|-", *s0))
 469         {
 470             dst_term[j++] = *s0;
 471             dst[i++] = *s0++;
 472         }
 473         else
 474         {
 475             const char *s1 = s0;
 476             int q_map_match = 0;
 477             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 478                                     &q_map_match);
 479             if (space_split && **map == *CHR_SPACE)
 480                 break;
 481
 482             /* add non-space char */
 483             memcpy(dst_term+j, s1, s0 - s1);
 484             j += (s0 - s1);
 485             if (!q_map_match)
 486             {
 487                 while (s1 < s0)
 488                 {
 489                     if (strchr(REGEX_CHARS, *s1))
 490                         dst[i++] = '\\';
 491                     dst[i++] = *s1++;
 492                 }
 493             }
 494             else
 495             {
 496                 char tmpbuf[80];
 497                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 498
 499                 strcpy(dst + i, map[0]);
 500                 i += strlen(map[0]);
 501             }
 502         }
 503     }
 504     dst[i] = '\0';
 505     dst_term[j] = '\0';
 506     *src = s0;
 507
 508     return i;
 509 }
 510
 511 /* term_103: handle term, where trunc = re-1 (regular expressions) */
 512 static int term_102(ZebraMaps zebra_maps, int reg_type, const char **src,
 513                     char *dst, int space_split, char *dst_term)
 514 {
 515     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 516                     dst_term);
 517 }
 518
 519
 520 /* term_104: handle term, where trunc = Process # and ! */
 521 static int term_104(ZebraMaps zebra_maps, int reg_type,
 522                     const char **src, char *dst, int space_split,
 523                     char *dst_term)
 524 {
 525     const char *s0;
 526     const char **map;
 527     int i = 0;
 528     int j = 0;
 529
 530     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 531         return 0;
 532     s0 = *src;
 533     while (*s0)
 534     {
 535         if (*s0 == '?')
 536         {
 537             dst_term[j++] = *s0++;
 538             if (*s0 >= '0' && *s0 <= '9')
 539             {
 540                 int limit = 0;
 541                 while (*s0 >= '0' && *s0 <= '9')
 542                 {
 543                     limit = limit * 10 + (*s0 - '0');
 544                     dst_term[j++] = *s0++;
 545                 }
 546                 if (limit > 20)
 547                     limit = 20;
 548                 while (--limit >= 0)
 549                 {
 550                     dst[i++] = '.';
 551                     dst[i++] = '?';
 552                 }
 553             }
 554             else
 555             {
 556                 dst[i++] = '.';
 557                 dst[i++] = '*';
 558             }
 559         }
 560         else if (*s0 == '*')
 561         {
 562             dst[i++] = '.';
 563             dst[i++] = '*';
 564             dst_term[j++] = *s0++;
 565         }
 566         else if (*s0 == '#')
 567         {
 568             dst[i++] = '.';
 569             dst_term[j++] = *s0++;
 570         }
 571         else
 572         {
 573             const char *s1 = s0;
 574             int q_map_match = 0;
 575             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 576                                     &q_map_match);
 577             if (space_split && **map == *CHR_SPACE)
 578                 break;
 579
 580             /* add non-space char */
 581             memcpy(dst_term+j, s1, s0 - s1);
 582             j += (s0 - s1);
 583             if (!q_map_match)
 584             {
 585                 while (s1 < s0)
 586                 {
 587                     if (strchr(REGEX_CHARS, *s1))
 588                         dst[i++] = '\\';
 589                     dst[i++] = *s1++;
 590                 }
 591             }
 592             else
 593             {
 594                 char tmpbuf[80];
 595                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 596
 597                 strcpy(dst + i, map[0]);
 598                 i += strlen(map[0]);
 599             }
 600         }
 601     }
 602     dst[i] = '\0';
 603     dst_term[j++] = '\0';
 604     *src = s0;
 605     return i;
 606 }
 607
 608 /* term_105/106: handle term, where trunc = Process * and ! and right trunc */
 609 static int term_105(ZebraMaps zebra_maps, int reg_type,
 610                     const char **src, char *dst, int space_split,
 611                     char *dst_term, int right_truncate)
 612 {
 613     const char *s0;
 614     const char **map;
 615     int i = 0;
 616     int j = 0;
 617
 618     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 619         return 0;
 620     s0 = *src;
 621     while (*s0)
 622     {
 623         if (*s0 == '*')
 624         {
 625             dst[i++] = '.';
 626             dst[i++] = '*';
 627             dst_term[j++] = *s0++;
 628         }
 629         else if (*s0 == '!')
 630         {
 631             dst[i++] = '.';
 632             dst_term[j++] = *s0++;
 633         }
 634         else
 635         {
 636             const char *s1 = s0;
 637             int q_map_match = 0;
 638             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 639                                     &q_map_match);
 640             if (space_split && **map == *CHR_SPACE)
 641                 break;
 642
 643             /* add non-space char */
 644             memcpy(dst_term+j, s1, s0 - s1);
 645             j += (s0 - s1);
 646             if (!q_map_match)
 647             {
 648                 while (s1 < s0)
 649                 {
 650                     if (strchr(REGEX_CHARS, *s1))
 651                         dst[i++] = '\\';
 652                     dst[i++] = *s1++;
 653                 }
 654             }
 655             else
 656             {
 657                 char tmpbuf[80];
 658                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 659
 660                 strcpy(dst + i, map[0]);
 661                 i += strlen(map[0]);
 662             }
 663         }
 664     }
 665     if (right_truncate)
 666     {
 667         dst[i++] = '.';
 668         dst[i++] = '*';
 669     }
 670     dst[i] = '\0';
 671
 672     dst_term[j++] = '\0';
 673     *src = s0;
 674     return i;
 675 }
 676
 677
 678 /* gen_regular_rel - generate regular expression from relation
 679  *  val:     border value (inclusive)
 680  *  islt:    1 if <=; 0 if >=.
 681  */
 682 static void gen_regular_rel(char *dst, int val, int islt)
 683 {
 684     int dst_p;
 685     int w, d, i;
 686     int pos = 0;
 687     char numstr[20];
 688
 689     yaz_log(YLOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 690     if (val >= 0)
 691     {
 692         if (islt)
 693             strcpy(dst, "(-[0-9]+|(");
 694         else
 695             strcpy(dst, "((");
 696     }
 697     else
 698     {
 699         if (!islt)
 700         {
 701             strcpy(dst, "([0-9]+|-(");
 702             dst_p = strlen(dst);
 703             islt = 1;
 704         }
 705         else
 706         {
 707             strcpy(dst, "(-(");
 708             islt = 0;
 709         }
 710         val = -val;
 711     }
 712     dst_p = strlen(dst);
 713     sprintf(numstr, "%d", val);
 714     for (w = strlen(numstr); --w >= 0; pos++)
 715     {
 716         d = numstr[w];
 717         if (pos > 0)
 718         {
 719             if (islt)
 720             {
 721                 if (d == '0')
 722                     continue;
 723                 d--;
 724             }
 725             else
 726             {
 727                 if (d == '9')
 728                     continue;
 729                 d++;
 730             }
 731         }
 732
 733         strcpy(dst + dst_p, numstr);
 734         dst_p = strlen(dst) - pos - 1;
 735
 736         if (islt)
 737         {
 738             if (d != '0')
 739             {
 740                 dst[dst_p++] = '[';
 741                 dst[dst_p++] = '0';
 742                 dst[dst_p++] = '-';
 743                 dst[dst_p++] = d;
 744                 dst[dst_p++] = ']';
 745             }
 746             else
 747                 dst[dst_p++] = d;
 748         }
 749         else
 750         {
 751             if (d != '9')
 752             {
 753                 dst[dst_p++] = '[';
 754                 dst[dst_p++] = d;
 755                 dst[dst_p++] = '-';
 756                 dst[dst_p++] = '9';
 757                 dst[dst_p++] = ']';
 758             }
 759             else
 760                 dst[dst_p++] = d;
 761         }
 762         for (i = 0; i<pos; i++)
 763         {
 764             dst[dst_p++] = '[';
 765             dst[dst_p++] = '0';
 766             dst[dst_p++] = '-';
 767             dst[dst_p++] = '9';
 768             dst[dst_p++] = ']';
 769         }
 770         dst[dst_p++] = '|';
 771     }
 772     dst[dst_p] = '\0';
 773     if (islt)
 774     {
 775         /* match everything less than 10^(pos-1) */
 776         strcat(dst, "0*");
 777         for (i = 1; i<pos; i++)
 778             strcat(dst, "[0-9]?");
 779     }
 780     else
 781     {
 782         /* match everything greater than 10^pos */
 783         for (i = 0; i <= pos; i++)
 784             strcat(dst, "[0-9]");
 785         strcat(dst, "[0-9]*");
 786     }
 787     strcat(dst, "))");
 788 }
 789
 790 void string_rel_add_char(char **term_p, const char *src, int *indx)
 791 {
 792     if (src[*indx] == '\\')
 793         *(*term_p)++ = src[(*indx)++];
 794     *(*term_p)++ = src[(*indx)++];
 795 }
 796
 797 /*
 798  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 799  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 800  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 801  *              ([^-a].*|a[^-b].*|ab[c-].*)
 802  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 803  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 804  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 805  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 806  */
 807 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 808                            const char **term_sub, char *term_dict,
 809                            oid_value attributeSet,
 810                            int reg_type, int space_split, char *term_dst,
 811                            int *error_code)
 812 {
 813     AttrType relation;
 814     int relation_value;
 815     int i;
 816     char *term_tmp = term_dict + strlen(term_dict);
 817     char term_component[2*IT_MAX_WORD+20];
 818
 819     attr_init(&relation, zapt, 2);
 820     relation_value = attr_find(&relation, NULL);
 821
 822     *error_code = 0;
 823     yaz_log(YLOG_DEBUG, "string relation value=%d", relation_value);
 824     switch (relation_value)
 825     {
 826     case 1:
 827         if (!term_100(zh->reg->zebra_maps, reg_type,
 828                       term_sub, term_component,
 829                       space_split, term_dst))
 830             return 0;
 831         yaz_log(log_level_rpn, "Relation <");
 832
 833         *term_tmp++ = '(';
 834         for (i = 0; term_component[i]; )
 835         {
 836             int j = 0;
 837
 838             if (i)
 839                 *term_tmp++ = '|';
 840             while (j < i)
 841                 string_rel_add_char(&term_tmp, term_component, &j);
 842
 843             *term_tmp++ = '[';
 844
 845             *term_tmp++ = '^';
 846             string_rel_add_char(&term_tmp, term_component, &i);
 847             *term_tmp++ = '-';
 848
 849             *term_tmp++ = ']';
 850             *term_tmp++ = '.';
 851             *term_tmp++ = '*';
 852
 853             if ((term_tmp - term_dict) > IT_MAX_WORD)
 854                 break;
 855         }
 856         *term_tmp++ = ')';
 857         *term_tmp = '\0';
 858         break;
 859     case 2:
 860         if (!term_100(zh->reg->zebra_maps, reg_type,
 861                       term_sub, term_component,
 862                       space_split, term_dst))
 863             return 0;
 864         yaz_log(log_level_rpn, "Relation <=");
 865
 866         *term_tmp++ = '(';
 867         for (i = 0; term_component[i]; )
 868         {
 869             int j = 0;
 870
 871             while (j < i)
 872                 string_rel_add_char(&term_tmp, term_component, &j);
 873             *term_tmp++ = '[';
 874
 875             *term_tmp++ = '^';
 876             string_rel_add_char(&term_tmp, term_component, &i);
 877             *term_tmp++ = '-';
 878
 879             *term_tmp++ = ']';
 880             *term_tmp++ = '.';
 881             *term_tmp++ = '*';
 882
 883             *term_tmp++ = '|';
 884
 885             if ((term_tmp - term_dict) > IT_MAX_WORD)
 886                 break;
 887         }
 888         for (i = 0; term_component[i]; )
 889             string_rel_add_char(&term_tmp, term_component, &i);
 890         *term_tmp++ = ')';
 891         *term_tmp = '\0';
 892         break;
 893     case 5:
 894         if (!term_100 (zh->reg->zebra_maps, reg_type,
 895                        term_sub, term_component, space_split, term_dst))
 896             return 0;
 897         yaz_log(log_level_rpn, "Relation >");
 898
 899         *term_tmp++ = '(';
 900         for (i = 0; term_component[i];)
 901         {
 902             int j = 0;
 903
 904             while (j < i)
 905                 string_rel_add_char(&term_tmp, term_component, &j);
 906             *term_tmp++ = '[';
 907
 908             *term_tmp++ = '^';
 909             *term_tmp++ = '-';
 910             string_rel_add_char(&term_tmp, term_component, &i);
 911
 912             *term_tmp++ = ']';
 913             *term_tmp++ = '.';
 914             *term_tmp++ = '*';
 915
 916             *term_tmp++ = '|';
 917
 918             if ((term_tmp - term_dict) > IT_MAX_WORD)
 919                 break;
 920         }
 921         for (i = 0; term_component[i];)
 922             string_rel_add_char(&term_tmp, term_component, &i);
 923         *term_tmp++ = '.';
 924         *term_tmp++ = '+';
 925         *term_tmp++ = ')';
 926         *term_tmp = '\0';
 927         break;
 928     case 4:
 929         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 930                       term_component, space_split, term_dst))
 931             return 0;
 932         yaz_log(log_level_rpn, "Relation >=");
 933
 934         *term_tmp++ = '(';
 935         for (i = 0; term_component[i];)
 936         {
 937             int j = 0;
 938
 939             if (i)
 940                 *term_tmp++ = '|';
 941             while (j < i)
 942                 string_rel_add_char(&term_tmp, term_component, &j);
 943             *term_tmp++ = '[';
 944
 945             if (term_component[i+1])
 946             {
 947                 *term_tmp++ = '^';
 948                 *term_tmp++ = '-';
 949                 string_rel_add_char(&term_tmp, term_component, &i);
 950             }
 951             else
 952             {
 953                 string_rel_add_char(&term_tmp, term_component, &i);
 954                 *term_tmp++ = '-';
 955             }
 956             *term_tmp++ = ']';
 957             *term_tmp++ = '.';
 958             *term_tmp++ = '*';
 959
 960             if ((term_tmp - term_dict) > IT_MAX_WORD)
 961                 break;
 962         }
 963         *term_tmp++ = ')';
 964         *term_tmp = '\0';
 965         break;
 966     case 3:
 967     case 102:
 968     case -1:
 969         yaz_log(log_level_rpn, "Relation =");
 970         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 971                       term_component, space_split, term_dst))
 972             return 0;
 973         strcat(term_tmp, "(");
 974         strcat(term_tmp, term_component);
 975         strcat(term_tmp, ")");
 976         break;
 977     default:
 978         *error_code = 117;
 979         return 0;
 980     }
 981     return 1;
 982 }
 983
 984 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 985                              const char **term_sub,
 986                              oid_value attributeSet, NMEM stream,
 987                              struct grep_info *grep_info,
 988                              int reg_type, int complete_flag,
 989                              int num_bases, char **basenames,
 990                              char *term_dst, int xpath_use);
 991
 992 static ZEBRA_RES term_trunc(ZebraHandle zh,
 993                             Z_AttributesPlusTerm *zapt,
 994                             const char **term_sub,
 995                             oid_value attributeSet, NMEM stream,
 996                             struct grep_info *grep_info,
 997                             int reg_type, int complete_flag,
 998                             int num_bases, char **basenames,
 999                             char *term_dst,
1000                             const char *rank_type, int xpath_use,
1001                             NMEM rset_nmem,
1002                             RSET *rset,
1003                             struct rset_key_control *kc)
1004 {
1005     ZEBRA_RES res;
1006     *rset = 0;
1007     grep_info->isam_p_indx = 0;
1008     res = string_term(zh, zapt, term_sub, attributeSet, stream, grep_info,
1009                       reg_type, complete_flag, num_bases, basenames,
1010                       term_dst, xpath_use);
1011     if (res != ZEBRA_OK)
1012         return res;
1013     if (!*term_sub)  /* no more terms ? */
1014         return res;
1015     yaz_log(log_level_rpn, "term: %s", term_dst);
1016     *rset = rset_trunc(zh, grep_info->isam_p_buf,
1017                        grep_info->isam_p_indx, term_dst,
1018                        strlen(term_dst), rank_type, 1 /* preserve pos */,
1019                        zapt->term->which, rset_nmem,
1020                        kc, kc->scope);
1021     if (!*rset)
1022         return ZEBRA_FAIL;
1023     return ZEBRA_OK;
1024 }
1025
1026 static char *nmem_strdup_i(NMEM nmem, int v)
1027 {
1028     char val_str[64];
1029     sprintf(val_str, "%d", v);
1030     return nmem_strdup(nmem, val_str);
1031 }
1032
1033 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1034                              const char **term_sub,
1035                              oid_value attributeSet, NMEM stream,
1036                              struct grep_info *grep_info,
1037                              int reg_type, int complete_flag,
1038                              int num_bases, char **basenames,
1039                              char *term_dst, int xpath_use)
1040 {
1041     char term_dict[2*IT_MAX_WORD+4000];
1042     int j, r, base_no;
1043     AttrType truncation;
1044     int truncation_value;
1045     AttrType use;
1046     int use_value;
1047     const char *use_string = 0;
1048     oid_value curAttributeSet = attributeSet;
1049     const char *termp;
1050     struct rpn_char_map_info rcmi;
1051     int space_split = complete_flag ? 0 : 1;
1052
1053     int bases_ok = 0;     /* no of databases with OK attribute */
1054     int errCode = 0;      /* err code (if any is not OK) */
1055     char *errString = 0;  /* addinfo */
1056
1057     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1058     attr_init(&use, zapt, 1);
1059     use_value = attr_find_ex(&use, &curAttributeSet, &use_string);
1060     yaz_log(log_level_rpn, "string_term, use value %d", use_value);
1061     attr_init(&truncation, zapt, 5);
1062     truncation_value = attr_find(&truncation, NULL);
1063     yaz_log(log_level_rpn, "truncation value %d", truncation_value);
1064
1065     if (use_value == -1)    /* no attribute - assumy "any" */
1066         use_value = 1016;
1067     for (base_no = 0; base_no < num_bases; base_no++)
1068     {
1069         int ord = -1;
1070         int attr_ok = 0;
1071         int regex_range = 0;
1072         int init_pos = 0;
1073         attent attp;
1074         data1_local_attribute id_xpath_attr;
1075         data1_local_attribute *local_attr;
1076         int max_pos, prefix_len = 0;
1077         int relation_error;
1078
1079         termp = *term_sub;
1080
1081         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1082         {
1083             zh->errCode = YAZ_BIB1_DATABASE_UNAVAILABLE;
1084             zh->errString = basenames[base_no];
1085             return ZEBRA_FAIL;
1086         }
1087         if (xpath_use > 0 && use_value == -2)
1088         {
1089             /* xpath mode and we have a string attribute */
1090             attp.local_attributes = &id_xpath_attr;
1091             attp.attset_ordinal = VAL_IDXPATH;
1092             id_xpath_attr.next = 0;
1093
1094             use_value = xpath_use;  /* xpath_use as use-attribute now */
1095             id_xpath_attr.local = use_value;
1096         }
1097         else if (curAttributeSet == VAL_IDXPATH && use_value >= 0)
1098         {
1099             /* X-Path attribute, use numeric value directly */
1100             attp.local_attributes = &id_xpath_attr;
1101             attp.attset_ordinal = VAL_IDXPATH;
1102             id_xpath_attr.next = 0;
1103             id_xpath_attr.local = use_value;
1104         }
1105         else if (use_string &&
1106                  (ord = zebraExplain_lookup_attr_str(zh->reg->zei,
1107                                                      use_string)) >= 0)
1108         {
1109             /* we have a match for a raw string attribute */
1110             char ord_buf[32];
1111             int i, ord_len;
1112
1113             if (prefix_len)
1114                 term_dict[prefix_len++] = '|';
1115             else
1116                 term_dict[prefix_len++] = '(';
1117
1118             ord_len = key_SU_encode (ord, ord_buf);
1119             for (i = 0; i<ord_len; i++)
1120             {
1121                 term_dict[prefix_len++] = 1;
1122                 term_dict[prefix_len++] = ord_buf[i];
1123             }
1124             attp.local_attributes = 0;  /* no more attributes */
1125         }
1126         else
1127         {
1128             /* lookup in the .att files . Allow string as well */
1129             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1130                                       use_string)))
1131             {
1132                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1133                         curAttributeSet, use_value, r);
1134                 if (r == -1)
1135                 {
1136                     /* set was found, but value wasn't defined */
1137                     errCode = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
1138                     if (use_string)
1139                         errString = nmem_strdup(stream, use_string);
1140                     else
1141                         errString = nmem_strdup_i (stream, use_value);
1142                 }
1143                 else
1144                 {
1145                     int oid[OID_SIZE];
1146                     struct oident oident;
1147
1148                     oident.proto = PROTO_Z3950;
1149                     oident.oclass = CLASS_ATTSET;
1150                     oident.value = curAttributeSet;
1151                     oid_ent_to_oid (&oident, oid);
1152
1153                     errCode = YAZ_BIB1_UNSUPP_ATTRIBUTE_SET;
1154                     errString = nmem_strdup(stream, oident.desc);
1155                 }
1156                 continue;
1157             }
1158         }
1159         for (local_attr = attp.local_attributes; local_attr;
1160              local_attr = local_attr->next)
1161         {
1162             char ord_buf[32];
1163             int i, ord_len;
1164
1165             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1166                                               attp.attset_ordinal,
1167                                               local_attr->local);
1168             if (ord < 0)
1169                 continue;
1170             if (prefix_len)
1171                 term_dict[prefix_len++] = '|';
1172             else
1173                 term_dict[prefix_len++] = '(';
1174
1175             ord_len = key_SU_encode (ord, ord_buf);
1176             for (i = 0; i<ord_len; i++)
1177             {
1178                 term_dict[prefix_len++] = 1;
1179                 term_dict[prefix_len++] = ord_buf[i];
1180             }
1181         }
1182         bases_ok++;
1183         if (prefix_len)
1184             attr_ok = 1;
1185
1186         term_dict[prefix_len++] = ')';
1187         term_dict[prefix_len++] = 1;
1188         term_dict[prefix_len++] = reg_type;
1189         yaz_log(log_level_rpn, "reg_type = %d", term_dict[prefix_len-1]);
1190         term_dict[prefix_len] = '\0';
1191         j = prefix_len;
1192         switch (truncation_value)
1193         {
1194         case -1:         /* not specified */
1195         case 100:        /* do not truncate */
1196             if (!string_relation (zh, zapt, &termp, term_dict,
1197                                   attributeSet,
1198                                   reg_type, space_split, term_dst,
1199                                   &relation_error))
1200             {
1201                 if (relation_error)
1202                 {
1203                     zh->errCode = relation_error;
1204                     return ZEBRA_FAIL;
1205                 }
1206                 *term_sub = 0;
1207                 return ZEBRA_OK;
1208             }
1209             break;
1210         case 1:          /* right truncation */
1211             term_dict[j++] = '(';
1212             if (!term_100(zh->reg->zebra_maps, reg_type,
1213                           &termp, term_dict + j, space_split, term_dst))
1214             {
1215                 *term_sub = 0;
1216                 return ZEBRA_OK;
1217             }
1218             strcat(term_dict, ".*)");
1219             break;
1220         case 2:          /* keft truncation */
1221             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1222             if (!term_100(zh->reg->zebra_maps, reg_type,
1223                           &termp, term_dict + j, space_split, term_dst))
1224             {
1225                 *term_sub = 0;
1226                 return ZEBRA_OK;
1227             }
1228             strcat(term_dict, ")");
1229             break;
1230         case 3:          /* left&right truncation */
1231             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1232             if (!term_100(zh->reg->zebra_maps, reg_type,
1233                           &termp, term_dict + j, space_split, term_dst))
1234             {
1235                 *term_sub = 0;
1236                 return ZEBRA_OK;
1237             }
1238             strcat(term_dict, ".*)");
1239             break;
1240         case 101:        /* process # in term */
1241             term_dict[j++] = '(';
1242             if (!term_101(zh->reg->zebra_maps, reg_type,
1243                           &termp, term_dict + j, space_split, term_dst))
1244             {
1245                 *term_sub = 0;
1246                 return ZEBRA_OK;
1247             }
1248             strcat(term_dict, ")");
1249             break;
1250         case 102:        /* Regexp-1 */
1251             term_dict[j++] = '(';
1252             if (!term_102(zh->reg->zebra_maps, reg_type,
1253                           &termp, term_dict + j, space_split, term_dst))
1254             {
1255                 *term_sub = 0;
1256                 return ZEBRA_OK;
1257             }
1258             strcat(term_dict, ")");
1259             break;
1260         case 103:       /* Regexp-2 */
1261             regex_range = 1;
1262             term_dict[j++] = '(';
1263             init_pos = 2;
1264             if (!term_103(zh->reg->zebra_maps, reg_type,
1265                           &termp, term_dict + j, &regex_range,
1266                           space_split, term_dst))
1267             {
1268                 *term_sub = 0;
1269                 return ZEBRA_OK;
1270             }
1271             strcat(term_dict, ")");
1272             break;
1273         case 104:        /* process # and ! in term */
1274             term_dict[j++] = '(';
1275             if (!term_104(zh->reg->zebra_maps, reg_type,
1276                           &termp, term_dict + j, space_split, term_dst))
1277             {
1278                 *term_sub = 0;
1279                 return ZEBRA_OK;
1280             }
1281             strcat(term_dict, ")");
1282             break;
1283         case 105:        /* process * and ! in term */
1284             term_dict[j++] = '(';
1285             if (!term_105(zh->reg->zebra_maps, reg_type,
1286                           &termp, term_dict + j, space_split, term_dst, 1))
1287             {
1288                 *term_sub = 0;
1289                 return ZEBRA_OK;
1290             }
1291             strcat(term_dict, ")");
1292             break;
1293         case 106:        /* process * and ! in term */
1294             term_dict[j++] = '(';
1295             if (!term_105(zh->reg->zebra_maps, reg_type,
1296                           &termp, term_dict + j, space_split, term_dst, 0))
1297             {
1298                 *term_sub = 0;
1299                 return ZEBRA_OK;
1300             }
1301             strcat(term_dict, ")");
1302             break;
1303         default:
1304             zh->errCode = YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE;
1305             zh->errString = nmem_strdup_i(stream, truncation_value);
1306             return ZEBRA_FAIL;
1307         }
1308         if (attr_ok)
1309         {
1310             char buf[80];
1311             const char *input = term_dict + prefix_len;
1312             esc_str(buf, sizeof(buf), input, strlen(input));
1313         }
1314         if (attr_ok)
1315         {
1316             yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_dict+prefix_len);
1317             r = dict_lookup_grep(zh->reg->dict, term_dict, regex_range,
1318                                  grep_info, &max_pos, init_pos,
1319                                  grep_handle);
1320             if (r)
1321                 yaz_log(YLOG_WARN, "dict_lookup_grep fail %d", r);
1322         }
1323     }
1324     if (!bases_ok)
1325     {
1326         zh->errCode = errCode;
1327         zh->errString = errString;
1328         return ZEBRA_FAIL;
1329     }
1330     *term_sub = termp;
1331     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1332     return ZEBRA_OK;
1333 }
1334
1335
1336 /* convert APT search term to UTF8 */
1337 static ZEBRA_RES zapt_term_to_utf8(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1338                                    char *termz)
1339 {
1340     size_t sizez;
1341     Z_Term *term = zapt->term;
1342
1343     switch (term->which)
1344     {
1345     case Z_Term_general:
1346         if (zh->iconv_to_utf8 != 0)
1347         {
1348             char *inbuf = term->u.general->buf;
1349             size_t inleft = term->u.general->len;
1350             char *outbuf = termz;
1351             size_t outleft = IT_MAX_WORD-1;
1352             size_t ret;
1353
1354             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1355                         &outbuf, &outleft);
1356             if (ret == (size_t)(-1))
1357             {
1358                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1359                 zh->errCode =
1360                     YAZ_BIB1_QUERY_TERM_INCLUDES_CHARS_THAT_DO_NOT_TRANSLATE_INTO_;
1361                 return -1;
1362             }
1363             *outbuf = 0;
1364         }
1365         else
1366         {
1367             sizez = term->u.general->len;
1368             if (sizez > IT_MAX_WORD-1)
1369                 sizez = IT_MAX_WORD-1;
1370             memcpy (termz, term->u.general->buf, sizez);
1371             termz[sizez] = '\0';
1372         }
1373         break;
1374     case Z_Term_characterString:
1375         sizez = strlen(term->u.characterString);
1376         if (sizez > IT_MAX_WORD-1)
1377             sizez = IT_MAX_WORD-1;
1378         memcpy (termz, term->u.characterString, sizez);
1379         termz[sizez] = '\0';
1380         break;
1381     default:
1382         zh->errCode = YAZ_BIB1_UNSUPP_CODED_VALUE_FOR_TERM;
1383         return ZEBRA_FAIL;
1384     }
1385     return ZEBRA_OK;
1386 }
1387
1388 /* convert APT SCAN term to internal cmap */
1389 static ZEBRA_RES trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1390                                  char *termz, int reg_type)
1391 {
1392     char termz0[IT_MAX_WORD];
1393
1394     if (zapt_term_to_utf8(zh, zapt, termz0) == ZEBRA_FAIL)
1395         return ZEBRA_FAIL;    /* error */
1396     else
1397     {
1398         const char **map;
1399         const char *cp = (const char *) termz0;
1400         const char *cp_end = cp + strlen(cp);
1401         const char *src;
1402         int i = 0;
1403         const char *space_map = NULL;
1404         int len;
1405
1406         while ((len = (cp_end - cp)) > 0)
1407         {
1408             map = zebra_maps_input(zh->reg->zebra_maps, reg_type, &cp, len, 0);
1409             if (**map == *CHR_SPACE)
1410                 space_map = *map;
1411             else
1412             {
1413                 if (i && space_map)
1414                     for (src = space_map; *src; src++)
1415                         termz[i++] = *src;
1416                 space_map = NULL;
1417                 for (src = *map; *src; src++)
1418                     termz[i++] = *src;
1419             }
1420         }
1421         termz[i] = '\0';
1422     }
1423     return ZEBRA_OK;
1424 }
1425
1426 char *normalize_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1427                      const char *termz, NMEM stream, unsigned reg_id)
1428 {
1429     WRBUF wrbuf = 0;
1430     AttrType truncation;
1431     int truncation_value;
1432     char *ex_list = 0;
1433
1434     attr_init(&truncation, zapt, 5);
1435     truncation_value = attr_find(&truncation, NULL);
1436
1437     switch (truncation_value)
1438     {
1439     default:
1440         ex_list = "";
1441         break;
1442     case 101:
1443         ex_list = "#";
1444         break;
1445     case 102:
1446     case 103:
1447         ex_list = 0;
1448         break;
1449     case 104:
1450         ex_list = "!#";
1451         break;
1452     case 105:
1453         ex_list = "!*";
1454         break;
1455     }
1456     if (ex_list)
1457         wrbuf = zebra_replace(zh->reg->zebra_maps, reg_id, ex_list,
1458                               termz, strlen(termz));
1459     if (!wrbuf)
1460         return nmem_strdup(stream, termz);
1461     else
1462     {
1463         char *buf = (char*) nmem_malloc(stream, wrbuf_len(wrbuf)+1);
1464         memcpy (buf, wrbuf_buf(wrbuf), wrbuf_len(wrbuf));
1465         buf[wrbuf_len(wrbuf)] = '\0';
1466         return buf;
1467     }
1468 }
1469
1470 static void grep_info_delete(struct grep_info *grep_info)
1471 {
1472 #ifdef TERM_COUNT
1473     xfree(grep_info->term_no);
1474 #endif
1475     xfree(grep_info->isam_p_buf);
1476 }
1477
1478 static int grep_info_prepare(ZebraHandle zh,
1479                              Z_AttributesPlusTerm *zapt,
1480                              struct grep_info *grep_info,
1481                              int reg_type,
1482                              NMEM stream)
1483 {
1484     AttrType termset;
1485     int termset_value_numeric;
1486     const char *termset_value_string;
1487
1488 #ifdef TERM_COUNT
1489     grep_info->term_no = 0;
1490 #endif
1491     grep_info->isam_p_size = 0;
1492     grep_info->isam_p_buf = NULL;
1493     grep_info->zh = zh;
1494     grep_info->reg_type = reg_type;
1495     grep_info->termset = 0;
1496
1497     if (!zapt)
1498         return 0;
1499     attr_init(&termset, zapt, 8);
1500     termset_value_numeric =
1501         attr_find_ex(&termset, NULL, &termset_value_string);
1502     if (termset_value_numeric != -1)
1503     {
1504         char resname[32];
1505         const char *termset_name = 0;
1506         if (termset_value_numeric != -2)
1507         {
1508
1509             sprintf(resname, "%d", termset_value_numeric);
1510             termset_name = resname;
1511         }
1512         else
1513             termset_name = termset_value_string;
1514         yaz_log(log_level_rpn, "creating termset set %s", termset_name);
1515         grep_info->termset = resultSetAdd(zh, termset_name, 1);
1516         if (!grep_info->termset)
1517         {
1518             zh->errCode = YAZ_BIB1_ILLEGAL_RESULT_SET_NAME;
1519             zh->errString = nmem_strdup(stream, termset_name);
1520             return -1;
1521         }
1522     }
1523     return 0;
1524 }
1525
1526
1527 static ZEBRA_RES term_list_trunc(ZebraHandle zh,
1528                                  Z_AttributesPlusTerm *zapt,
1529                                  const char *termz_org,
1530                                  oid_value attributeSet,
1531                                  NMEM stream,
1532                                  int reg_type, int complete_flag,
1533                                  const char *rank_type, int xpath_use,
1534                                  int num_bases, char **basenames,
1535                                  NMEM rset_nmem,
1536                                  RSET **result_sets, int *num_result_sets,
1537                                  struct rset_key_control *kc)
1538 {
1539     char term_dst[IT_MAX_WORD+1];
1540     struct grep_info grep_info;
1541     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1542     const char *termp = termz;
1543     int alloc_sets = 0;
1544
1545     *num_result_sets = 0;
1546     *term_dst = 0;
1547     if (grep_info_prepare(zh, zapt, &grep_info, reg_type, stream))
1548         return ZEBRA_FAIL;
1549     while(1)
1550     {
1551         ZEBRA_RES res;
1552
1553         if (alloc_sets == *num_result_sets)
1554         {
1555             int add = 10;
1556             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1557                                               sizeof(*rnew));
1558             if (alloc_sets)
1559                 memcpy(rnew, *result_sets, alloc_sets * sizeof(*rnew));
1560             alloc_sets = alloc_sets + add;
1561             *result_sets = rnew;
1562         }
1563         res = term_trunc(zh, zapt, &termp, attributeSet,
1564                          stream, &grep_info,
1565                          reg_type, complete_flag,
1566                          num_bases, basenames,
1567                          term_dst, rank_type,
1568                          xpath_use, rset_nmem,
1569                          &(*result_sets)[*num_result_sets],
1570                          kc);
1571         if (res != ZEBRA_OK)
1572         {
1573             int i;
1574             for (i = 0; i < *num_result_sets; i++)
1575                 rset_delete((*result_sets)[i]);
1576             grep_info_delete (&grep_info);
1577             return res;
1578         }
1579         if ((*result_sets)[*num_result_sets] == 0)
1580             break;
1581         (*num_result_sets)++;
1582     }
1583     grep_info_delete(&grep_info);
1584     return ZEBRA_OK;
1585 }
1586
1587 static ZEBRA_RES rpn_search_APT_phrase(ZebraHandle zh,
1588                                        Z_AttributesPlusTerm *zapt,
1589                                        const char *termz_org,
1590                                        oid_value attributeSet,
1591                                        NMEM stream,
1592                                        int reg_type, int complete_flag,
1593                                        const char *rank_type, int xpath_use,
1594                                        int num_bases, char **basenames,
1595                                        NMEM rset_nmem,
1596                                        RSET *rset,
1597                                        struct rset_key_control *kc)
1598 {
1599     RSET *result_sets = 0;
1600     int num_result_sets = 0;
1601     ZEBRA_RES res =
1602         term_list_trunc(zh, zapt, termz_org, attributeSet,
1603                         stream, reg_type, complete_flag,
1604                         rank_type, xpath_use,
1605                         num_bases, basenames,
1606                         rset_nmem,
1607                         &result_sets, &num_result_sets, kc);
1608     if (res != ZEBRA_OK)
1609         return res;
1610     if (num_result_sets == 0)
1611         *rset = rsnull_create (rset_nmem, kc);
1612     else if (num_result_sets == 1)
1613         *rset = result_sets[0];
1614     else
1615         *rset = rsprox_create(rset_nmem, kc, kc->scope,
1616                               num_result_sets, result_sets,
1617                               1 /* ordered */, 0 /* exclusion */,
1618                               3 /* relation */, 1 /* distance */);
1619     if (!*rset)
1620         return ZEBRA_FAIL;
1621     return ZEBRA_OK;
1622 }
1623
1624 static ZEBRA_RES rpn_search_APT_or_list(ZebraHandle zh,
1625                                         Z_AttributesPlusTerm *zapt,
1626                                         const char *termz_org,
1627                                         oid_value attributeSet,
1628                                         NMEM stream,
1629                                         int reg_type, int complete_flag,
1630                                         const char *rank_type,
1631                                         int xpath_use,
1632                                         int num_bases, char **basenames,
1633                                         NMEM rset_nmem,
1634                                         RSET *rset,
1635                                         struct rset_key_control *kc)
1636 {
1637     RSET *result_sets = 0;
1638     int num_result_sets = 0;
1639     ZEBRA_RES res =
1640         term_list_trunc(zh, zapt, termz_org, attributeSet,
1641                         stream, reg_type, complete_flag,
1642                         rank_type, xpath_use,
1643                         num_bases, basenames,
1644                         rset_nmem,
1645                         &result_sets, &num_result_sets, kc);
1646     if (res != ZEBRA_OK)
1647         return res;
1648     if (num_result_sets == 0)
1649         *rset = rsnull_create (rset_nmem, kc);
1650     else if (num_result_sets == 1)
1651         *rset = result_sets[0];
1652     else
1653         *rset = rsmulti_or_create(rset_nmem, kc, kc->scope,
1654                                   num_result_sets, result_sets);
1655     if (!*rset)
1656         return ZEBRA_FAIL;
1657     return ZEBRA_OK;
1658 }
1659
1660 static ZEBRA_RES rpn_search_APT_and_list(ZebraHandle zh,
1661                                          Z_AttributesPlusTerm *zapt,
1662                                          const char *termz_org,
1663                                          oid_value attributeSet,
1664                                          NMEM stream,
1665                                          int reg_type, int complete_flag,
1666                                          const char *rank_type,
1667                                          int xpath_use,
1668                                          int num_bases, char **basenames,
1669                                          NMEM rset_nmem,
1670                                          RSET *rset,
1671                                          struct rset_key_control *kc)
1672 {
1673     RSET *result_sets = 0;
1674     int num_result_sets = 0;
1675     ZEBRA_RES res =
1676         term_list_trunc(zh, zapt, termz_org, attributeSet,
1677                         stream, reg_type, complete_flag,
1678                         rank_type, xpath_use,
1679                         num_bases, basenames,
1680                         rset_nmem,
1681                         &result_sets, &num_result_sets,
1682                         kc);
1683     if (res != ZEBRA_OK)
1684         return res;
1685     if (num_result_sets == 0)
1686         *rset = rsnull_create (rset_nmem, kc);
1687     else if (num_result_sets == 1)
1688         *rset = result_sets[0];
1689     else
1690         *rset = rsmulti_and_create(rset_nmem, kc, kc->scope,
1691                                    num_result_sets, result_sets);
1692     if (!*rset)
1693         return ZEBRA_FAIL;
1694     return ZEBRA_OK;
1695 }
1696
1697 static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1698                             const char **term_sub,
1699                             char *term_dict,
1700                             oid_value attributeSet,
1701                             struct grep_info *grep_info,
1702                             int *max_pos,
1703                             int reg_type,
1704                             char *term_dst,
1705                             int *error_code)
1706 {
1707     AttrType relation;
1708     int relation_value;
1709     int term_value;
1710     int r;
1711     char *term_tmp = term_dict + strlen(term_dict);
1712
1713     *error_code = 0;
1714     attr_init(&relation, zapt, 2);
1715     relation_value = attr_find(&relation, NULL);
1716
1717     yaz_log(log_level_rpn, "numeric relation value=%d", relation_value);
1718
1719     if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1720                   term_dst))
1721         return 0;
1722     term_value = atoi (term_tmp);
1723     switch (relation_value)
1724     {
1725     case 1:
1726         yaz_log(log_level_rpn, "Relation <");
1727         gen_regular_rel(term_tmp, term_value-1, 1);
1728         break;
1729     case 2:
1730         yaz_log(log_level_rpn, "Relation <=");
1731         gen_regular_rel(term_tmp, term_value, 1);
1732         break;
1733     case 4:
1734         yaz_log(log_level_rpn, "Relation >=");
1735         gen_regular_rel(term_tmp, term_value, 0);
1736         break;
1737     case 5:
1738         yaz_log(log_level_rpn, "Relation >");
1739         gen_regular_rel(term_tmp, term_value+1, 0);
1740         break;
1741     case -1:
1742     case 3:
1743         yaz_log(log_level_rpn, "Relation =");
1744         sprintf(term_tmp, "(0*%d)", term_value);
1745         break;
1746     default:
1747         *error_code = 117;
1748         return 0;
1749     }
1750     yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_tmp);
1751     r = dict_lookup_grep(zh->reg->dict, term_dict, 0, grep_info, max_pos,
1752                           0, grep_handle);
1753     if (r)
1754         yaz_log(YLOG_WARN, "dict_lookup_grep fail, rel = gt: %d", r);
1755     yaz_log(log_level_rpn, "%d positions", grep_info->isam_p_indx);
1756     return 1;
1757 }
1758
1759 static ZEBRA_RES numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1760                               const char **term_sub,
1761                               oid_value attributeSet,
1762                               struct grep_info *grep_info,
1763                               int reg_type, int complete_flag,
1764                               int num_bases, char **basenames,
1765                               char *term_dst, int xpath_use, NMEM stream)
1766 {
1767     char term_dict[2*IT_MAX_WORD+2];
1768     int r, base_no;
1769     AttrType use;
1770     int use_value;
1771     const char *use_string = 0;
1772     oid_value curAttributeSet = attributeSet;
1773     const char *termp;
1774     struct rpn_char_map_info rcmi;
1775
1776     int bases_ok = 0;     /* no of databases with OK attribute */
1777     int errCode = 0;      /* err code (if any is not OK) */
1778     char *errString = 0;  /* addinfo */
1779
1780     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1781     attr_init(&use, zapt, 1);
1782     use_value = attr_find_ex(&use, &curAttributeSet, &use_string);
1783
1784     if (use_value == -1)
1785         use_value = 1016;
1786
1787     for (base_no = 0; base_no < num_bases; base_no++)
1788     {
1789         attent attp;
1790         data1_local_attribute id_xpath_attr;
1791         data1_local_attribute *local_attr;
1792         int max_pos, prefix_len = 0;
1793         int relation_error = 0;
1794
1795         termp = *term_sub;
1796         if (use_value == -2)  /* string attribute (assume IDXPATH/any) */
1797         {
1798             use_value = xpath_use;
1799             attp.local_attributes = &id_xpath_attr;
1800             attp.attset_ordinal = VAL_IDXPATH;
1801             id_xpath_attr.next = 0;
1802             id_xpath_attr.local = use_value;
1803         }
1804         else if (curAttributeSet == VAL_IDXPATH)
1805         {
1806             attp.local_attributes = &id_xpath_attr;
1807             attp.attset_ordinal = VAL_IDXPATH;
1808             id_xpath_attr.next = 0;
1809             id_xpath_attr.local = use_value;
1810         }
1811         else
1812         {
1813             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1814                                             use_string)))
1815             {
1816                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1817                       curAttributeSet, use_value, r);
1818                 if (r == -1)
1819                 {
1820                     errCode = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
1821                     if (use_string)
1822                         errString = nmem_strdup(stream, use_string);
1823                     else
1824                         errString = nmem_strdup_i (stream, use_value);
1825                 }
1826                 else
1827                     errCode = YAZ_BIB1_UNSUPP_ATTRIBUTE_SET;
1828                 continue;
1829             }
1830         }
1831         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1832         {
1833             zh->errCode = YAZ_BIB1_DATABASE_UNAVAILABLE;
1834             zh->errString = basenames[base_no];
1835             return -1;
1836         }
1837         for (local_attr = attp.local_attributes; local_attr;
1838              local_attr = local_attr->next)
1839         {
1840             int ord;
1841             char ord_buf[32];
1842             int i, ord_len;
1843
1844             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1845                                               attp.attset_ordinal,
1846                                               local_attr->local);
1847             if (ord < 0)
1848                 continue;
1849             if (prefix_len)
1850                 term_dict[prefix_len++] = '|';
1851             else
1852                 term_dict[prefix_len++] = '(';
1853
1854             ord_len = key_SU_encode (ord, ord_buf);
1855             for (i = 0; i<ord_len; i++)
1856             {
1857                 term_dict[prefix_len++] = 1;
1858                 term_dict[prefix_len++] = ord_buf[i];
1859             }
1860         }
1861         if (!prefix_len)
1862         {
1863             errCode = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
1864             errString = nmem_strdup_i(stream, use_value);
1865             continue;
1866         }
1867         bases_ok++;
1868         term_dict[prefix_len++] = ')';
1869         term_dict[prefix_len++] = 1;
1870         term_dict[prefix_len++] = reg_type;
1871         yaz_log(YLOG_DEBUG, "reg_type = %d", term_dict[prefix_len-1]);
1872         term_dict[prefix_len] = '\0';
1873         if (!numeric_relation(zh, zapt, &termp, term_dict,
1874                               attributeSet, grep_info, &max_pos, reg_type,
1875                               term_dst, &relation_error))
1876         {
1877             if (relation_error)
1878             {
1879                 zh->errCode = relation_error;
1880                 zh->errString = 0;
1881                 return ZEBRA_FAIL;
1882             }
1883             *term_sub = 0;
1884             return ZEBRA_OK;
1885         }
1886     }
1887     if (!bases_ok)
1888     {
1889         zh->errCode = errCode;
1890         zh->errString = errString;
1891         return ZEBRA_FAIL;
1892     }
1893     *term_sub = termp;
1894     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1895     return ZEBRA_OK;
1896 }
1897
1898 static ZEBRA_RES rpn_search_APT_numeric(ZebraHandle zh,
1899                                         Z_AttributesPlusTerm *zapt,
1900                                         const char *termz,
1901                                         oid_value attributeSet,
1902                                         NMEM stream,
1903                                         int reg_type, int complete_flag,
1904                                         const char *rank_type, int xpath_use,
1905                                         int num_bases, char **basenames,
1906                                         NMEM rset_nmem,
1907                                         RSET *rset,
1908                                         struct rset_key_control *kc)
1909 {
1910     char term_dst[IT_MAX_WORD+1];
1911     const char *termp = termz;
1912     RSET *result_sets = 0;
1913     int num_result_sets = 0;
1914     ZEBRA_RES res;
1915     struct grep_info grep_info;
1916     int alloc_sets = 0;
1917
1918     yaz_log(log_level_rpn, "APT_numeric t='%s'", termz);
1919     if (grep_info_prepare(zh, zapt, &grep_info, reg_type, stream))
1920         return ZEBRA_FAIL;
1921     while (1)
1922     {
1923         if (alloc_sets == num_result_sets)
1924         {
1925             int add = 10;
1926             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1927                                               sizeof(*rnew));
1928             if (alloc_sets)
1929                 memcpy(rnew, result_sets, alloc_sets * sizeof(*rnew));
1930             alloc_sets = alloc_sets + add;
1931             result_sets = rnew;
1932         }
1933         yaz_log(YLOG_DEBUG, "APT_numeric termp=%s", termp);
1934         grep_info.isam_p_indx = 0;
1935         res = numeric_term(zh, zapt, &termp, attributeSet, &grep_info,
1936                            reg_type, complete_flag, num_bases, basenames,
1937                            term_dst, xpath_use,
1938                            stream);
1939         if (res == ZEBRA_FAIL || termp == 0)
1940             break;
1941         yaz_log(YLOG_DEBUG, "term: %s", term_dst);
1942         result_sets[num_result_sets] =
1943             rset_trunc(zh, grep_info.isam_p_buf,
1944                        grep_info.isam_p_indx, term_dst,
1945                        strlen(term_dst), rank_type,
1946                        0 /* preserve position */,
1947                        zapt->term->which, rset_nmem,
1948                        kc, kc->scope);
1949         if (!result_sets[num_result_sets])
1950             break;
1951         num_result_sets++;
1952     }
1953     grep_info_delete(&grep_info);
1954     if (termp)
1955     {
1956         int i;
1957         for (i = 0; i<num_result_sets; i++)
1958             rset_delete(result_sets[i]);
1959         return ZEBRA_FAIL;
1960     }
1961     if (num_result_sets == 0)
1962         *rset = rsnull_create(rset_nmem, kc);
1963     if (num_result_sets == 1)
1964         *rset = result_sets[0];
1965     else
1966         *rset = rsmulti_and_create(rset_nmem, kc, kc->scope,
1967                                    num_result_sets, result_sets);
1968     if (!*rset)
1969         return ZEBRA_FAIL;
1970     return ZEBRA_OK;
1971 }
1972
1973 static ZEBRA_RES rpn_search_APT_local(ZebraHandle zh,
1974                                       Z_AttributesPlusTerm *zapt,
1975                                       const char *termz,
1976                                       oid_value attributeSet,
1977                                       NMEM stream,
1978                                       const char *rank_type, NMEM rset_nmem,
1979                                       RSET *rset,
1980                                       struct rset_key_control *kc)
1981 {
1982     RSFD rsfd;
1983     struct it_key key;
1984     int sys;
1985     *rset = rstemp_create(rset_nmem, kc, kc->scope,
1986                           res_get (zh->res, "setTmpDir"),0 );
1987     rsfd = rset_open(*rset, RSETF_WRITE);
1988
1989     sys = atoi(termz);
1990     if (sys <= 0)
1991         sys = 1;
1992     key.mem[0] = sys;
1993     key.mem[1] = 1;
1994     key.len = 2;
1995     rset_write (rsfd, &key);
1996     rset_close (rsfd);
1997     return ZEBRA_OK;
1998 }
1999
2000 static ZEBRA_RES rpn_sort_spec(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2001                                oid_value attributeSet, NMEM stream,
2002                                Z_SortKeySpecList *sort_sequence,
2003                                const char *rank_type,
2004                                RSET *rset,
2005                                struct rset_key_control *kc)
2006 {
2007     int i;
2008     int sort_relation_value;
2009     AttrType sort_relation_type;
2010     Z_SortKeySpec *sks;
2011     Z_SortKey *sk;
2012     int oid[OID_SIZE];
2013     oident oe;
2014     char termz[20];
2015
2016     attr_init(&sort_relation_type, zapt, 7);
2017     sort_relation_value = attr_find(&sort_relation_type, &attributeSet);
2018
2019     if (!sort_sequence->specs)
2020     {
2021         sort_sequence->num_specs = 10;
2022         sort_sequence->specs = (Z_SortKeySpec **)
2023             nmem_malloc(stream, sort_sequence->num_specs *
2024                          sizeof(*sort_sequence->specs));
2025         for (i = 0; i<sort_sequence->num_specs; i++)
2026             sort_sequence->specs[i] = 0;
2027     }
2028     if (zapt->term->which != Z_Term_general)
2029         i = 0;
2030     else
2031         i = atoi_n ((char *) zapt->term->u.general->buf,
2032                     zapt->term->u.general->len);
2033     if (i >= sort_sequence->num_specs)
2034         i = 0;
2035     sprintf(termz, "%d", i);
2036
2037     oe.proto = PROTO_Z3950;
2038     oe.oclass = CLASS_ATTSET;
2039     oe.value = attributeSet;
2040     if (!oid_ent_to_oid (&oe, oid))
2041         return ZEBRA_FAIL;
2042
2043     sks = (Z_SortKeySpec *) nmem_malloc(stream, sizeof(*sks));
2044     sks->sortElement = (Z_SortElement *)
2045         nmem_malloc(stream, sizeof(*sks->sortElement));
2046     sks->sortElement->which = Z_SortElement_generic;
2047     sk = sks->sortElement->u.generic = (Z_SortKey *)
2048         nmem_malloc(stream, sizeof(*sk));
2049     sk->which = Z_SortKey_sortAttributes;
2050     sk->u.sortAttributes = (Z_SortAttributes *)
2051         nmem_malloc(stream, sizeof(*sk->u.sortAttributes));
2052
2053     sk->u.sortAttributes->id = oid;
2054     sk->u.sortAttributes->list = zapt->attributes;
2055
2056     sks->sortRelation = (int *)
2057         nmem_malloc(stream, sizeof(*sks->sortRelation));
2058     if (sort_relation_value == 1)
2059         *sks->sortRelation = Z_SortKeySpec_ascending;
2060     else if (sort_relation_value == 2)
2061         *sks->sortRelation = Z_SortKeySpec_descending;
2062     else
2063         *sks->sortRelation = Z_SortKeySpec_ascending;
2064
2065     sks->caseSensitivity = (int *)
2066         nmem_malloc(stream, sizeof(*sks->caseSensitivity));
2067     *sks->caseSensitivity = 0;
2068
2069     sks->which = Z_SortKeySpec_null;
2070     sks->u.null = odr_nullval ();
2071     sort_sequence->specs[i] = sks;
2072     *rset = rsnull_create (NULL, kc);
2073     return ZEBRA_OK;
2074 }
2075
2076
2077 static int parse_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2078                        oid_value attributeSet,
2079                        struct xpath_location_step *xpath, int max, NMEM mem)
2080 {
2081     oid_value curAttributeSet = attributeSet;
2082     AttrType use;
2083     const char *use_string = 0;
2084
2085     attr_init(&use, zapt, 1);
2086     attr_find_ex(&use, &curAttributeSet, &use_string);
2087
2088     if (!use_string || *use_string != '/')
2089         return -1;
2090
2091     return zebra_parse_xpath_str(use_string, xpath, max, mem);
2092 }
2093
2094
2095
2096 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
2097                         int reg_type, const char *term, int use,
2098                         oid_value curAttributeSet, NMEM rset_nmem,
2099                         struct rset_key_control *kc)
2100 {
2101     RSET rset;
2102     struct grep_info grep_info;
2103     char term_dict[2048];
2104     char ord_buf[32];
2105     int prefix_len = 0;
2106     int ord = zebraExplain_lookup_attr_su(zh->reg->zei, curAttributeSet, use);
2107     int ord_len, i, r, max_pos;
2108     int term_type = Z_Term_characterString;
2109     const char *flags = "void";
2110
2111     if (grep_info_prepare(zh, 0 /* zapt */, &grep_info, '0', stream))
2112         return rsnull_create(rset_nmem, kc);
2113
2114     if (ord < 0)
2115         return rsnull_create(rset_nmem, kc);
2116     if (prefix_len)
2117         term_dict[prefix_len++] = '|';
2118     else
2119         term_dict[prefix_len++] = '(';
2120
2121     ord_len = key_SU_encode (ord, ord_buf);
2122     for (i = 0; i<ord_len; i++)
2123     {
2124         term_dict[prefix_len++] = 1;
2125         term_dict[prefix_len++] = ord_buf[i];
2126     }
2127     term_dict[prefix_len++] = ')';
2128     term_dict[prefix_len++] = 1;
2129     term_dict[prefix_len++] = reg_type;
2130
2131     strcpy(term_dict+prefix_len, term);
2132
2133     grep_info.isam_p_indx = 0;
2134     r = dict_lookup_grep(zh->reg->dict, term_dict, 0,
2135                           &grep_info, &max_pos, 0, grep_handle);
2136     yaz_log(YLOG_DEBUG, "%s %d positions", term,
2137              grep_info.isam_p_indx);
2138     rset = rset_trunc(zh, grep_info.isam_p_buf,
2139                       grep_info.isam_p_indx, term, strlen(term),
2140                       flags, 1, term_type,rset_nmem,
2141                       kc, kc->scope);
2142     grep_info_delete(&grep_info);
2143     return rset;
2144 }
2145
2146 static RSET rpn_search_xpath(ZebraHandle zh,
2147                              oid_value attributeSet,
2148                              int num_bases, char **basenames,
2149                              NMEM stream, const char *rank_type, RSET rset,
2150                              int xpath_len, struct xpath_location_step *xpath,
2151                              NMEM rset_nmem,
2152                              struct rset_key_control *kc)
2153 {
2154     oid_value curAttributeSet = attributeSet;
2155     int base_no;
2156     int i;
2157
2158     if (xpath_len < 0)
2159         return rset;
2160
2161     yaz_log(YLOG_DEBUG, "xpath len=%d", xpath_len);
2162     for (i = 0; i<xpath_len; i++)
2163     {
2164         yaz_log(log_level_rpn, "XPATH %d %s", i, xpath[i].part);
2165
2166     }
2167
2168     curAttributeSet = VAL_IDXPATH;
2169
2170     /*
2171       //a    ->    a/.*
2172       //a/b  ->    b/a/.*
2173       /a     ->    a/
2174       /a/b   ->    b/a/
2175
2176       /      ->    none
2177
2178    a[@attr = value]/b[@other = othervalue]
2179
2180  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
2181  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
2182  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
2183  /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y)
2184  /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y)
2185  /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x)
2186
2187     */
2188
2189     dict_grep_cmap (zh->reg->dict, 0, 0);
2190
2191     for (base_no = 0; base_no < num_bases; base_no++)
2192     {
2193         int level = xpath_len;
2194         int first_path = 1;
2195
2196         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2197         {
2198             zh->errCode = YAZ_BIB1_DATABASE_UNAVAILABLE;
2199             zh->errString = basenames[base_no];
2200             return rset;
2201         }
2202         while (--level >= 0)
2203         {
2204             char xpath_rev[128];
2205             int i, len;
2206             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
2207
2208             *xpath_rev = 0;
2209             len = 0;
2210             for (i = level; i >= 1; --i)
2211             {
2212                 const char *cp = xpath[i].part;
2213                 if (*cp)
2214                 {
2215                     for (;*cp; cp++)
2216                         if (*cp == '*')
2217                         {
2218                             memcpy (xpath_rev + len, "[^/]*", 5);
2219                             len += 5;
2220                         }
2221                         else if (*cp == ' ')
2222                         {
2223
2224                             xpath_rev[len++] = 1;
2225                             xpath_rev[len++] = ' ';
2226                         }
2227
2228                         else
2229                             xpath_rev[len++] = *cp;
2230                     xpath_rev[len++] = '/';
2231                 }
2232                 else if (i == 1)  /* // case */
2233                 {
2234                     xpath_rev[len++] = '.';
2235                     xpath_rev[len++] = '*';
2236                 }
2237             }
2238             xpath_rev[len] = 0;
2239
2240             if (xpath[level].predicate &&
2241                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
2242                 xpath[level].predicate->u.relation.name[0])
2243             {
2244                 WRBUF wbuf = wrbuf_alloc();
2245                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
2246                 if (xpath[level].predicate->u.relation.value)
2247                 {
2248                     const char *cp = xpath[level].predicate->u.relation.value;
2249                     wrbuf_putc(wbuf, '=');
2250
2251                     while (*cp)
2252                     {
2253                         if (strchr(REGEX_CHARS, *cp))
2254                             wrbuf_putc(wbuf, '\\');
2255                         wrbuf_putc(wbuf, *cp);
2256                         cp++;
2257                     }
2258                 }
2259                 wrbuf_puts(wbuf, "");
2260                 rset_attr = xpath_trunc(
2261                     zh, stream, '0', wrbuf_buf(wbuf), 3,
2262                     curAttributeSet, rset_nmem, kc);
2263                 wrbuf_free(wbuf, 1);
2264             }
2265             else
2266             {
2267                 if (!first_path)
2268                     continue;
2269             }
2270             yaz_log(log_level_rpn, "xpath_rev (%d) = %s", level, xpath_rev);
2271             if (strlen(xpath_rev))
2272             {
2273                 rset_start_tag = xpath_trunc(zh, stream, '0',
2274                         xpath_rev, 1, curAttributeSet, rset_nmem, kc);
2275
2276                 rset_end_tag = xpath_trunc(zh, stream, '0',
2277                         xpath_rev, 2, curAttributeSet, rset_nmem, kc);
2278
2279                 rset = rsbetween_create(rset_nmem, kc, kc->scope,
2280                                         rset_start_tag, rset,
2281                                         rset_end_tag, rset_attr);
2282             }
2283             first_path = 0;
2284         }
2285     }
2286
2287     return rset;
2288 }
2289
2290 static ZEBRA_RES rpn_search_APT(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2291                                 oid_value attributeSet, NMEM stream,
2292                                 Z_SortKeySpecList *sort_sequence,
2293                                 int num_bases, char **basenames,
2294                                 NMEM rset_nmem,
2295                                 RSET *rset,
2296                                 struct rset_key_control *kc)
2297 {
2298     ZEBRA_RES res = ZEBRA_OK;
2299     unsigned reg_id;
2300     char *search_type = NULL;
2301     char rank_type[128];
2302     int complete_flag;
2303     int sort_flag;
2304     char termz[IT_MAX_WORD+1];
2305     int xpath_len;
2306     int xpath_use = 0;
2307     struct xpath_location_step xpath[10];
2308
2309     if (!log_level_set)
2310     {
2311         log_level_rpn = yaz_log_module_level("rpn");
2312         log_level_set = 1;
2313     }
2314     zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2315                     rank_type, &complete_flag, &sort_flag);
2316
2317     yaz_log(YLOG_DEBUG, "reg_id=%c", reg_id);
2318     yaz_log(YLOG_DEBUG, "complete_flag=%d", complete_flag);
2319     yaz_log(YLOG_DEBUG, "search_type=%s", search_type);
2320     yaz_log(YLOG_DEBUG, "rank_type=%s", rank_type);
2321
2322     if (zapt_term_to_utf8(zh, zapt, termz) == ZEBRA_FAIL)
2323         return ZEBRA_FAIL;
2324
2325     if (sort_flag)
2326         return rpn_sort_spec(zh, zapt, attributeSet, stream, sort_sequence,
2327                              rank_type, rset, kc);
2328     xpath_len = parse_xpath(zh, zapt, attributeSet, xpath, 10, stream);
2329     if (xpath_len >= 0)
2330     {
2331         xpath_use = 1016;
2332         if (xpath[xpath_len-1].part[0] == '@')
2333             xpath_use = 1015;
2334     }
2335
2336     if (!strcmp(search_type, "phrase"))
2337     {
2338         res = rpn_search_APT_phrase(zh, zapt, termz, attributeSet, stream,
2339                                     reg_id, complete_flag, rank_type,
2340                                     xpath_use,
2341                                     num_bases, basenames, rset_nmem,
2342                                     rset, kc);
2343     }
2344     else if (!strcmp(search_type, "and-list"))
2345     {
2346         res = rpn_search_APT_and_list(zh, zapt, termz, attributeSet, stream,
2347                                       reg_id, complete_flag, rank_type,
2348                                       xpath_use,
2349                                       num_bases, basenames, rset_nmem,
2350                                       rset, kc);
2351     }
2352     else if (!strcmp(search_type, "or-list"))
2353     {
2354         res = rpn_search_APT_or_list(zh, zapt, termz, attributeSet, stream,
2355                                      reg_id, complete_flag, rank_type,
2356                                      xpath_use,
2357                                      num_bases, basenames, rset_nmem,
2358                                      rset, kc);
2359     }
2360     else if (!strcmp(search_type, "local"))
2361     {
2362         res = rpn_search_APT_local(zh, zapt, termz, attributeSet, stream,
2363                                    rank_type, rset_nmem, rset, kc);
2364     }
2365     else if (!strcmp(search_type, "numeric"))
2366     {
2367         res = rpn_search_APT_numeric(zh, zapt, termz, attributeSet, stream,
2368                                      reg_id, complete_flag, rank_type,
2369                                      xpath_use,
2370                                      num_bases, basenames, rset_nmem,
2371                                      rset, kc);
2372     }
2373     else
2374     {
2375         zh->errCode = YAZ_BIB1_UNSUPP_STRUCTURE_ATTRIBUTE;
2376         return ZEBRA_FAIL;
2377     }
2378     if (res != ZEBRA_OK)
2379         return res;
2380     if (!*rset)
2381         return ZEBRA_FAIL;
2382     *rset = rpn_search_xpath(zh, attributeSet, num_bases, basenames,
2383                              stream, rank_type, *rset,
2384                              xpath_len, xpath, rset_nmem, kc);
2385     if (!*rset)
2386         return ZEBRA_FAIL;
2387     return ZEBRA_OK;
2388 }
2389
2390 static ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2391                                       oid_value attributeSet,
2392                                       NMEM stream, NMEM rset_nmem,
2393                                       Z_SortKeySpecList *sort_sequence,
2394                                       int num_bases, char **basenames,
2395                                       RSET **result_sets, int *num_result_sets,
2396                                       Z_Operator *parent_op,
2397                                       struct rset_key_control *kc);
2398
2399 ZEBRA_RES rpn_search_top(ZebraHandle zh, Z_RPNStructure *zs,
2400                          oid_value attributeSet,
2401                          NMEM stream, NMEM rset_nmem,
2402                          Z_SortKeySpecList *sort_sequence,
2403                          int num_bases, char **basenames,
2404                          RSET *result_set)
2405 {
2406     RSET *result_sets = 0;
2407     int num_result_sets = 0;
2408     ZEBRA_RES res;
2409     struct rset_key_control *kc = zebra_key_control_create(zh);
2410
2411     res = rpn_search_structure(zh, zs, attributeSet,
2412                                stream, rset_nmem,
2413                                sort_sequence,
2414                                num_bases, basenames,
2415                                &result_sets, &num_result_sets,
2416                                0 /* no parent op */,
2417                                kc);
2418     if (res != ZEBRA_OK)
2419     {
2420         int i;
2421         for (i = 0; i<num_result_sets; i++)
2422             rset_delete(result_sets[i]);
2423         *result_set = 0;
2424         return res;
2425     }
2426     assert(num_result_sets == 1);
2427     assert(result_sets);
2428     assert(*result_sets);
2429     *result_set = *result_sets;
2430
2431     (*kc->dec)(kc);
2432     return ZEBRA_OK;
2433 }
2434
2435 ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2436                                oid_value attributeSet,
2437                                NMEM stream, NMEM rset_nmem,
2438                                Z_SortKeySpecList *sort_sequence,
2439                                int num_bases, char **basenames,
2440                                RSET **result_sets, int *num_result_sets,
2441                                Z_Operator *parent_op,
2442                                struct rset_key_control *kc)
2443 {
2444     *num_result_sets = 0;
2445     if (zs->which == Z_RPNStructure_complex)
2446     {
2447         ZEBRA_RES res;
2448         Z_Operator *zop = zs->u.complex->roperator;
2449         RSET *result_sets_l = 0;
2450         int num_result_sets_l = 0;
2451         RSET *result_sets_r = 0;
2452         int num_result_sets_r = 0;
2453
2454         res = rpn_search_structure(zh, zs->u.complex->s1,
2455                                    attributeSet, stream, rset_nmem,
2456                                    sort_sequence,
2457                                    num_bases, basenames,
2458                                    &result_sets_l, &num_result_sets_l,
2459                                    zop, kc);
2460         if (res != ZEBRA_OK)
2461         {
2462             int i;
2463             for (i = 0; i<num_result_sets_l; i++)
2464                 rset_delete(result_sets_l[i]);
2465             return res;
2466         }
2467         res = rpn_search_structure(zh, zs->u.complex->s2,
2468                                    attributeSet, stream, rset_nmem,
2469                                    sort_sequence,
2470                                    num_bases, basenames,
2471                                    &result_sets_r, &num_result_sets_r,
2472                                    zop, kc);
2473         if (res != ZEBRA_OK)
2474         {
2475             int i;
2476             for (i = 0; i<num_result_sets_l; i++)
2477                 rset_delete(result_sets_l[i]);
2478             for (i = 0; i<num_result_sets_r; i++)
2479                 rset_delete(result_sets_r[i]);
2480             return res;
2481         }
2482
2483         /* make a new list of result for all children */
2484         *num_result_sets = num_result_sets_l + num_result_sets_r;
2485         *result_sets = nmem_malloc(stream, *num_result_sets *
2486                                    sizeof(**result_sets));
2487         memcpy(*result_sets, result_sets_l,
2488                num_result_sets_l * sizeof(**result_sets));
2489         memcpy(*result_sets + num_result_sets_l, result_sets_r,
2490                num_result_sets_r * sizeof(**result_sets));
2491
2492         if (!parent_op || parent_op->which != zop->which
2493             || (zop->which != Z_Operator_and &&
2494                 zop->which != Z_Operator_or))
2495         {
2496             /* parent node different from this one (or non-present) */
2497             /* we must combine result sets now */
2498             RSET rset;
2499             switch (zop->which)
2500             {
2501             case Z_Operator_and:
2502                 rset = rsmulti_and_create(rset_nmem, kc,
2503                                           kc->scope,
2504                                           *num_result_sets, *result_sets);
2505                 break;
2506             case Z_Operator_or:
2507                 rset = rsmulti_or_create(rset_nmem, kc,
2508                                          kc->scope,
2509                                          *num_result_sets, *result_sets);
2510                 break;
2511             case Z_Operator_and_not:
2512                 rset = rsbool_create_not(rset_nmem, kc,
2513                                          kc->scope,
2514                                          (*result_sets)[0],
2515                                          (*result_sets)[1]);
2516                 break;
2517             case Z_Operator_prox:
2518                 if (zop->u.prox->which != Z_ProximityOperator_known)
2519                 {
2520                     zh->errCode = YAZ_BIB1_UNSUPP_PROX_UNIT_CODE;
2521                     return ZEBRA_FAIL;
2522                 }
2523                 if (*zop->u.prox->u.known != Z_ProxUnit_word)
2524                 {
2525                     char *val = (char *) nmem_malloc(stream, 16);
2526                     zh->errCode = YAZ_BIB1_UNSUPP_PROX_UNIT_CODE;
2527                     zh->errString = val;
2528                     sprintf(val, "%d", *zop->u.prox->u.known);
2529                     return ZEBRA_FAIL;
2530                 }
2531                 else
2532                 {
2533                     rset = rsprox_create(rset_nmem, kc,
2534                                          kc->scope,
2535                                          *num_result_sets, *result_sets,
2536                                          *zop->u.prox->ordered,
2537                                          (!zop->u.prox->exclusion ?
2538                                           0 : *zop->u.prox->exclusion),
2539                                          *zop->u.prox->relationType,
2540                                          *zop->u.prox->distance );
2541                 }
2542                 break;
2543             default:
2544                 zh->errCode = YAZ_BIB1_OPERATOR_UNSUPP;
2545                 return ZEBRA_FAIL;
2546             }
2547             *num_result_sets = 1;
2548             *result_sets = nmem_malloc(stream, *num_result_sets *
2549                                        sizeof(**result_sets));
2550             (*result_sets)[0] = rset;
2551         }
2552     }
2553     else if (zs->which == Z_RPNStructure_simple)
2554     {
2555         RSET rset;
2556         ZEBRA_RES res;
2557
2558         if (zs->u.simple->which == Z_Operand_APT)
2559         {
2560             yaz_log(YLOG_DEBUG, "rpn_search_APT");
2561             res = rpn_search_APT(zh, zs->u.simple->u.attributesPlusTerm,
2562                                  attributeSet, stream, sort_sequence,
2563                                  num_bases, basenames, rset_nmem, &rset,
2564                                  kc);
2565             if (res != ZEBRA_OK)
2566                 return res;
2567         }
2568         else if (zs->u.simple->which == Z_Operand_resultSetId)
2569         {
2570             yaz_log(YLOG_DEBUG, "rpn_search_ref");
2571             rset = resultSetRef(zh, zs->u.simple->u.resultSetId);
2572             if (!rset)
2573             {
2574                 zh->errCode = YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST;
2575                 zh->errString =
2576                     nmem_strdup(stream, zs->u.simple->u.resultSetId);
2577                 return ZEBRA_FAIL;
2578             }
2579             rset_dup(rset);
2580         }
2581         else
2582         {
2583             zh->errCode = YAZ_BIB1_UNSUPP_SEARCH;
2584             return ZEBRA_FAIL;
2585         }
2586         *num_result_sets = 1;
2587         *result_sets = nmem_malloc(stream, *num_result_sets *
2588                                    sizeof(**result_sets));
2589         (*result_sets)[0] = rset;
2590     }
2591     else
2592     {
2593         zh->errCode = YAZ_BIB1_UNSUPP_SEARCH;
2594         return ZEBRA_FAIL;
2595     }
2596     return ZEBRA_OK;
2597 }
2598
2599 struct scan_info_entry {
2600     char *term;
2601     ISAM_P isam_p;
2602 };
2603
2604 struct scan_info {
2605     struct scan_info_entry *list;
2606     ODR odr;
2607     int before, after;
2608     char prefix[20];
2609 };
2610
2611 static int scan_handle (char *name, const char *info, int pos, void *client)
2612 {
2613     int len_prefix, idx;
2614     struct scan_info *scan_info = (struct scan_info *) client;
2615
2616     len_prefix = strlen(scan_info->prefix);
2617     if (memcmp (name, scan_info->prefix, len_prefix))
2618         return 1;
2619     if (pos > 0)
2620         idx = scan_info->after - pos + scan_info->before;
2621     else
2622         idx = - pos - 1;
2623
2624     if (idx < 0)
2625         return 0;
2626     scan_info->list[idx].term = (char *)
2627         odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
2628     strcpy(scan_info->list[idx].term, name + len_prefix);
2629     assert (*info == sizeof(ISAM_P));
2630     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
2631     return 0;
2632 }
2633
2634 static void scan_term_untrans (ZebraHandle zh, NMEM stream, int reg_type,
2635                                char **dst, const char *src)
2636 {
2637     char term_src[IT_MAX_WORD];
2638     char term_dst[IT_MAX_WORD];
2639
2640     term_untrans (zh, reg_type, term_src, src);
2641
2642     if (zh->iconv_from_utf8 != 0)
2643     {
2644         int len;
2645         char *inbuf = term_src;
2646         size_t inleft = strlen(term_src);
2647         char *outbuf = term_dst;
2648         size_t outleft = sizeof(term_dst)-1;
2649         size_t ret;
2650
2651         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2652                          &outbuf, &outleft);
2653         if (ret == (size_t)(-1))
2654             len = 0;
2655         else
2656             len = outbuf - term_dst;
2657         *dst = nmem_malloc(stream, len + 1);
2658         if (len > 0)
2659             memcpy (*dst, term_dst, len);
2660         (*dst)[len] = '\0';
2661     }
2662     else
2663         *dst = nmem_strdup(stream, term_src);
2664 }
2665
2666 static void count_set (RSET r, int *count)
2667 {
2668     zint psysno = 0;
2669     int kno = 0;
2670     struct it_key key;
2671     RSFD rfd;
2672
2673     yaz_log(YLOG_DEBUG, "count_set");
2674
2675     *count = 0;
2676     rfd = rset_open (r, RSETF_READ);
2677     while (rset_read (rfd, &key,0 /* never mind terms */))
2678     {
2679         if (key.mem[0] != psysno)
2680         {
2681             psysno = key.mem[0];
2682             (*count)++;
2683         }
2684         kno++;
2685     }
2686     rset_close (rfd);
2687     yaz_log(YLOG_DEBUG, "%d keys, %d records", kno, *count);
2688 }
2689
2690 ZEBRA_RES rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2691                    oid_value attributeset,
2692                    int num_bases, char **basenames,
2693                    int *position, int *num_entries, ZebraScanEntry **list,
2694                    int *is_partial, RSET limit_set, int return_zero)
2695 {
2696     int i;
2697     int pos = *position;
2698     int num = *num_entries;
2699     int before;
2700     int after;
2701     int base_no;
2702     char termz[IT_MAX_WORD+20];
2703     AttrType use;
2704     int use_value;
2705     const char *use_string = 0;
2706     struct scan_info *scan_info_array;
2707     ZebraScanEntry *glist;
2708     int ords[32], ord_no = 0;
2709     int ptr[32];
2710
2711     int bases_ok = 0;     /* no of databases with OK attribute */
2712     int errCode = 0;      /* err code (if any is not OK) */
2713     char *errString = 0;  /* addinfo */
2714
2715     unsigned reg_id;
2716     char *search_type = NULL;
2717     char rank_type[128];
2718     int complete_flag;
2719     int sort_flag;
2720     NMEM rset_nmem = NULL;
2721     struct rset_key_control *kc = 0;
2722
2723     *list = 0;
2724     *is_partial = 0;
2725
2726     if (attributeset == VAL_NONE)
2727         attributeset = VAL_BIB1;
2728
2729     if (!limit_set)
2730     {
2731         AttrType termset;
2732         int termset_value_numeric;
2733         const char *termset_value_string;
2734         attr_init(&termset, zapt, 8);
2735         termset_value_numeric =
2736             attr_find_ex(&termset, NULL, &termset_value_string);
2737         if (termset_value_numeric != -1)
2738         {
2739             char resname[32];
2740             const char *termset_name = 0;
2741
2742             if (termset_value_numeric != -2)
2743             {
2744
2745                 sprintf(resname, "%d", termset_value_numeric);
2746                 termset_name = resname;
2747             }
2748             else
2749                 termset_name = termset_value_string;
2750
2751             limit_set = resultSetRef (zh, termset_name);
2752         }
2753     }
2754
2755     yaz_log(YLOG_DEBUG, "position = %d, num = %d set=%d",
2756             pos, num, attributeset);
2757
2758     attr_init(&use, zapt, 1);
2759     use_value = attr_find_ex(&use, &attributeset, &use_string);
2760
2761     if (zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2762                         rank_type, &complete_flag, &sort_flag))
2763     {
2764         *num_entries = 0;
2765         zh->errCode = YAZ_BIB1_UNSUPP_ATTRIBUTE_TYPE;
2766         return ZEBRA_FAIL;
2767     }
2768     yaz_log(YLOG_DEBUG, "use_value = %d", use_value);
2769
2770     if (use_value == -1)
2771         use_value = 1016;
2772     for (base_no = 0; base_no < num_bases && ord_no < 32; base_no++)
2773     {
2774         data1_local_attribute *local_attr;
2775         attent attp;
2776         int ord;
2777
2778         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2779         {
2780             zh->errString = basenames[base_no];
2781             zh->errCode = YAZ_BIB1_DATABASE_UNAVAILABLE;
2782             *num_entries = 0;
2783             return ZEBRA_FAIL;
2784         }
2785
2786         if (use_string &&
2787             (ord = zebraExplain_lookup_attr_str(zh->reg->zei,
2788                                                 use_string)) >= 0)
2789         {
2790             /* we have a match for a raw string attribute */
2791             if (ord > 0)
2792                 ords[ord_no++] = ord;
2793             attp.local_attributes = 0;  /* no more attributes */
2794         }
2795         else
2796         {
2797             int r;
2798
2799             if ((r = att_getentbyatt (zh, &attp, attributeset, use_value,
2800                                       use_string)))
2801             {
2802                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
2803                         attributeset, use_value);
2804                 if (r == -1)
2805                 {
2806                     errCode = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
2807                     if (use_string)
2808                         errString = odr_strdup(stream, use_string);
2809                     else
2810                     {
2811                         char val_str[32];
2812                         sprintf(val_str, "%d", use_value);
2813                         errString = odr_strdup(stream, val_str);
2814                     }
2815                 }
2816                 else
2817                     errCode = YAZ_BIB1_UNSUPP_ATTRIBUTE_SET;
2818                 continue;
2819             }
2820         }
2821         bases_ok++;
2822         for (local_attr = attp.local_attributes; local_attr && ord_no < 32;
2823              local_attr = local_attr->next)
2824         {
2825             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
2826                                               attp.attset_ordinal,
2827                                               local_attr->local);
2828             if (ord > 0)
2829                 ords[ord_no++] = ord;
2830         }
2831     }
2832     if (!bases_ok && errCode)
2833     {
2834         zh->errCode = errCode;
2835         zh->errString = errString;
2836         *num_entries = 0;
2837         return ZEBRA_FAIL;
2838     }
2839     if (ord_no == 0)
2840     {
2841         *num_entries = 0;
2842         return ZEBRA_OK;
2843     }
2844     /* prepare dictionary scanning */
2845     if (num < 1)
2846     {
2847         *num_entries = 0;
2848         return ZEBRA_OK;
2849     }
2850     before = pos-1;
2851     if (before < 0)
2852         before = 0;
2853     after = 1+num-pos;
2854     if (after < 0)
2855         after = 0;
2856     yaz_log(YLOG_DEBUG, "rpn_scan pos=%d num=%d before=%d "
2857             "after=%d before+after=%d",
2858             pos, num, before, after, before+after);
2859     scan_info_array = (struct scan_info *)
2860         odr_malloc(stream, ord_no * sizeof(*scan_info_array));
2861     for (i = 0; i < ord_no; i++)
2862     {
2863         int j, prefix_len = 0;
2864         int before_tmp = before, after_tmp = after;
2865         struct scan_info *scan_info = scan_info_array + i;
2866         struct rpn_char_map_info rcmi;
2867
2868         rpn_char_map_prepare (zh->reg, reg_id, &rcmi);
2869
2870         scan_info->before = before;
2871         scan_info->after = after;
2872         scan_info->odr = stream;
2873
2874         scan_info->list = (struct scan_info_entry *)
2875             odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
2876         for (j = 0; j<before+after; j++)
2877             scan_info->list[j].term = NULL;
2878
2879         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2880         termz[prefix_len++] = reg_id;
2881         termz[prefix_len] = 0;
2882         strcpy(scan_info->prefix, termz);
2883
2884         if (trans_scan_term(zh, zapt, termz+prefix_len, reg_id) == ZEBRA_FAIL)
2885             return ZEBRA_FAIL;
2886
2887         dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
2888                   scan_info, scan_handle);
2889     }
2890     glist = (ZebraScanEntry *)
2891         odr_malloc(stream, (before+after)*sizeof(*glist));
2892
2893     rset_nmem = nmem_create();
2894     kc = zebra_key_control_create(zh);
2895
2896     /* consider terms after main term */
2897     for (i = 0; i < ord_no; i++)
2898         ptr[i] = before;
2899
2900     *is_partial = 0;
2901     for (i = 0; i<after; i++)
2902     {
2903         int j, j0 = -1;
2904         const char *mterm = NULL;
2905         const char *tst;
2906         RSET rset = 0;
2907         int lo = i + pos-1; /* offset in result list */
2908
2909         /* find: j0 is the first of the minimal values */
2910         for (j = 0; j < ord_no; j++)
2911         {
2912             if (ptr[j] < before+after && ptr[j] >= 0 &&
2913                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2914                 (!mterm || strcmp (tst, mterm) < 0))
2915             {
2916                 j0 = j;
2917                 mterm = tst;
2918             }
2919         }
2920         if (j0 == -1)
2921             break;  /* no value found, stop */
2922
2923         /* get result set for first one , but only if it's within bounds */
2924         if (lo >= 0)
2925         {
2926             /* get result set for first term */
2927             scan_term_untrans(zh, stream->mem, reg_id,
2928                               &glist[lo].term, mterm);
2929             rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2930                               glist[lo].term, strlen(glist[lo].term),
2931                               NULL, 0, zapt->term->which, rset_nmem,
2932                               kc, kc->scope);
2933         }
2934         ptr[j0]++; /* move index for this set .. */
2935         /* get result set for remaining scan terms */
2936         for (j = j0+1; j<ord_no; j++)
2937         {
2938             if (ptr[j] < before+after && ptr[j] >= 0 &&
2939                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2940                 !strcmp (tst, mterm))
2941             {
2942                 if (lo >= 0)
2943                 {
2944                     RSET rsets[2];
2945
2946                     rsets[0] = rset;
2947                     rsets[1] =
2948                         rset_trunc(
2949                             zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
2950                             glist[lo].term,
2951                             strlen(glist[lo].term), NULL, 0,
2952                             zapt->term->which,rset_nmem,
2953                             kc, kc->scope);
2954                     rset = rsmulti_or_create(rset_nmem, kc,
2955                                              2, kc->scope, rsets);
2956                 }
2957                 ptr[j]++;
2958             }
2959         }
2960         if (lo >= 0)
2961         {
2962             /* merge with limit_set if given */
2963             if (limit_set)
2964             {
2965                 RSET rsets[2];
2966                 rsets[0] = rset;
2967                 rsets[1] = rset_dup(limit_set);
2968
2969                 rset = rsmulti_and_create(rset_nmem, kc,
2970                                           kc->scope, 2, rsets);
2971             }
2972             /* count it */
2973             count_set(rset, &glist[lo].occurrences);
2974             rset_delete(rset);
2975         }
2976     }
2977     if (i < after)
2978     {
2979         *num_entries -= (after-i);
2980         *is_partial = 1;
2981         if (*num_entries < 0)
2982         {
2983             (*kc->dec)(kc);
2984             nmem_destroy(rset_nmem);
2985             *num_entries = 0;
2986             return ZEBRA_OK;
2987         }
2988     }
2989     /* consider terms before main term */
2990     for (i = 0; i<ord_no; i++)
2991         ptr[i] = 0;
2992
2993     for (i = 0; i<before; i++)
2994     {
2995         int j, j0 = -1;
2996         const char *mterm = NULL;
2997         const char *tst;
2998         RSET rset;
2999         int lo = before-1-i; /* offset in result list */
3000
3001         for (j = 0; j <ord_no; j++)
3002         {
3003             if (ptr[j] < before && ptr[j] >= 0 &&
3004                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
3005                 (!mterm || strcmp (tst, mterm) > 0))
3006             {
3007                 j0 = j;
3008                     mterm = tst;
3009             }
3010         }
3011         if (j0 == -1)
3012             break;
3013
3014         scan_term_untrans (zh, stream->mem, reg_id,
3015                            &glist[lo].term, mterm);
3016
3017         rset = rset_trunc
3018             (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
3019              glist[lo].term, strlen(glist[lo].term),
3020              NULL, 0, zapt->term->which,rset_nmem,
3021              kc, kc->scope);
3022
3023         ptr[j0]++;
3024
3025         for (j = j0+1; j<ord_no; j++)
3026         {
3027             if (ptr[j] < before && ptr[j] >= 0 &&
3028                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
3029                 !strcmp (tst, mterm))
3030             {
3031                 RSET rsets[2];
3032
3033                 rsets[0] = rset;
3034                 rsets[1] = rset_trunc(
3035                     zh,
3036                     &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
3037                     glist[lo].term,
3038                     strlen(glist[lo].term), NULL, 0,
3039                     zapt->term->which, rset_nmem,
3040                     kc, kc->scope);
3041                 rset = rsmulti_or_create(rset_nmem, kc,
3042                                          2, kc->scope, rsets);
3043
3044                 ptr[j]++;
3045             }
3046         }
3047         if (limit_set)
3048         {
3049             RSET rsets[2];
3050             rsets[0] = rset;
3051             rsets[1] = rset_dup(limit_set);
3052
3053             rset = rsmulti_and_create(rset_nmem, kc,
3054                                       kc->scope, 2, rsets);
3055         }
3056         count_set (rset, &glist[lo].occurrences);
3057         rset_delete (rset);
3058     }
3059     (*kc->dec)(kc);
3060     nmem_destroy(rset_nmem);
3061     i = before-i;
3062     if (i)
3063     {
3064         *is_partial = 1;
3065         *position -= i;
3066         *num_entries -= i;
3067         if (*num_entries <= 0)
3068         {
3069             *num_entries = 0;
3070             return ZEBRA_OK;
3071         }
3072     }
3073
3074     *list = glist + i;               /* list is set to first 'real' entry */
3075
3076     yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
3077             *position, *num_entries);
3078     return ZEBRA_OK;
3079 }
3080