index/zrpn.c

   1 /* $Id: zrpn.c,v 1.178 2005-04-25 21:40:34 adam Exp $
   2    Copyright (C) 1995-2005
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with Zebra; see the file LICENSE.zebra.  If not, write to the
  19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  20 02111-1307, USA.
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #ifdef WIN32
  26 #include <io.h>
  27 #else
  28 #include <unistd.h>
  29 #endif
  30 #include <ctype.h>
  31
  32 #include "index.h"
  33 #include <zebra_xpath.h>
  34
  35 #include <charmap.h>
  36 #include <rset.h>
  37
  38 static const struct key_control it_ctrl =
  39 {
  40     sizeof(struct it_key),
  41     2, /* we have sysnos and seqnos in this key, nothing more */
  42     key_compare_it,
  43     key_logdump_txt,   /* FIXME  - clean up these functions */
  44     key_get_seq,
  45 };
  46
  47
  48 const struct key_control *key_it_ctrl = &it_ctrl;
  49
  50 struct rpn_char_map_info
  51 {
  52     ZebraMaps zm;
  53     int reg_type;
  54 };
  55
  56 typedef struct
  57 {
  58     int type;
  59     int major;
  60     int minor;
  61     Z_AttributesPlusTerm *zapt;
  62 } AttrType;
  63
  64
  65 static int log_level_set = 0;
  66 static int log_level_rpn = 0;
  67
  68 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  69 {
  70     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  71     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  72 #if 0
  73     if (out && *out)
  74     {
  75         const char *outp = *out;
  76         yaz_log(YLOG_LOG, "---");
  77         while (*outp)
  78         {
  79             yaz_log(YLOG_LOG, "%02X", *outp);
  80             outp++;
  81         }
  82     }
  83 #endif
  84     return out;
  85 }
  86
  87 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  88                                   struct rpn_char_map_info *map_info)
  89 {
  90     map_info->zm = reg->zebra_maps;
  91     map_info->reg_type = reg_type;
  92     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  93 }
  94
  95 static int attr_find_ex(AttrType *src, oid_value *attributeSetP,
  96                          const char **string_value)
  97 {
  98     int num_attributes;
  99
 100     num_attributes = src->zapt->attributes->num_attributes;
 101     while (src->major < num_attributes)
 102     {
 103         Z_AttributeElement *element;
 104
 105         element = src->zapt->attributes->attributes[src->major];
 106         if (src->type == *element->attributeType)
 107         {
 108             switch (element->which)
 109             {
 110             case Z_AttributeValue_numeric:
 111                 ++(src->major);
 112                 if (element->attributeSet && attributeSetP)
 113                 {
 114                     oident *attrset;
 115
 116                     attrset = oid_getentbyoid(element->attributeSet);
 117                     *attributeSetP = attrset->value;
 118                 }
 119                 return *element->value.numeric;
 120                 break;
 121             case Z_AttributeValue_complex:
 122                 if (src->minor >= element->value.complex->num_list)
 123                     break;
 124                 if (element->attributeSet && attributeSetP)
 125                 {
 126                     oident *attrset;
 127
 128                     attrset = oid_getentbyoid(element->attributeSet);
 129                     *attributeSetP = attrset->value;
 130                 }
 131                 if (element->value.complex->list[src->minor]->which ==
 132                     Z_StringOrNumeric_numeric)
 133                 {
 134                     ++(src->minor);
 135                     return
 136                         *element->value.complex->list[src->minor-1]->u.numeric;
 137                 }
 138                 else if (element->value.complex->list[src->minor]->which ==
 139                          Z_StringOrNumeric_string)
 140                 {
 141                     if (!string_value)
 142                         break;
 143                     ++(src->minor);
 144                     *string_value =
 145                         element->value.complex->list[src->minor-1]->u.string;
 146                     return -2;
 147                 }
 148                 else
 149                     break;
 150             default:
 151                 assert(0);
 152             }
 153         }
 154         ++(src->major);
 155     }
 156     return -1;
 157 }
 158
 159 static int attr_find(AttrType *src, oid_value *attributeSetP)
 160 {
 161     return attr_find_ex(src, attributeSetP, 0);
 162 }
 163
 164 static void attr_init(AttrType *src, Z_AttributesPlusTerm *zapt,
 165                        int type)
 166 {
 167     src->zapt = zapt;
 168     src->type = type;
 169     src->major = 0;
 170     src->minor = 0;
 171 }
 172
 173 #define TERM_COUNT
 174
 175 struct grep_info {
 176 #ifdef TERM_COUNT
 177     int *term_no;
 178 #endif
 179     ISAM_P *isam_p_buf;
 180     int isam_p_size;
 181     int isam_p_indx;
 182     ZebraHandle zh;
 183     int reg_type;
 184     ZebraSet termset;
 185 };
 186
 187 static void term_untrans(ZebraHandle zh, int reg_type,
 188                            char *dst, const char *src)
 189 {
 190     int len = 0;
 191     while (*src)
 192     {
 193         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
 194                                            reg_type, &src);
 195         if (!cp && len < IT_MAX_WORD-1)
 196             dst[len++] = *src++;
 197         else
 198             while (*cp && len < IT_MAX_WORD-1)
 199                 dst[len++] = *cp++;
 200     }
 201     dst[len] = '\0';
 202 }
 203
 204 static void add_isam_p(const char *name, const char *info,
 205                        struct grep_info *p)
 206 {
 207     if (!log_level_set)
 208     {
 209         log_level_rpn = yaz_log_module_level("rpn");
 210         log_level_set = 1;
 211     }
 212     if (p->isam_p_indx == p->isam_p_size)
 213     {
 214         ISAM_P *new_isam_p_buf;
 215 #ifdef TERM_COUNT
 216         int *new_term_no;
 217 #endif
 218         p->isam_p_size = 2*p->isam_p_size + 100;
 219         new_isam_p_buf = (ISAM_P *) xmalloc(sizeof(*new_isam_p_buf) *
 220                                             p->isam_p_size);
 221         if (p->isam_p_buf)
 222         {
 223             memcpy(new_isam_p_buf, p->isam_p_buf,
 224                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 225             xfree(p->isam_p_buf);
 226         }
 227         p->isam_p_buf = new_isam_p_buf;
 228
 229 #ifdef TERM_COUNT
 230         new_term_no = (int *) xmalloc(sizeof(*new_term_no) * p->isam_p_size);
 231         if (p->term_no)
 232         {
 233             memcpy(new_term_no, p->isam_p_buf,
 234                     p->isam_p_indx * sizeof(*p->term_no));
 235             xfree(p->term_no);
 236         }
 237         p->term_no = new_term_no;
 238 #endif
 239     }
 240     assert(*info == sizeof(*p->isam_p_buf));
 241     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 242
 243 #if 1
 244     if (p->termset)
 245     {
 246         const char *db;
 247         int set, use;
 248         char term_tmp[IT_MAX_WORD];
 249         int su_code = 0;
 250         int len = key_SU_decode (&su_code, name);
 251
 252         term_untrans  (p->zh, p->reg_type, term_tmp, name+len+1);
 253         yaz_log(log_level_rpn, "grep: %d %c %s", su_code, name[len], term_tmp);
 254         zebraExplain_lookup_ord (p->zh->reg->zei,
 255                                  su_code, &db, &set, &use);
 256         yaz_log(log_level_rpn, "grep:  set=%d use=%d db=%s", set, use, db);
 257
 258         resultSetAddTerm(p->zh, p->termset, name[len], db,
 259                          set, use, term_tmp);
 260     }
 261 #endif
 262     (p->isam_p_indx)++;
 263 }
 264
 265 static int grep_handle(char *name, const char *info, void *p)
 266 {
 267     add_isam_p(name, info, (struct grep_info *) p);
 268     return 0;
 269 }
 270
 271 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 272                     const char *ct1, const char *ct2, int first)
 273 {
 274     const char *s1, *s0 = *src;
 275     const char **map;
 276
 277     /* skip white space */
 278     while (*s0)
 279     {
 280         if (ct1 && strchr(ct1, *s0))
 281             break;
 282         if (ct2 && strchr(ct2, *s0))
 283             break;
 284         s1 = s0;
 285         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 286         if (**map != *CHR_SPACE)
 287             break;
 288         s0 = s1;
 289     }
 290     *src = s0;
 291     return *s0;
 292 }
 293
 294
 295 static void esc_str(char *out_buf, int out_size,
 296                     const char *in_buf, int in_size)
 297 {
 298     int k;
 299
 300     assert(out_buf);
 301     assert(in_buf);
 302     assert(out_size > 20);
 303     *out_buf = '\0';
 304     for (k = 0; k<in_size; k++)
 305     {
 306         int c = in_buf[k] & 0xff;
 307         int pc;
 308         if (c < 32 || c > 126)
 309             pc = '?';
 310         else
 311             pc = c;
 312         sprintf(out_buf +strlen(out_buf), "%02X:%c  ", c, pc);
 313         if (strlen(out_buf) > out_size-20)
 314         {
 315             strcat(out_buf, "..");
 316             break;
 317         }
 318     }
 319 }
 320
 321 #define REGEX_CHARS " []()|.*+?!"
 322
 323 /* term_100: handle term, where trunc = none(no operators at all) */
 324 static int term_100(ZebraMaps zebra_maps, int reg_type,
 325                     const char **src, char *dst, int space_split,
 326                     char *dst_term)
 327 {
 328     const char *s0;
 329     const char **map;
 330     int i = 0;
 331     int j = 0;
 332
 333     const char *space_start = 0;
 334     const char *space_end = 0;
 335
 336     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 337         return 0;
 338     s0 = *src;
 339     while (*s0)
 340     {
 341         const char *s1 = s0;
 342         int q_map_match = 0;
 343         map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 344                                 &q_map_match);
 345         if (space_split)
 346         {
 347             if (**map == *CHR_SPACE)
 348                 break;
 349         }
 350         else  /* complete subfield only. */
 351         {
 352             if (**map == *CHR_SPACE)
 353             {   /* save space mapping for later  .. */
 354                 space_start = s1;
 355                 space_end = s0;
 356                 continue;
 357             }
 358             else if (space_start)
 359             {   /* reload last space */
 360                 while (space_start < space_end)
 361                 {
 362                     if (strchr(REGEX_CHARS, *space_start))
 363                         dst[i++] = '\\';
 364                     dst_term[j++] = *space_start;
 365                     dst[i++] = *space_start++;
 366                 }
 367                 /* and reset */
 368                 space_start = space_end = 0;
 369             }
 370         }
 371         /* add non-space char */
 372         memcpy(dst_term+j, s1, s0 - s1);
 373         j += (s0 - s1);
 374         if (!q_map_match)
 375         {
 376             while (s1 < s0)
 377             {
 378                 if (strchr(REGEX_CHARS, *s1))
 379                     dst[i++] = '\\';
 380                 dst[i++] = *s1++;
 381             }
 382         }
 383         else
 384         {
 385             char tmpbuf[80];
 386             esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 387
 388             strcpy(dst + i, map[0]);
 389             i += strlen(map[0]);
 390         }
 391     }
 392     dst[i] = '\0';
 393     dst_term[j] = '\0';
 394     *src = s0;
 395     return i;
 396 }
 397
 398 /* term_101: handle term, where trunc = Process # */
 399 static int term_101(ZebraMaps zebra_maps, int reg_type,
 400                     const char **src, char *dst, int space_split,
 401                     char *dst_term)
 402 {
 403     const char *s0;
 404     const char **map;
 405     int i = 0;
 406     int j = 0;
 407
 408     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 409         return 0;
 410     s0 = *src;
 411     while (*s0)
 412     {
 413         if (*s0 == '#')
 414         {
 415             dst[i++] = '.';
 416             dst[i++] = '*';
 417             dst_term[j++] = *s0++;
 418         }
 419         else
 420         {
 421             const char *s1 = s0;
 422             int q_map_match = 0;
 423             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 424                                     &q_map_match);
 425             if (space_split && **map == *CHR_SPACE)
 426                 break;
 427
 428             /* add non-space char */
 429             memcpy(dst_term+j, s1, s0 - s1);
 430             j += (s0 - s1);
 431             if (!q_map_match)
 432             {
 433                 while (s1 < s0)
 434                 {
 435                     if (strchr(REGEX_CHARS, *s1))
 436                         dst[i++] = '\\';
 437                     dst[i++] = *s1++;
 438                 }
 439             }
 440             else
 441             {
 442                 char tmpbuf[80];
 443                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 444
 445                 strcpy(dst + i, map[0]);
 446                 i += strlen(map[0]);
 447             }
 448         }
 449     }
 450     dst[i] = '\0';
 451     dst_term[j++] = '\0';
 452     *src = s0;
 453     return i;
 454 }
 455
 456 /* term_103: handle term, where trunc = re-2 (regular expressions) */
 457 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 458                     char *dst, int *errors, int space_split,
 459                     char *dst_term)
 460 {
 461     int i = 0;
 462     int j = 0;
 463     const char *s0;
 464     const char **map;
 465
 466     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 467         return 0;
 468     s0 = *src;
 469     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 470         isdigit(((const unsigned char *)s0)[1]))
 471     {
 472         *errors = s0[1] - '0';
 473         s0 += 3;
 474         if (*errors > 3)
 475             *errors = 3;
 476     }
 477     while (*s0)
 478     {
 479         if (strchr("^\\()[].*+?|-", *s0))
 480         {
 481             dst_term[j++] = *s0;
 482             dst[i++] = *s0++;
 483         }
 484         else
 485         {
 486             const char *s1 = s0;
 487             int q_map_match = 0;
 488             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 489                                     &q_map_match);
 490             if (space_split && **map == *CHR_SPACE)
 491                 break;
 492
 493             /* add non-space char */
 494             memcpy(dst_term+j, s1, s0 - s1);
 495             j += (s0 - s1);
 496             if (!q_map_match)
 497             {
 498                 while (s1 < s0)
 499                 {
 500                     if (strchr(REGEX_CHARS, *s1))
 501                         dst[i++] = '\\';
 502                     dst[i++] = *s1++;
 503                 }
 504             }
 505             else
 506             {
 507                 char tmpbuf[80];
 508                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 509
 510                 strcpy(dst + i, map[0]);
 511                 i += strlen(map[0]);
 512             }
 513         }
 514     }
 515     dst[i] = '\0';
 516     dst_term[j] = '\0';
 517     *src = s0;
 518
 519     return i;
 520 }
 521
 522 /* term_103: handle term, where trunc = re-1 (regular expressions) */
 523 static int term_102(ZebraMaps zebra_maps, int reg_type, const char **src,
 524                     char *dst, int space_split, char *dst_term)
 525 {
 526     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 527                     dst_term);
 528 }
 529
 530
 531 /* term_104: handle term, where trunc = Process # and ! */
 532 static int term_104(ZebraMaps zebra_maps, int reg_type,
 533                     const char **src, char *dst, int space_split,
 534                     char *dst_term)
 535 {
 536     const char *s0, *s1;
 537     const char **map;
 538     int i = 0;
 539     int j = 0;
 540
 541     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 542         return 0;
 543     s0 = *src;
 544     while (*s0)
 545     {
 546         if (*s0 == '?')
 547         {
 548             dst_term[j++] = *s0++;
 549             if (*s0 >= '0' && *s0 <= '9')
 550             {
 551                 int limit = 0;
 552                 while (*s0 >= '0' && *s0 <= '9')
 553                 {
 554                     limit = limit * 10 + (*s0 - '0');
 555                     dst_term[j++] = *s0++;
 556                 }
 557                 if (limit > 20)
 558                     limit = 20;
 559                 while (--limit >= 0)
 560                 {
 561                     dst[i++] = '.';
 562                     dst[i++] = '?';
 563                 }
 564             }
 565             else
 566             {
 567                 dst[i++] = '.';
 568                 dst[i++] = '*';
 569             }
 570         }
 571         else if (*s0 == '*')
 572         {
 573             dst[i++] = '.';
 574             dst[i++] = '*';
 575             dst_term[j++] = *s0++;
 576         }
 577         else if (*s0 == '#')
 578         {
 579             dst[i++] = '.';
 580             dst_term[j++] = *s0++;
 581         }
 582         else
 583         {
 584             s1 = s0;
 585             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 586             if (space_split && **map == *CHR_SPACE)
 587                 break;
 588             while (s1 < s0)
 589             {
 590                 if (strchr(REGEX_CHARS, *s1))
 591                     dst[i++] = '\\';
 592                 dst_term[j++] = *s1;
 593                 dst[i++] = *s1++;
 594             }
 595         }
 596     }
 597     dst[i] = '\0';
 598     dst_term[j++] = '\0';
 599     *src = s0;
 600     return i;
 601 }
 602
 603 /* term_105/106: handle term, where trunc = Process * and ! and right trunc */
 604 static int term_105(ZebraMaps zebra_maps, int reg_type,
 605                     const char **src, char *dst, int space_split,
 606                     char *dst_term, int right_truncate)
 607 {
 608     const char *s0, *s1;
 609     const char **map;
 610     int i = 0;
 611     int j = 0;
 612
 613     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 614         return 0;
 615     s0 = *src;
 616     while (*s0)
 617     {
 618         if (*s0 == '*')
 619         {
 620             dst[i++] = '.';
 621             dst[i++] = '*';
 622             dst_term[j++] = *s0++;
 623         }
 624         else if (*s0 == '!')
 625         {
 626             dst[i++] = '.';
 627             dst_term[j++] = *s0++;
 628         }
 629         else
 630         {
 631             s1 = s0;
 632             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 633             if (space_split && **map == *CHR_SPACE)
 634                 break;
 635             while (s1 < s0)
 636             {
 637                 if (strchr(REGEX_CHARS, *s1))
 638                     dst[i++] = '\\';
 639                 dst_term[j++] = *s1;
 640                 dst[i++] = *s1++;
 641             }
 642         }
 643     }
 644     if (right_truncate)
 645     {
 646         dst[i++] = '.';
 647         dst[i++] = '*';
 648     }
 649     dst[i] = '\0';
 650
 651     dst_term[j++] = '\0';
 652     *src = s0;
 653     return i;
 654 }
 655
 656
 657 /* gen_regular_rel - generate regular expression from relation
 658  *  val:     border value (inclusive)
 659  *  islt:    1 if <=; 0 if >=.
 660  */
 661 static void gen_regular_rel(char *dst, int val, int islt)
 662 {
 663     int dst_p;
 664     int w, d, i;
 665     int pos = 0;
 666     char numstr[20];
 667
 668     yaz_log(YLOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 669     if (val >= 0)
 670     {
 671         if (islt)
 672             strcpy(dst, "(-[0-9]+|(");
 673         else
 674             strcpy(dst, "((");
 675     }
 676     else
 677     {
 678         if (!islt)
 679         {
 680             strcpy(dst, "([0-9]+|-(");
 681             dst_p = strlen(dst);
 682             islt = 1;
 683         }
 684         else
 685         {
 686             strcpy(dst, "(-(");
 687             islt = 0;
 688         }
 689         val = -val;
 690     }
 691     dst_p = strlen(dst);
 692     sprintf(numstr, "%d", val);
 693     for (w = strlen(numstr); --w >= 0; pos++)
 694     {
 695         d = numstr[w];
 696         if (pos > 0)
 697         {
 698             if (islt)
 699             {
 700                 if (d == '0')
 701                     continue;
 702                 d--;
 703             }
 704             else
 705             {
 706                 if (d == '9')
 707                     continue;
 708                 d++;
 709             }
 710         }
 711
 712         strcpy(dst + dst_p, numstr);
 713         dst_p = strlen(dst) - pos - 1;
 714
 715         if (islt)
 716         {
 717             if (d != '0')
 718             {
 719                 dst[dst_p++] = '[';
 720                 dst[dst_p++] = '0';
 721                 dst[dst_p++] = '-';
 722                 dst[dst_p++] = d;
 723                 dst[dst_p++] = ']';
 724             }
 725             else
 726                 dst[dst_p++] = d;
 727         }
 728         else
 729         {
 730             if (d != '9')
 731             {
 732                 dst[dst_p++] = '[';
 733                 dst[dst_p++] = d;
 734                 dst[dst_p++] = '-';
 735                 dst[dst_p++] = '9';
 736                 dst[dst_p++] = ']';
 737             }
 738             else
 739                 dst[dst_p++] = d;
 740         }
 741         for (i = 0; i<pos; i++)
 742         {
 743             dst[dst_p++] = '[';
 744             dst[dst_p++] = '0';
 745             dst[dst_p++] = '-';
 746             dst[dst_p++] = '9';
 747             dst[dst_p++] = ']';
 748         }
 749         dst[dst_p++] = '|';
 750     }
 751     dst[dst_p] = '\0';
 752     if (islt)
 753     {
 754         /* match everything less than 10^(pos-1) */
 755         strcat(dst, "0*");
 756         for (i = 1; i<pos; i++)
 757             strcat(dst, "[0-9]?");
 758     }
 759     else
 760     {
 761         /* match everything greater than 10^pos */
 762         for (i = 0; i <= pos; i++)
 763             strcat(dst, "[0-9]");
 764         strcat(dst, "[0-9]*");
 765     }
 766     strcat(dst, "))");
 767 }
 768
 769 void string_rel_add_char(char **term_p, const char *src, int *indx)
 770 {
 771     if (src[*indx] == '\\')
 772         *(*term_p)++ = src[(*indx)++];
 773     *(*term_p)++ = src[(*indx)++];
 774 }
 775
 776 /*
 777  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 778  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 779  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 780  *              ([^-a].*|a[^-b].*|ab[c-].*)
 781  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 782  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 783  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 784  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 785  */
 786 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 787                            const char **term_sub, char *term_dict,
 788                            oid_value attributeSet,
 789                            int reg_type, int space_split, char *term_dst,
 790                            int *error_code)
 791 {
 792     AttrType relation;
 793     int relation_value;
 794     int i;
 795     char *term_tmp = term_dict + strlen(term_dict);
 796     char term_component[2*IT_MAX_WORD+20];
 797
 798     attr_init(&relation, zapt, 2);
 799     relation_value = attr_find(&relation, NULL);
 800
 801     *error_code = 0;
 802     yaz_log(YLOG_DEBUG, "string relation value=%d", relation_value);
 803     switch (relation_value)
 804     {
 805     case 1:
 806         if (!term_100(zh->reg->zebra_maps, reg_type,
 807                       term_sub, term_component,
 808                       space_split, term_dst))
 809             return 0;
 810         yaz_log(log_level_rpn, "Relation <");
 811
 812         *term_tmp++ = '(';
 813         for (i = 0; term_component[i]; )
 814         {
 815             int j = 0;
 816
 817             if (i)
 818                 *term_tmp++ = '|';
 819             while (j < i)
 820                 string_rel_add_char(&term_tmp, term_component, &j);
 821
 822             *term_tmp++ = '[';
 823
 824             *term_tmp++ = '^';
 825             string_rel_add_char(&term_tmp, term_component, &i);
 826             *term_tmp++ = '-';
 827
 828             *term_tmp++ = ']';
 829             *term_tmp++ = '.';
 830             *term_tmp++ = '*';
 831
 832             if ((term_tmp - term_dict) > IT_MAX_WORD)
 833                 break;
 834         }
 835         *term_tmp++ = ')';
 836         *term_tmp = '\0';
 837         break;
 838     case 2:
 839         if (!term_100(zh->reg->zebra_maps, reg_type,
 840                       term_sub, term_component,
 841                       space_split, term_dst))
 842             return 0;
 843         yaz_log(log_level_rpn, "Relation <=");
 844
 845         *term_tmp++ = '(';
 846         for (i = 0; term_component[i]; )
 847         {
 848             int j = 0;
 849
 850             while (j < i)
 851                 string_rel_add_char(&term_tmp, term_component, &j);
 852             *term_tmp++ = '[';
 853
 854             *term_tmp++ = '^';
 855             string_rel_add_char(&term_tmp, term_component, &i);
 856             *term_tmp++ = '-';
 857
 858             *term_tmp++ = ']';
 859             *term_tmp++ = '.';
 860             *term_tmp++ = '*';
 861
 862             *term_tmp++ = '|';
 863
 864             if ((term_tmp - term_dict) > IT_MAX_WORD)
 865                 break;
 866         }
 867         for (i = 0; term_component[i]; )
 868             string_rel_add_char(&term_tmp, term_component, &i);
 869         *term_tmp++ = ')';
 870         *term_tmp = '\0';
 871         break;
 872     case 5:
 873         if (!term_100 (zh->reg->zebra_maps, reg_type,
 874                        term_sub, term_component, space_split, term_dst))
 875             return 0;
 876         yaz_log(log_level_rpn, "Relation >");
 877
 878         *term_tmp++ = '(';
 879         for (i = 0; term_component[i];)
 880         {
 881             int j = 0;
 882
 883             while (j < i)
 884                 string_rel_add_char(&term_tmp, term_component, &j);
 885             *term_tmp++ = '[';
 886
 887             *term_tmp++ = '^';
 888             *term_tmp++ = '-';
 889             string_rel_add_char(&term_tmp, term_component, &i);
 890
 891             *term_tmp++ = ']';
 892             *term_tmp++ = '.';
 893             *term_tmp++ = '*';
 894
 895             *term_tmp++ = '|';
 896
 897             if ((term_tmp - term_dict) > IT_MAX_WORD)
 898                 break;
 899         }
 900         for (i = 0; term_component[i];)
 901             string_rel_add_char(&term_tmp, term_component, &i);
 902         *term_tmp++ = '.';
 903         *term_tmp++ = '+';
 904         *term_tmp++ = ')';
 905         *term_tmp = '\0';
 906         break;
 907     case 4:
 908         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 909                       term_component, space_split, term_dst))
 910             return 0;
 911         yaz_log(log_level_rpn, "Relation >=");
 912
 913         *term_tmp++ = '(';
 914         for (i = 0; term_component[i];)
 915         {
 916             int j = 0;
 917
 918             if (i)
 919                 *term_tmp++ = '|';
 920             while (j < i)
 921                 string_rel_add_char(&term_tmp, term_component, &j);
 922             *term_tmp++ = '[';
 923
 924             if (term_component[i+1])
 925             {
 926                 *term_tmp++ = '^';
 927                 *term_tmp++ = '-';
 928                 string_rel_add_char(&term_tmp, term_component, &i);
 929             }
 930             else
 931             {
 932                 string_rel_add_char(&term_tmp, term_component, &i);
 933                 *term_tmp++ = '-';
 934             }
 935             *term_tmp++ = ']';
 936             *term_tmp++ = '.';
 937             *term_tmp++ = '*';
 938
 939             if ((term_tmp - term_dict) > IT_MAX_WORD)
 940                 break;
 941         }
 942         *term_tmp++ = ')';
 943         *term_tmp = '\0';
 944         break;
 945     case 3:
 946     case 102:
 947     case -1:
 948         yaz_log(log_level_rpn, "Relation =");
 949         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 950                       term_component, space_split, term_dst))
 951             return 0;
 952         strcat(term_tmp, "(");
 953         strcat(term_tmp, term_component);
 954         strcat(term_tmp, ")");
 955         break;
 956     default:
 957         *error_code = 117;
 958         return 0;
 959     }
 960     return 1;
 961 }
 962
 963 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 964                              const char **term_sub,
 965                              oid_value attributeSet, NMEM stream,
 966                              struct grep_info *grep_info,
 967                              int reg_type, int complete_flag,
 968                              int num_bases, char **basenames,
 969                              char *term_dst, int xpath_use);
 970
 971 static ZEBRA_RES term_trunc(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 972                             const char **term_sub,
 973                             oid_value attributeSet, NMEM stream,
 974                             struct grep_info *grep_info,
 975                             int reg_type, int complete_flag,
 976                             int num_bases, char **basenames,
 977                             char *term_dst,
 978                             const char *rank_type, int xpath_use,
 979                             NMEM rset_nmem,
 980                             RSET *rset)
 981 {
 982     ZEBRA_RES res;
 983     *rset = 0;
 984     grep_info->isam_p_indx = 0;
 985     res = string_term(zh, zapt, term_sub, attributeSet, stream, grep_info,
 986                       reg_type, complete_flag, num_bases, basenames,
 987                       term_dst, xpath_use);
 988     if (res != ZEBRA_OK)
 989         return res;
 990     if (!*term_sub)  /* no more terms ? */
 991         return res;
 992     yaz_log(log_level_rpn, "term: %s", term_dst);
 993     *rset = rset_trunc(zh, grep_info->isam_p_buf,
 994                        grep_info->isam_p_indx, term_dst,
 995                        strlen(term_dst), rank_type, 1 /* preserve pos */,
 996                        zapt->term->which, rset_nmem,
 997                        key_it_ctrl, key_it_ctrl->scope);
 998     if (!*rset)
 999         return ZEBRA_FAIL;
1000     return ZEBRA_OK;
1001 }
1002
1003 static char *nmem_strdup_i(NMEM nmem, int v)
1004 {
1005     char val_str[64];
1006     sprintf(val_str, "%d", v);
1007     return nmem_strdup(nmem, val_str);
1008 }
1009
1010 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1011                              const char **term_sub,
1012                              oid_value attributeSet, NMEM stream,
1013                              struct grep_info *grep_info,
1014                              int reg_type, int complete_flag,
1015                              int num_bases, char **basenames,
1016                              char *term_dst, int xpath_use)
1017 {
1018     char term_dict[2*IT_MAX_WORD+4000];
1019     int j, r, base_no;
1020     AttrType truncation;
1021     int truncation_value;
1022     AttrType use;
1023     int use_value;
1024     const char *use_string = 0;
1025     oid_value curAttributeSet = attributeSet;
1026     const char *termp;
1027     struct rpn_char_map_info rcmi;
1028     int space_split = complete_flag ? 0 : 1;
1029
1030     int bases_ok = 0;     /* no of databases with OK attribute */
1031     int errCode = 0;      /* err code (if any is not OK) */
1032     char *errString = 0;  /* addinfo */
1033
1034     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1035     attr_init(&use, zapt, 1);
1036     use_value = attr_find_ex(&use, &curAttributeSet, &use_string);
1037     yaz_log(log_level_rpn, "string_term, use value %d", use_value);
1038     attr_init(&truncation, zapt, 5);
1039     truncation_value = attr_find(&truncation, NULL);
1040     yaz_log(log_level_rpn, "truncation value %d", truncation_value);
1041
1042     if (use_value == -1)    /* no attribute - assumy "any" */
1043         use_value = 1016;
1044     for (base_no = 0; base_no < num_bases; base_no++)
1045     {
1046         int ord = -1;
1047         int attr_ok = 0;
1048         int regex_range = 0;
1049         int init_pos = 0;
1050         attent attp;
1051         data1_local_attribute id_xpath_attr;
1052         data1_local_attribute *local_attr;
1053         int max_pos, prefix_len = 0;
1054         int relation_error;
1055
1056         termp = *term_sub;
1057
1058         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1059         {
1060             zh->errCode = 109; /* Database unavailable */
1061             zh->errString = basenames[base_no];
1062             return ZEBRA_FAIL;
1063         }
1064         if (xpath_use > 0 && use_value == -2)
1065         {
1066             /* xpath mode and we have a string attribute */
1067             attp.local_attributes = &id_xpath_attr;
1068             attp.attset_ordinal = VAL_IDXPATH;
1069             id_xpath_attr.next = 0;
1070
1071             use_value = xpath_use;  /* xpath_use as use-attribute now */
1072             id_xpath_attr.local = use_value;
1073         }
1074         else if (curAttributeSet == VAL_IDXPATH && use_value >= 0)
1075         {
1076             /* X-Path attribute, use numeric value directly */
1077             attp.local_attributes = &id_xpath_attr;
1078             attp.attset_ordinal = VAL_IDXPATH;
1079             id_xpath_attr.next = 0;
1080             id_xpath_attr.local = use_value;
1081         }
1082         else if (use_string &&
1083                  (ord = zebraExplain_lookup_attr_str(zh->reg->zei,
1084                                                      use_string)) >= 0)
1085         {
1086             /* we have a match for a raw string attribute */
1087             char ord_buf[32];
1088             int i, ord_len;
1089
1090             if (prefix_len)
1091                 term_dict[prefix_len++] = '|';
1092             else
1093                 term_dict[prefix_len++] = '(';
1094
1095             ord_len = key_SU_encode (ord, ord_buf);
1096             for (i = 0; i<ord_len; i++)
1097             {
1098                 term_dict[prefix_len++] = 1;
1099                 term_dict[prefix_len++] = ord_buf[i];
1100             }
1101             attp.local_attributes = 0;  /* no more attributes */
1102         }
1103         else
1104         {
1105             /* lookup in the .att files . Allow string as well */
1106             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1107                                       use_string)))
1108             {
1109                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1110                         curAttributeSet, use_value, r);
1111                 if (r == -1)
1112                 {
1113                     /* set was found, but value wasn't defined */
1114                     errCode = 114;
1115                     if (use_string)
1116                         errString = nmem_strdup(stream, use_string);
1117                     else
1118                         errString = nmem_strdup_i (stream, use_value);
1119                 }
1120                 else
1121                 {
1122                     int oid[OID_SIZE];
1123                     struct oident oident;
1124
1125                     oident.proto = PROTO_Z3950;
1126                     oident.oclass = CLASS_ATTSET;
1127                     oident.value = curAttributeSet;
1128                     oid_ent_to_oid (&oident, oid);
1129
1130                     errCode = 121;
1131                     errString = nmem_strdup(stream, oident.desc);
1132                 }
1133                 continue;
1134             }
1135         }
1136         for (local_attr = attp.local_attributes; local_attr;
1137              local_attr = local_attr->next)
1138         {
1139             char ord_buf[32];
1140             int i, ord_len;
1141
1142             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1143                                               attp.attset_ordinal,
1144                                               local_attr->local);
1145             if (ord < 0)
1146                 continue;
1147             if (prefix_len)
1148                 term_dict[prefix_len++] = '|';
1149             else
1150                 term_dict[prefix_len++] = '(';
1151
1152             ord_len = key_SU_encode (ord, ord_buf);
1153             for (i = 0; i<ord_len; i++)
1154             {
1155                 term_dict[prefix_len++] = 1;
1156                 term_dict[prefix_len++] = ord_buf[i];
1157             }
1158         }
1159         bases_ok++;
1160         if (prefix_len)
1161             attr_ok = 1;
1162
1163         term_dict[prefix_len++] = ')';
1164         term_dict[prefix_len++] = 1;
1165         term_dict[prefix_len++] = reg_type;
1166         yaz_log(log_level_rpn, "reg_type = %d", term_dict[prefix_len-1]);
1167         term_dict[prefix_len] = '\0';
1168         j = prefix_len;
1169         switch (truncation_value)
1170         {
1171         case -1:         /* not specified */
1172         case 100:        /* do not truncate */
1173             if (!string_relation (zh, zapt, &termp, term_dict,
1174                                   attributeSet,
1175                                   reg_type, space_split, term_dst,
1176                                   &relation_error))
1177             {
1178                 if (relation_error)
1179                 {
1180                     zh->errCode = relation_error;
1181                     return ZEBRA_FAIL;
1182                 }
1183                 *term_sub = 0;
1184                 return ZEBRA_OK;
1185             }
1186             break;
1187         case 1:          /* right truncation */
1188             term_dict[j++] = '(';
1189             if (!term_100(zh->reg->zebra_maps, reg_type,
1190                           &termp, term_dict + j, space_split, term_dst))
1191             {
1192                 *term_sub = 0;
1193                 return ZEBRA_OK;
1194             }
1195             strcat(term_dict, ".*)");
1196             break;
1197         case 2:          /* keft truncation */
1198             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1199             if (!term_100(zh->reg->zebra_maps, reg_type,
1200                           &termp, term_dict + j, space_split, term_dst))
1201             {
1202                 *term_sub = 0;
1203                 return ZEBRA_OK;
1204             }
1205             strcat(term_dict, ")");
1206             break;
1207         case 3:          /* left&right truncation */
1208             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1209             if (!term_100(zh->reg->zebra_maps, reg_type,
1210                           &termp, term_dict + j, space_split, term_dst))
1211             {
1212                 *term_sub = 0;
1213                 return ZEBRA_OK;
1214             }
1215             strcat(term_dict, ".*)");
1216             break;
1217         case 101:        /* process # in term */
1218             term_dict[j++] = '(';
1219             if (!term_101(zh->reg->zebra_maps, reg_type,
1220                           &termp, term_dict + j, space_split, term_dst))
1221             {
1222                 *term_sub = 0;
1223                 return ZEBRA_OK;
1224             }
1225             strcat(term_dict, ")");
1226             break;
1227         case 102:        /* Regexp-1 */
1228             term_dict[j++] = '(';
1229             if (!term_102(zh->reg->zebra_maps, reg_type,
1230                           &termp, term_dict + j, space_split, term_dst))
1231             {
1232                 *term_sub = 0;
1233                 return ZEBRA_OK;
1234             }
1235             strcat(term_dict, ")");
1236             break;
1237         case 103:       /* Regexp-2 */
1238             r = 1;
1239             term_dict[j++] = '(';
1240             init_pos = 2;
1241             if (!term_103(zh->reg->zebra_maps, reg_type,
1242                           &termp, term_dict + j, &regex_range,
1243                           space_split, term_dst))
1244             {
1245                 *term_sub = 0;
1246                 return ZEBRA_OK;
1247             }
1248             strcat(term_dict, ")");
1249         case 104:        /* process # and ! in term */
1250             term_dict[j++] = '(';
1251             if (!term_104(zh->reg->zebra_maps, reg_type,
1252                           &termp, term_dict + j, space_split, term_dst))
1253             {
1254                 *term_sub = 0;
1255                 return ZEBRA_OK;
1256             }
1257             strcat(term_dict, ")");
1258             break;
1259         case 105:        /* process * and ! in term */
1260             term_dict[j++] = '(';
1261             if (!term_105(zh->reg->zebra_maps, reg_type,
1262                           &termp, term_dict + j, space_split, term_dst, 1))
1263             {
1264                 *term_sub = 0;
1265                 return ZEBRA_OK;
1266             }
1267             strcat(term_dict, ")");
1268             break;
1269         case 106:        /* process * and ! in term */
1270             term_dict[j++] = '(';
1271             if (!term_105(zh->reg->zebra_maps, reg_type,
1272                           &termp, term_dict + j, space_split, term_dst, 0))
1273             {
1274                 *term_sub = 0;
1275                 return ZEBRA_OK;
1276             }
1277             strcat(term_dict, ")");
1278             break;
1279         default:
1280             zh->errCode = 120;
1281             zh->errString = nmem_strdup_i(stream, truncation_value);
1282             return ZEBRA_FAIL;
1283         }
1284         if (attr_ok)
1285         {
1286             char buf[80];
1287             const char *input = term_dict + prefix_len;
1288             esc_str(buf, sizeof(buf), input, strlen(input));
1289         }
1290         if (attr_ok)
1291         {
1292             yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_dict+prefix_len);
1293             r = dict_lookup_grep(zh->reg->dict, term_dict, regex_range,
1294                                  grep_info, &max_pos, init_pos,
1295                                  grep_handle);
1296             if (r)
1297                 yaz_log(YLOG_WARN, "dict_lookup_grep fail %d", r);
1298         }
1299     }
1300     if (!bases_ok)
1301     {
1302         zh->errCode = errCode;
1303         zh->errString = errString;
1304         return ZEBRA_FAIL;
1305     }
1306     *term_sub = termp;
1307     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1308     return ZEBRA_OK;
1309 }
1310
1311
1312 /* convert APT search term to UTF8 */
1313 static ZEBRA_RES zapt_term_to_utf8(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1314                                    char *termz)
1315 {
1316     size_t sizez;
1317     Z_Term *term = zapt->term;
1318
1319     switch (term->which)
1320     {
1321     case Z_Term_general:
1322         if (zh->iconv_to_utf8 != 0)
1323         {
1324             char *inbuf = term->u.general->buf;
1325             size_t inleft = term->u.general->len;
1326             char *outbuf = termz;
1327             size_t outleft = IT_MAX_WORD-1;
1328             size_t ret;
1329
1330             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1331                         &outbuf, &outleft);
1332             if (ret == (size_t)(-1))
1333             {
1334                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1335                 zh->errCode = 125;
1336                 return -1;
1337             }
1338             *outbuf = 0;
1339         }
1340         else
1341         {
1342             sizez = term->u.general->len;
1343             if (sizez > IT_MAX_WORD-1)
1344                 sizez = IT_MAX_WORD-1;
1345             memcpy (termz, term->u.general->buf, sizez);
1346             termz[sizez] = '\0';
1347         }
1348         break;
1349     case Z_Term_characterString:
1350         sizez = strlen(term->u.characterString);
1351         if (sizez > IT_MAX_WORD-1)
1352             sizez = IT_MAX_WORD-1;
1353         memcpy (termz, term->u.characterString, sizez);
1354         termz[sizez] = '\0';
1355         break;
1356     default:
1357         zh->errCode = 124;
1358         return ZEBRA_FAIL;
1359     }
1360     return ZEBRA_OK;
1361 }
1362
1363 /* convert APT SCAN term to internal cmap */
1364 static ZEBRA_RES trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1365                                  char *termz, int reg_type)
1366 {
1367     char termz0[IT_MAX_WORD];
1368
1369     if (zapt_term_to_utf8(zh, zapt, termz0) == ZEBRA_FAIL)
1370         return ZEBRA_FAIL;    /* error */
1371     else
1372     {
1373         const char **map;
1374         const char *cp = (const char *) termz0;
1375         const char *cp_end = cp + strlen(cp);
1376         const char *src;
1377         int i = 0;
1378         const char *space_map = NULL;
1379         int len;
1380
1381         while ((len = (cp_end - cp)) > 0)
1382         {
1383             map = zebra_maps_input(zh->reg->zebra_maps, reg_type, &cp, len, 0);
1384             if (**map == *CHR_SPACE)
1385                 space_map = *map;
1386             else
1387             {
1388                 if (i && space_map)
1389                     for (src = space_map; *src; src++)
1390                         termz[i++] = *src;
1391                 space_map = NULL;
1392                 for (src = *map; *src; src++)
1393                     termz[i++] = *src;
1394             }
1395         }
1396         termz[i] = '\0';
1397     }
1398     return ZEBRA_OK;
1399 }
1400
1401 char *normalize_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1402                      const char *termz, NMEM stream, unsigned reg_id)
1403 {
1404     WRBUF wrbuf = 0;
1405     AttrType truncation;
1406     int truncation_value;
1407     char *ex_list = 0;
1408
1409     attr_init(&truncation, zapt, 5);
1410     truncation_value = attr_find(&truncation, NULL);
1411
1412     switch (truncation_value)
1413     {
1414     default:
1415         ex_list = "";
1416         break;
1417     case 101:
1418         ex_list = "#";
1419         break;
1420     case 102:
1421     case 103:
1422         ex_list = 0;
1423         break;
1424     case 104:
1425         ex_list = "!#";
1426         break;
1427     case 105:
1428         ex_list = "!*";
1429         break;
1430     }
1431     if (ex_list)
1432         wrbuf = zebra_replace(zh->reg->zebra_maps, reg_id, ex_list,
1433                               termz, strlen(termz));
1434     if (!wrbuf)
1435         return nmem_strdup(stream, termz);
1436     else
1437     {
1438         char *buf = (char*) nmem_malloc(stream, wrbuf_len(wrbuf)+1);
1439         memcpy (buf, wrbuf_buf(wrbuf), wrbuf_len(wrbuf));
1440         buf[wrbuf_len(wrbuf)] = '\0';
1441         return buf;
1442     }
1443 }
1444
1445 static void grep_info_delete(struct grep_info *grep_info)
1446 {
1447 #ifdef TERM_COUNT
1448     xfree(grep_info->term_no);
1449 #endif
1450     xfree(grep_info->isam_p_buf);
1451 }
1452
1453 static int grep_info_prepare(ZebraHandle zh,
1454                              Z_AttributesPlusTerm *zapt,
1455                              struct grep_info *grep_info,
1456                              int reg_type,
1457                              NMEM stream)
1458 {
1459     AttrType termset;
1460     int termset_value_numeric;
1461     const char *termset_value_string;
1462
1463 #ifdef TERM_COUNT
1464     grep_info->term_no = 0;
1465 #endif
1466     grep_info->isam_p_size = 0;
1467     grep_info->isam_p_buf = NULL;
1468     grep_info->zh = zh;
1469     grep_info->reg_type = reg_type;
1470     grep_info->termset = 0;
1471
1472     if (!zapt)
1473         return 0;
1474     attr_init(&termset, zapt, 8);
1475     termset_value_numeric =
1476         attr_find_ex(&termset, NULL, &termset_value_string);
1477     if (termset_value_numeric != -1)
1478     {
1479         char resname[32];
1480         const char *termset_name = 0;
1481         if (termset_value_numeric != -2)
1482         {
1483
1484             sprintf(resname, "%d", termset_value_numeric);
1485             termset_name = resname;
1486         }
1487         else
1488             termset_name = termset_value_string;
1489         yaz_log(log_level_rpn, "creating termset set %s", termset_name);
1490         grep_info->termset = resultSetAdd(zh, termset_name, 1);
1491         if (!grep_info->termset)
1492         {
1493             zh->errCode = 128;
1494             zh->errString = nmem_strdup(stream, termset_name);
1495             return -1;
1496         }
1497     }
1498     return 0;
1499 }
1500
1501
1502 static ZEBRA_RES term_list_trunc(ZebraHandle zh,
1503                                  Z_AttributesPlusTerm *zapt,
1504                                  const char *termz_org,
1505                                  oid_value attributeSet,
1506                                  NMEM stream,
1507                                  int reg_type, int complete_flag,
1508                                  const char *rank_type, int xpath_use,
1509                                  int num_bases, char **basenames,
1510                                  NMEM rset_nmem,
1511                                  RSET **result_sets, int *num_result_sets)
1512 {
1513     char term_dst[IT_MAX_WORD+1];
1514     struct grep_info grep_info;
1515     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1516     const char *termp = termz;
1517     int alloc_sets = 0;
1518
1519     *num_result_sets = 0;
1520     *term_dst = 0;
1521     if (grep_info_prepare(zh, zapt, &grep_info, reg_type, stream))
1522         return ZEBRA_FAIL;
1523     while(1)
1524     {
1525         ZEBRA_RES res;
1526
1527         if (alloc_sets == *num_result_sets)
1528         {
1529             int add = 10;
1530             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1531                                               sizeof(*rnew));
1532             if (alloc_sets)
1533                 memcpy(rnew, *result_sets, alloc_sets * sizeof(*rnew));
1534             alloc_sets = alloc_sets + add;
1535             *result_sets = rnew;
1536         }
1537         res = term_trunc(zh, zapt, &termp, attributeSet,
1538                          stream, &grep_info,
1539                          reg_type, complete_flag,
1540                          num_bases, basenames,
1541                          term_dst, rank_type,
1542                          xpath_use, rset_nmem,
1543                          &(*result_sets)[*num_result_sets]);
1544         if (res != ZEBRA_OK)
1545         {
1546             int i;
1547             for (i = 0; i < *num_result_sets; i++)
1548                 rset_delete((*result_sets)[i]);
1549             grep_info_delete (&grep_info);
1550             return res;
1551         }
1552         if ((*result_sets)[*num_result_sets] == 0)
1553             break;
1554         (*num_result_sets)++;
1555     }
1556     grep_info_delete(&grep_info);
1557     return ZEBRA_OK;
1558 }
1559
1560 static ZEBRA_RES rpn_search_APT_phrase(ZebraHandle zh,
1561                                        Z_AttributesPlusTerm *zapt,
1562                                        const char *termz_org,
1563                                        oid_value attributeSet,
1564                                        NMEM stream,
1565                                        int reg_type, int complete_flag,
1566                                        const char *rank_type, int xpath_use,
1567                                        int num_bases, char **basenames,
1568                                        NMEM rset_nmem,
1569                                        RSET *rset)
1570 {
1571     RSET *result_sets = 0;
1572     int num_result_sets = 0;
1573     ZEBRA_RES res =
1574         term_list_trunc(zh, zapt, termz_org, attributeSet,
1575                         stream, reg_type, complete_flag,
1576                         rank_type, xpath_use,
1577                         num_bases, basenames,
1578                         rset_nmem,
1579                         &result_sets, &num_result_sets);
1580     if (res != ZEBRA_OK)
1581         return res;
1582     if (num_result_sets == 0)
1583         *rset = rsnull_create (rset_nmem, key_it_ctrl);
1584     else if (num_result_sets == 1)
1585         *rset = result_sets[0];
1586     else
1587         *rset = rsprox_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
1588                               num_result_sets, result_sets,
1589                               1 /* ordered */, 0 /* exclusion */,
1590                               3 /* relation */, 1 /* distance */);
1591     if (!*rset)
1592         return ZEBRA_FAIL;
1593     return ZEBRA_OK;
1594 }
1595
1596 static ZEBRA_RES rpn_search_APT_or_list(ZebraHandle zh,
1597                                         Z_AttributesPlusTerm *zapt,
1598                                         const char *termz_org,
1599                                         oid_value attributeSet,
1600                                         NMEM stream,
1601                                         int reg_type, int complete_flag,
1602                                         const char *rank_type,
1603                                         int xpath_use,
1604                                         int num_bases, char **basenames,
1605                                         NMEM rset_nmem,
1606                                         RSET *rset)
1607 {
1608     RSET *result_sets = 0;
1609     int num_result_sets = 0;
1610     ZEBRA_RES res =
1611         term_list_trunc(zh, zapt, termz_org, attributeSet,
1612                         stream, reg_type, complete_flag,
1613                         rank_type, xpath_use,
1614                         num_bases, basenames,
1615                         rset_nmem,
1616                         &result_sets, &num_result_sets);
1617     if (res != ZEBRA_OK)
1618         return res;
1619     if (num_result_sets == 0)
1620         *rset = rsnull_create (rset_nmem, key_it_ctrl);
1621     else if (num_result_sets == 1)
1622         *rset = result_sets[0];
1623     else
1624         *rset = rsmulti_or_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
1625                                   num_result_sets, result_sets);
1626     if (!*rset)
1627         return ZEBRA_FAIL;
1628     return ZEBRA_OK;
1629 }
1630
1631 static ZEBRA_RES rpn_search_APT_and_list(ZebraHandle zh,
1632                                          Z_AttributesPlusTerm *zapt,
1633                                          const char *termz_org,
1634                                          oid_value attributeSet,
1635                                          NMEM stream,
1636                                          int reg_type, int complete_flag,
1637                                          const char *rank_type,
1638                                          int xpath_use,
1639                                          int num_bases, char **basenames,
1640                                          NMEM rset_nmem,
1641                                          RSET *rset)
1642 {
1643     RSET *result_sets = 0;
1644     int num_result_sets = 0;
1645     ZEBRA_RES res =
1646         term_list_trunc(zh, zapt, termz_org, attributeSet,
1647                         stream, reg_type, complete_flag,
1648                         rank_type, xpath_use,
1649                         num_bases, basenames,
1650                         rset_nmem,
1651                         &result_sets, &num_result_sets);
1652     if (res != ZEBRA_OK)
1653         return res;
1654     if (num_result_sets == 0)
1655         *rset = rsnull_create (rset_nmem, key_it_ctrl);
1656     else if (num_result_sets == 1)
1657         *rset = result_sets[0];
1658     else
1659         *rset = rsmulti_and_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
1660                                    num_result_sets, result_sets);
1661     if (!*rset)
1662         return ZEBRA_FAIL;
1663     return ZEBRA_OK;
1664 }
1665
1666 static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1667                             const char **term_sub,
1668                             char *term_dict,
1669                             oid_value attributeSet,
1670                             struct grep_info *grep_info,
1671                             int *max_pos,
1672                             int reg_type,
1673                             char *term_dst,
1674                             int *error_code)
1675 {
1676     AttrType relation;
1677     int relation_value;
1678     int term_value;
1679     int r;
1680     char *term_tmp = term_dict + strlen(term_dict);
1681
1682     *error_code = 0;
1683     attr_init(&relation, zapt, 2);
1684     relation_value = attr_find(&relation, NULL);
1685
1686     yaz_log(log_level_rpn, "numeric relation value=%d", relation_value);
1687
1688     if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1689                   term_dst))
1690         return 0;
1691     term_value = atoi (term_tmp);
1692     switch (relation_value)
1693     {
1694     case 1:
1695         yaz_log(log_level_rpn, "Relation <");
1696         gen_regular_rel(term_tmp, term_value-1, 1);
1697         break;
1698     case 2:
1699         yaz_log(log_level_rpn, "Relation <=");
1700         gen_regular_rel(term_tmp, term_value, 1);
1701         break;
1702     case 4:
1703         yaz_log(log_level_rpn, "Relation >=");
1704         gen_regular_rel(term_tmp, term_value, 0);
1705         break;
1706     case 5:
1707         yaz_log(log_level_rpn, "Relation >");
1708         gen_regular_rel(term_tmp, term_value+1, 0);
1709         break;
1710     case -1:
1711     case 3:
1712         yaz_log(log_level_rpn, "Relation =");
1713         sprintf(term_tmp, "(0*%d)", term_value);
1714         break;
1715     default:
1716         *error_code = 117;
1717         return 0;
1718     }
1719     yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_tmp);
1720     r = dict_lookup_grep(zh->reg->dict, term_dict, 0, grep_info, max_pos,
1721                           0, grep_handle);
1722     if (r)
1723         yaz_log(YLOG_WARN, "dict_lookup_grep fail, rel = gt: %d", r);
1724     yaz_log(log_level_rpn, "%d positions", grep_info->isam_p_indx);
1725     return 1;
1726 }
1727
1728 static ZEBRA_RES numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1729                               const char **term_sub,
1730                               oid_value attributeSet,
1731                               struct grep_info *grep_info,
1732                               int reg_type, int complete_flag,
1733                               int num_bases, char **basenames,
1734                               char *term_dst, int xpath_use, NMEM stream)
1735 {
1736     char term_dict[2*IT_MAX_WORD+2];
1737     int r, base_no;
1738     AttrType use;
1739     int use_value;
1740     const char *use_string = 0;
1741     oid_value curAttributeSet = attributeSet;
1742     const char *termp;
1743     struct rpn_char_map_info rcmi;
1744
1745     int bases_ok = 0;     /* no of databases with OK attribute */
1746     int errCode = 0;      /* err code (if any is not OK) */
1747     char *errString = 0;  /* addinfo */
1748
1749     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1750     attr_init(&use, zapt, 1);
1751     use_value = attr_find_ex(&use, &curAttributeSet, &use_string);
1752
1753     if (use_value == -1)
1754         use_value = 1016;
1755
1756     for (base_no = 0; base_no < num_bases; base_no++)
1757     {
1758         attent attp;
1759         data1_local_attribute id_xpath_attr;
1760         data1_local_attribute *local_attr;
1761         int max_pos, prefix_len = 0;
1762         int relation_error = 0;
1763
1764         termp = *term_sub;
1765         if (use_value == -2)  /* string attribute (assume IDXPATH/any) */
1766         {
1767             use_value = xpath_use;
1768             attp.local_attributes = &id_xpath_attr;
1769             attp.attset_ordinal = VAL_IDXPATH;
1770             id_xpath_attr.next = 0;
1771             id_xpath_attr.local = use_value;
1772         }
1773         else if (curAttributeSet == VAL_IDXPATH)
1774         {
1775             attp.local_attributes = &id_xpath_attr;
1776             attp.attset_ordinal = VAL_IDXPATH;
1777             id_xpath_attr.next = 0;
1778             id_xpath_attr.local = use_value;
1779         }
1780         else
1781         {
1782             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1783                                             use_string)))
1784             {
1785                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1786                       curAttributeSet, use_value, r);
1787                 if (r == -1)
1788                 {
1789                     errCode = 114;
1790                     if (use_string)
1791                         errString = nmem_strdup(stream, use_string);
1792                     else
1793                         errString = nmem_strdup_i (stream, use_value);
1794                 }
1795                 else
1796                     errCode = 121;
1797                 continue;
1798             }
1799         }
1800         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1801         {
1802             zh->errCode = 109; /* Database unavailable */
1803             zh->errString = basenames[base_no];
1804             return -1;
1805         }
1806         for (local_attr = attp.local_attributes; local_attr;
1807              local_attr = local_attr->next)
1808         {
1809             int ord;
1810             char ord_buf[32];
1811             int i, ord_len;
1812
1813             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1814                                               attp.attset_ordinal,
1815                                               local_attr->local);
1816             if (ord < 0)
1817                 continue;
1818             if (prefix_len)
1819                 term_dict[prefix_len++] = '|';
1820             else
1821                 term_dict[prefix_len++] = '(';
1822
1823             ord_len = key_SU_encode (ord, ord_buf);
1824             for (i = 0; i<ord_len; i++)
1825             {
1826                 term_dict[prefix_len++] = 1;
1827                 term_dict[prefix_len++] = ord_buf[i];
1828             }
1829         }
1830         if (!prefix_len)
1831         {
1832             errCode = 114;
1833             errString = nmem_strdup_i(stream, use_value);
1834             continue;
1835         }
1836         bases_ok++;
1837         term_dict[prefix_len++] = ')';
1838         term_dict[prefix_len++] = 1;
1839         term_dict[prefix_len++] = reg_type;
1840         yaz_log(YLOG_DEBUG, "reg_type = %d", term_dict[prefix_len-1]);
1841         term_dict[prefix_len] = '\0';
1842         if (!numeric_relation(zh, zapt, &termp, term_dict,
1843                               attributeSet, grep_info, &max_pos, reg_type,
1844                               term_dst, &relation_error))
1845         {
1846             if (relation_error)
1847             {
1848                 zh->errCode = relation_error;
1849                 zh->errString = 0;
1850                 return ZEBRA_FAIL;
1851             }
1852             *term_sub = 0;
1853             return ZEBRA_OK;
1854         }
1855     }
1856     if (!bases_ok)
1857     {
1858         zh->errCode = errCode;
1859         zh->errString = errString;
1860         return ZEBRA_FAIL;
1861     }
1862     *term_sub = termp;
1863     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1864     return ZEBRA_OK;
1865 }
1866
1867 static ZEBRA_RES rpn_search_APT_numeric(ZebraHandle zh,
1868                                         Z_AttributesPlusTerm *zapt,
1869                                         const char *termz,
1870                                         oid_value attributeSet,
1871                                         NMEM stream,
1872                                         int reg_type, int complete_flag,
1873                                         const char *rank_type, int xpath_use,
1874                                         int num_bases, char **basenames,
1875                                         NMEM rset_nmem,
1876                                         RSET *rset)
1877 {
1878     char term_dst[IT_MAX_WORD+1];
1879     const char *termp = termz;
1880     RSET *result_sets = 0;
1881     int num_result_sets = 0;
1882     ZEBRA_RES res;
1883     struct grep_info grep_info;
1884     int alloc_sets = 0;
1885
1886     yaz_log(log_level_rpn, "APT_numeric t='%s'", termz);
1887     if (grep_info_prepare(zh, zapt, &grep_info, reg_type, stream))
1888         return ZEBRA_FAIL;
1889     while (1)
1890     {
1891         if (alloc_sets == num_result_sets)
1892         {
1893             int add = 10;
1894             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1895                                               sizeof(*rnew));
1896             if (alloc_sets)
1897                 memcpy(rnew, result_sets, alloc_sets * sizeof(*rnew));
1898             alloc_sets = alloc_sets + add;
1899             result_sets = rnew;
1900         }
1901         yaz_log(YLOG_DEBUG, "APT_numeric termp=%s", termp);
1902         grep_info.isam_p_indx = 0;
1903         res = numeric_term(zh, zapt, &termp, attributeSet, &grep_info,
1904                            reg_type, complete_flag, num_bases, basenames,
1905                            term_dst, xpath_use,
1906                            stream);
1907         if (res == ZEBRA_FAIL || termp == 0)
1908             break;
1909         yaz_log(YLOG_DEBUG, "term: %s", term_dst);
1910         result_sets[num_result_sets] =
1911             rset_trunc(zh, grep_info.isam_p_buf,
1912                        grep_info.isam_p_indx, term_dst,
1913                        strlen(term_dst), rank_type,
1914                        0 /* preserve position */,
1915                        zapt->term->which, rset_nmem,
1916                        key_it_ctrl,key_it_ctrl->scope);
1917         if (!result_sets[num_result_sets])
1918             break;
1919         num_result_sets++;
1920     }
1921     grep_info_delete(&grep_info);
1922     if (termp)
1923     {
1924         int i;
1925         for (i = 0; i<num_result_sets; i++)
1926             rset_delete(result_sets[i]);
1927         return ZEBRA_FAIL;
1928     }
1929     if (num_result_sets == 0)
1930         *rset = rsnull_create(rset_nmem, key_it_ctrl);
1931     if (num_result_sets == 1)
1932         *rset = result_sets[0];
1933     else
1934         *rset = rsmulti_and_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
1935                                    num_result_sets, result_sets);
1936     if (!*rset)
1937         return ZEBRA_FAIL;
1938     return ZEBRA_OK;
1939 }
1940
1941 static ZEBRA_RES rpn_search_APT_local(ZebraHandle zh,
1942                                       Z_AttributesPlusTerm *zapt,
1943                                       const char *termz,
1944                                       oid_value attributeSet,
1945                                       NMEM stream,
1946                                       const char *rank_type, NMEM rset_nmem,
1947                                       RSET *rset)
1948 {
1949     RSFD rsfd;
1950     struct it_key key;
1951     int sys;
1952     *rset = rstemp_create(rset_nmem,key_it_ctrl,key_it_ctrl->scope,
1953                           res_get (zh->res, "setTmpDir"),0 );
1954     rsfd = rset_open(*rset, RSETF_WRITE);
1955
1956     sys = atoi(termz);
1957     if (sys <= 0)
1958         sys = 1;
1959     key.mem[0] = sys;
1960     key.mem[1] = 1;
1961     key.len = 2;
1962     rset_write (rsfd, &key);
1963     rset_close (rsfd);
1964     return ZEBRA_OK;
1965 }
1966
1967 static ZEBRA_RES rpn_sort_spec(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1968                                oid_value attributeSet, NMEM stream,
1969                                Z_SortKeySpecList *sort_sequence,
1970                                const char *rank_type,
1971                                RSET *rset)
1972 {
1973     int i;
1974     int sort_relation_value;
1975     AttrType sort_relation_type;
1976     int use_value;
1977     AttrType use_type;
1978     Z_SortKeySpec *sks;
1979     Z_SortKey *sk;
1980     Z_AttributeElement *ae;
1981     int oid[OID_SIZE];
1982     oident oe;
1983     char termz[20];
1984
1985     attr_init(&sort_relation_type, zapt, 7);
1986     sort_relation_value = attr_find(&sort_relation_type, &attributeSet);
1987
1988     attr_init(&use_type, zapt, 1);
1989     use_value = attr_find(&use_type, &attributeSet);
1990
1991     if (!sort_sequence->specs)
1992     {
1993         sort_sequence->num_specs = 10;
1994         sort_sequence->specs = (Z_SortKeySpec **)
1995             nmem_malloc(stream, sort_sequence->num_specs *
1996                          sizeof(*sort_sequence->specs));
1997         for (i = 0; i<sort_sequence->num_specs; i++)
1998             sort_sequence->specs[i] = 0;
1999     }
2000     if (zapt->term->which != Z_Term_general)
2001         i = 0;
2002     else
2003         i = atoi_n ((char *) zapt->term->u.general->buf,
2004                     zapt->term->u.general->len);
2005     if (i >= sort_sequence->num_specs)
2006         i = 0;
2007     sprintf(termz, "%d", i);
2008
2009     oe.proto = PROTO_Z3950;
2010     oe.oclass = CLASS_ATTSET;
2011     oe.value = attributeSet;
2012     if (!oid_ent_to_oid (&oe, oid))
2013         return ZEBRA_FAIL;
2014
2015     sks = (Z_SortKeySpec *) nmem_malloc(stream, sizeof(*sks));
2016     sks->sortElement = (Z_SortElement *)
2017         nmem_malloc(stream, sizeof(*sks->sortElement));
2018     sks->sortElement->which = Z_SortElement_generic;
2019     sk = sks->sortElement->u.generic = (Z_SortKey *)
2020         nmem_malloc(stream, sizeof(*sk));
2021     sk->which = Z_SortKey_sortAttributes;
2022     sk->u.sortAttributes = (Z_SortAttributes *)
2023         nmem_malloc(stream, sizeof(*sk->u.sortAttributes));
2024
2025     sk->u.sortAttributes->id = oid;
2026     sk->u.sortAttributes->list = (Z_AttributeList *)
2027         nmem_malloc(stream, sizeof(*sk->u.sortAttributes->list));
2028     sk->u.sortAttributes->list->num_attributes = 1;
2029     sk->u.sortAttributes->list->attributes = (Z_AttributeElement **)
2030         nmem_malloc(stream, sizeof(*sk->u.sortAttributes->list->attributes));
2031     ae = *sk->u.sortAttributes->list->attributes = (Z_AttributeElement *)
2032         nmem_malloc(stream, sizeof(**sk->u.sortAttributes->list->attributes));
2033     ae->attributeSet = 0;
2034     ae->attributeType = (int *)
2035         nmem_malloc(stream, sizeof(*ae->attributeType));
2036     *ae->attributeType = 1;
2037     ae->which = Z_AttributeValue_numeric;
2038     ae->value.numeric = (int *)
2039         nmem_malloc(stream, sizeof(*ae->value.numeric));
2040     *ae->value.numeric = use_value;
2041
2042     sks->sortRelation = (int *)
2043         nmem_malloc(stream, sizeof(*sks->sortRelation));
2044     if (sort_relation_value == 1)
2045         *sks->sortRelation = Z_SortKeySpec_ascending;
2046     else if (sort_relation_value == 2)
2047         *sks->sortRelation = Z_SortKeySpec_descending;
2048     else
2049         *sks->sortRelation = Z_SortKeySpec_ascending;
2050
2051     sks->caseSensitivity = (int *)
2052         nmem_malloc(stream, sizeof(*sks->caseSensitivity));
2053     *sks->caseSensitivity = 0;
2054
2055     sks->which = Z_SortKeySpec_null;
2056     sks->u.null = odr_nullval ();
2057     sort_sequence->specs[i] = sks;
2058     *rset = rsnull_create (NULL, key_it_ctrl);
2059     return ZEBRA_OK;
2060 }
2061
2062
2063 static int parse_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2064                        oid_value attributeSet,
2065                        struct xpath_location_step *xpath, int max, NMEM mem)
2066 {
2067     oid_value curAttributeSet = attributeSet;
2068     AttrType use;
2069     const char *use_string = 0;
2070
2071     attr_init(&use, zapt, 1);
2072     attr_find_ex(&use, &curAttributeSet, &use_string);
2073
2074     if (!use_string || *use_string != '/')
2075         return -1;
2076
2077     return zebra_parse_xpath_str(use_string, xpath, max, mem);
2078 }
2079
2080
2081
2082 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
2083                         int reg_type, const char *term, int use,
2084                         oid_value curAttributeSet, NMEM rset_nmem)
2085 {
2086     RSET rset;
2087     struct grep_info grep_info;
2088     char term_dict[2048];
2089     char ord_buf[32];
2090     int prefix_len = 0;
2091     int ord = zebraExplain_lookup_attr_su(zh->reg->zei, curAttributeSet, use);
2092     int ord_len, i, r, max_pos;
2093     int term_type = Z_Term_characterString;
2094     const char *flags = "void";
2095
2096     if (grep_info_prepare(zh, 0 /* zapt */, &grep_info, '0', stream))
2097         return rsnull_create (rset_nmem,key_it_ctrl);
2098
2099     if (ord < 0)
2100         return rsnull_create (rset_nmem,key_it_ctrl);
2101     if (prefix_len)
2102         term_dict[prefix_len++] = '|';
2103     else
2104         term_dict[prefix_len++] = '(';
2105
2106     ord_len = key_SU_encode (ord, ord_buf);
2107     for (i = 0; i<ord_len; i++)
2108     {
2109         term_dict[prefix_len++] = 1;
2110         term_dict[prefix_len++] = ord_buf[i];
2111     }
2112     term_dict[prefix_len++] = ')';
2113     term_dict[prefix_len++] = 1;
2114     term_dict[prefix_len++] = reg_type;
2115
2116     strcpy(term_dict+prefix_len, term);
2117
2118     grep_info.isam_p_indx = 0;
2119     r = dict_lookup_grep(zh->reg->dict, term_dict, 0,
2120                           &grep_info, &max_pos, 0, grep_handle);
2121     yaz_log(YLOG_DEBUG, "%s %d positions", term,
2122              grep_info.isam_p_indx);
2123     rset = rset_trunc(zh, grep_info.isam_p_buf,
2124                       grep_info.isam_p_indx, term, strlen(term),
2125                       flags, 1, term_type,rset_nmem,
2126                       key_it_ctrl, key_it_ctrl->scope);
2127     grep_info_delete(&grep_info);
2128     return rset;
2129 }
2130
2131 static RSET rpn_search_xpath(ZebraHandle zh,
2132                              oid_value attributeSet,
2133                              int num_bases, char **basenames,
2134                              NMEM stream, const char *rank_type, RSET rset,
2135                              int xpath_len, struct xpath_location_step *xpath,
2136                              NMEM rset_nmem)
2137 {
2138     oid_value curAttributeSet = attributeSet;
2139     int base_no;
2140     int i;
2141
2142     if (xpath_len < 0)
2143         return rset;
2144
2145     yaz_log(YLOG_DEBUG, "xpath len=%d", xpath_len);
2146     for (i = 0; i<xpath_len; i++)
2147     {
2148         yaz_log(log_level_rpn, "XPATH %d %s", i, xpath[i].part);
2149
2150     }
2151
2152     curAttributeSet = VAL_IDXPATH;
2153
2154     /*
2155       //a    ->    a/.*
2156       //a/b  ->    b/a/.*
2157       /a     ->    a/
2158       /a/b   ->    b/a/
2159
2160       /      ->    none
2161
2162    a[@attr = value]/b[@other = othervalue]
2163
2164  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
2165  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
2166  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
2167  /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y)
2168  /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y)
2169  /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x)
2170
2171     */
2172
2173     dict_grep_cmap (zh->reg->dict, 0, 0);
2174
2175     for (base_no = 0; base_no < num_bases; base_no++)
2176     {
2177         int level = xpath_len;
2178         int first_path = 1;
2179
2180         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2181         {
2182             zh->errCode = 109; /* Database unavailable */
2183             zh->errString = basenames[base_no];
2184             return rset;
2185         }
2186         while (--level >= 0)
2187         {
2188             char xpath_rev[128];
2189             int i, len;
2190             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
2191
2192             *xpath_rev = 0;
2193             len = 0;
2194             for (i = level; i >= 1; --i)
2195             {
2196                 const char *cp = xpath[i].part;
2197                 if (*cp)
2198                 {
2199                     for (;*cp; cp++)
2200                         if (*cp == '*')
2201                         {
2202                             memcpy (xpath_rev + len, "[^/]*", 5);
2203                             len += 5;
2204                         }
2205                         else if (*cp == ' ')
2206                         {
2207
2208                             xpath_rev[len++] = 1;
2209                             xpath_rev[len++] = ' ';
2210                         }
2211
2212                         else
2213                             xpath_rev[len++] = *cp;
2214                     xpath_rev[len++] = '/';
2215                 }
2216                 else if (i == 1)  /* // case */
2217                 {
2218                     xpath_rev[len++] = '.';
2219                     xpath_rev[len++] = '*';
2220                 }
2221             }
2222             xpath_rev[len] = 0;
2223
2224             if (xpath[level].predicate &&
2225                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
2226                 xpath[level].predicate->u.relation.name[0])
2227             {
2228                 WRBUF wbuf = wrbuf_alloc();
2229                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
2230                 if (xpath[level].predicate->u.relation.value)
2231                 {
2232                     const char *cp = xpath[level].predicate->u.relation.value;
2233                     wrbuf_putc(wbuf, '=');
2234
2235                     while (*cp)
2236                     {
2237                         if (strchr(REGEX_CHARS, *cp))
2238                             wrbuf_putc(wbuf, '\\');
2239                         wrbuf_putc(wbuf, *cp);
2240                         cp++;
2241                     }
2242                 }
2243                 wrbuf_puts(wbuf, "");
2244                 rset_attr = xpath_trunc(
2245                     zh, stream, '0', wrbuf_buf(wbuf), 3,
2246                     curAttributeSet,rset_nmem);
2247                 wrbuf_free(wbuf, 1);
2248             }
2249             else
2250             {
2251                 if (!first_path)
2252                     continue;
2253             }
2254             yaz_log(log_level_rpn, "xpath_rev (%d) = %s", level, xpath_rev);
2255             if (strlen(xpath_rev))
2256             {
2257                 rset_start_tag = xpath_trunc(zh, stream, '0',
2258                         xpath_rev, 1, curAttributeSet, rset_nmem);
2259
2260                 rset_end_tag = xpath_trunc(zh, stream, '0',
2261                         xpath_rev, 2, curAttributeSet, rset_nmem);
2262
2263                 rset = rsbetween_create(rset_nmem, key_it_ctrl,
2264                                         key_it_ctrl->scope,
2265                                         rset_start_tag, rset,
2266                                         rset_end_tag, rset_attr);
2267             }
2268             first_path = 0;
2269         }
2270     }
2271
2272     return rset;
2273 }
2274
2275
2276
2277 static ZEBRA_RES rpn_search_APT(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2278                                 oid_value attributeSet, NMEM stream,
2279                                 Z_SortKeySpecList *sort_sequence,
2280                                 int num_bases, char **basenames,
2281                                 NMEM rset_nmem,
2282                                 RSET *rset)
2283 {
2284     ZEBRA_RES res = ZEBRA_OK;
2285     unsigned reg_id;
2286     char *search_type = NULL;
2287     char rank_type[128];
2288     int complete_flag;
2289     int sort_flag;
2290     char termz[IT_MAX_WORD+1];
2291     int xpath_len;
2292     int xpath_use = 0;
2293     struct xpath_location_step xpath[10];
2294
2295     if (!log_level_set)
2296     {
2297         log_level_rpn = yaz_log_module_level("rpn");
2298         log_level_set = 1;
2299     }
2300     zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2301                     rank_type, &complete_flag, &sort_flag);
2302
2303     yaz_log(YLOG_DEBUG, "reg_id=%c", reg_id);
2304     yaz_log(YLOG_DEBUG, "complete_flag=%d", complete_flag);
2305     yaz_log(YLOG_DEBUG, "search_type=%s", search_type);
2306     yaz_log(YLOG_DEBUG, "rank_type=%s", rank_type);
2307
2308     if (zapt_term_to_utf8(zh, zapt, termz) == ZEBRA_FAIL)
2309         return ZEBRA_FAIL;
2310
2311     if (sort_flag)
2312         return rpn_sort_spec(zh, zapt, attributeSet, stream, sort_sequence,
2313                              rank_type, rset);
2314     xpath_len = parse_xpath(zh, zapt, attributeSet, xpath, 10, stream);
2315     if (xpath_len >= 0)
2316     {
2317         xpath_use = 1016;
2318         if (xpath[xpath_len-1].part[0] == '@')
2319             xpath_use = 1015;
2320     }
2321
2322     if (!strcmp(search_type, "phrase"))
2323     {
2324         res = rpn_search_APT_phrase(zh, zapt, termz, attributeSet, stream,
2325                                     reg_id, complete_flag, rank_type,
2326                                     xpath_use,
2327                                     num_bases, basenames, rset_nmem,
2328                                     rset);
2329     }
2330     else if (!strcmp(search_type, "and-list"))
2331     {
2332         res = rpn_search_APT_and_list(zh, zapt, termz, attributeSet, stream,
2333                                       reg_id, complete_flag, rank_type,
2334                                       xpath_use,
2335                                       num_bases, basenames, rset_nmem,
2336                                       rset);
2337     }
2338     else if (!strcmp(search_type, "or-list"))
2339     {
2340         res = rpn_search_APT_or_list(zh, zapt, termz, attributeSet, stream,
2341                                      reg_id, complete_flag, rank_type,
2342                                      xpath_use,
2343                                      num_bases, basenames, rset_nmem,
2344                                      rset);
2345     }
2346     else if (!strcmp(search_type, "local"))
2347     {
2348         res = rpn_search_APT_local(zh, zapt, termz, attributeSet, stream,
2349                                    rank_type, rset_nmem, rset);
2350     }
2351     else if (!strcmp(search_type, "numeric"))
2352     {
2353         res = rpn_search_APT_numeric(zh, zapt, termz, attributeSet, stream,
2354                                      reg_id, complete_flag, rank_type,
2355                                      xpath_use,
2356                                      num_bases, basenames, rset_nmem, rset);
2357     }
2358     else
2359     {
2360         zh->errCode = 118;
2361         return ZEBRA_FAIL;
2362     }
2363     if (res != ZEBRA_OK)
2364         return res;
2365     if (!*rset)
2366         return ZEBRA_FAIL;
2367     *rset = rpn_search_xpath(zh, attributeSet, num_bases, basenames,
2368                              stream, rank_type, *rset,
2369                              xpath_len, xpath, rset_nmem);
2370     if (!*rset)
2371         return ZEBRA_FAIL;
2372     return ZEBRA_OK;
2373 }
2374
2375 static ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2376                                       oid_value attributeSet,
2377                                       NMEM stream, NMEM rset_nmem,
2378                                       Z_SortKeySpecList *sort_sequence,
2379                                       int num_bases, char **basenames,
2380                                       RSET **result_sets, int *num_result_sets,
2381                                       Z_Operator *parent_op);
2382
2383 ZEBRA_RES rpn_search_top(ZebraHandle zh, Z_RPNStructure *zs,
2384                          oid_value attributeSet,
2385                          NMEM stream, NMEM rset_nmem,
2386                          Z_SortKeySpecList *sort_sequence,
2387                          int num_bases, char **basenames,
2388                          RSET *result_set)
2389 {
2390     RSET *result_sets = 0;
2391     int num_result_sets = 0;
2392     ZEBRA_RES res = rpn_search_structure(zh, zs, attributeSet,
2393                                          stream, rset_nmem,
2394                                          sort_sequence,
2395                                          num_bases, basenames,
2396                                          &result_sets, &num_result_sets,
2397                                          0 /* no op */);
2398     if (res != ZEBRA_OK)
2399     {
2400         int i;
2401         for (i = 0; i<num_result_sets; i++)
2402             rset_delete(result_sets[i]);
2403         *result_set = 0;
2404         return res;
2405     }
2406     assert(num_result_sets == 1);
2407     assert(result_sets);
2408     assert(*result_sets);
2409     *result_set = *result_sets;
2410     return ZEBRA_OK;
2411 }
2412
2413 ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2414                                oid_value attributeSet,
2415                                NMEM stream, NMEM rset_nmem,
2416                                Z_SortKeySpecList *sort_sequence,
2417                                int num_bases, char **basenames,
2418                                RSET **result_sets, int *num_result_sets,
2419                                Z_Operator *parent_op)
2420 {
2421     *num_result_sets = 0;
2422     if (zs->which == Z_RPNStructure_complex)
2423     {
2424         ZEBRA_RES res;
2425         Z_Operator *zop = zs->u.complex->roperator;
2426         RSET *result_sets_l = 0;
2427         int num_result_sets_l = 0;
2428         RSET *result_sets_r = 0;
2429         int num_result_sets_r = 0;
2430
2431         res = rpn_search_structure(zh, zs->u.complex->s1,
2432                                    attributeSet, stream, rset_nmem,
2433                                    sort_sequence,
2434                                    num_bases, basenames,
2435                                    &result_sets_l, &num_result_sets_l,
2436                                    zop);
2437         if (res != ZEBRA_OK)
2438         {
2439             int i;
2440             for (i = 0; i<num_result_sets_l; i++)
2441                 rset_delete(result_sets_l[i]);
2442             return res;
2443         }
2444         res = rpn_search_structure(zh, zs->u.complex->s2,
2445                                    attributeSet, stream, rset_nmem,
2446                                    sort_sequence,
2447                                    num_bases, basenames,
2448                                    &result_sets_r, &num_result_sets_r,
2449                                    zop);
2450         if (res != ZEBRA_OK)
2451         {
2452             int i;
2453             for (i = 0; i<num_result_sets_l; i++)
2454                 rset_delete(result_sets_l[i]);
2455             for (i = 0; i<num_result_sets_r; i++)
2456                 rset_delete(result_sets_r[i]);
2457             return res;
2458         }
2459
2460         /* make a new list of result for all children */
2461         *num_result_sets = num_result_sets_l + num_result_sets_r;
2462         *result_sets = nmem_malloc(stream, *num_result_sets *
2463                                    sizeof(**result_sets));
2464         memcpy(*result_sets, result_sets_l,
2465                num_result_sets_l * sizeof(**result_sets));
2466         memcpy(*result_sets + num_result_sets_l, result_sets_r,
2467                num_result_sets_r * sizeof(**result_sets));
2468
2469         if (!parent_op || parent_op->which != zop->which
2470             || (zop->which != Z_Operator_and &&
2471                 zop->which != Z_Operator_or))
2472         {
2473             /* parent node different from this one (or non-present) */
2474             /* we must combine result sets now */
2475             RSET rset;
2476             switch (zop->which)
2477             {
2478             case Z_Operator_and:
2479                 rset = rsmulti_and_create(rset_nmem, key_it_ctrl,
2480                                           key_it_ctrl->scope,
2481                                           *num_result_sets, *result_sets);
2482                 break;
2483             case Z_Operator_or:
2484                 rset = rsmulti_or_create(rset_nmem, key_it_ctrl,
2485                                          key_it_ctrl->scope,
2486                                          *num_result_sets, *result_sets);
2487                 break;
2488             case Z_Operator_and_not:
2489                 rset = rsbool_create_not(rset_nmem, key_it_ctrl,
2490                                          key_it_ctrl->scope,
2491                                          (*result_sets)[0],
2492                                          (*result_sets)[1]);
2493                 break;
2494             case Z_Operator_prox:
2495                 if (zop->u.prox->which != Z_ProximityOperator_known)
2496                 {
2497                     zh->errCode = 132;
2498                     return ZEBRA_FAIL;
2499                 }
2500                 if (*zop->u.prox->u.known != Z_ProxUnit_word)
2501                 {
2502                     char *val = (char *) nmem_malloc(stream, 16);
2503                     zh->errCode = 132;
2504                     zh->errString = val;
2505                     sprintf(val, "%d", *zop->u.prox->u.known);
2506                     return ZEBRA_FAIL;
2507                 }
2508                 else
2509                 {
2510                     rset = rsprox_create(rset_nmem, key_it_ctrl,
2511                                          key_it_ctrl->scope,
2512                                          *num_result_sets, *result_sets,
2513                                          *zop->u.prox->ordered,
2514                                          (!zop->u.prox->exclusion ?
2515                                           0 : *zop->u.prox->exclusion),
2516                                          *zop->u.prox->relationType,
2517                                          *zop->u.prox->distance );
2518                 }
2519                 break;
2520             default:
2521                 zh->errCode = 110;
2522                 return ZEBRA_FAIL;
2523             }
2524             *num_result_sets = 1;
2525             *result_sets = nmem_malloc(stream, *num_result_sets *
2526                                        sizeof(**result_sets));
2527             (*result_sets)[0] = rset;
2528         }
2529     }
2530     else if (zs->which == Z_RPNStructure_simple)
2531     {
2532         RSET rset;
2533         ZEBRA_RES res;
2534
2535         if (zs->u.simple->which == Z_Operand_APT)
2536         {
2537             yaz_log(YLOG_DEBUG, "rpn_search_APT");
2538             res = rpn_search_APT(zh, zs->u.simple->u.attributesPlusTerm,
2539                                  attributeSet, stream, sort_sequence,
2540                                  num_bases, basenames, rset_nmem, &rset);
2541             if (res != ZEBRA_OK)
2542                 return res;
2543         }
2544         else if (zs->u.simple->which == Z_Operand_resultSetId)
2545         {
2546             yaz_log(YLOG_DEBUG, "rpn_search_ref");
2547             rset = resultSetRef(zh, zs->u.simple->u.resultSetId);
2548             if (!rset)
2549             {
2550                 zh->errCode = 30;
2551                 zh->errString =
2552                     nmem_strdup(stream, zs->u.simple->u.resultSetId);
2553                 return ZEBRA_FAIL;
2554             }
2555             rset_dup(rset);
2556         }
2557         else
2558         {
2559             zh->errCode = 3;
2560             return ZEBRA_FAIL;
2561         }
2562         *num_result_sets = 1;
2563         *result_sets = nmem_malloc(stream, *num_result_sets *
2564                                    sizeof(**result_sets));
2565         (*result_sets)[0] = rset;
2566     }
2567     else
2568     {
2569         zh->errCode = 3;
2570         return ZEBRA_FAIL;
2571     }
2572     return ZEBRA_OK;
2573 }
2574
2575 struct scan_info_entry {
2576     char *term;
2577     ISAM_P isam_p;
2578 };
2579
2580 struct scan_info {
2581     struct scan_info_entry *list;
2582     ODR odr;
2583     int before, after;
2584     char prefix[20];
2585 };
2586
2587 static int scan_handle (char *name, const char *info, int pos, void *client)
2588 {
2589     int len_prefix, idx;
2590     struct scan_info *scan_info = (struct scan_info *) client;
2591
2592     len_prefix = strlen(scan_info->prefix);
2593     if (memcmp (name, scan_info->prefix, len_prefix))
2594         return 1;
2595     if (pos > 0)        idx = scan_info->after - pos + scan_info->before;
2596     else
2597         idx = - pos - 1;
2598     scan_info->list[idx].term = (char *)
2599         odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
2600     strcpy(scan_info->list[idx].term, name + len_prefix);
2601     assert (*info == sizeof(ISAM_P));
2602     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
2603     return 0;
2604 }
2605
2606 static void scan_term_untrans (ZebraHandle zh, NMEM stream, int reg_type,
2607                                char **dst, const char *src)
2608 {
2609     char term_src[IT_MAX_WORD];
2610     char term_dst[IT_MAX_WORD];
2611
2612     term_untrans (zh, reg_type, term_src, src);
2613
2614     if (zh->iconv_from_utf8 != 0)
2615     {
2616         int len;
2617         char *inbuf = term_src;
2618         size_t inleft = strlen(term_src);
2619         char *outbuf = term_dst;
2620         size_t outleft = sizeof(term_dst)-1;
2621         size_t ret;
2622
2623         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2624                          &outbuf, &outleft);
2625         if (ret == (size_t)(-1))
2626             len = 0;
2627         else
2628             len = outbuf - term_dst;
2629         *dst = nmem_malloc(stream, len + 1);
2630         if (len > 0)
2631             memcpy (*dst, term_dst, len);
2632         (*dst)[len] = '\0';
2633     }
2634     else
2635         *dst = nmem_strdup(stream, term_src);
2636 }
2637
2638 static void count_set (RSET r, int *count)
2639 {
2640     zint psysno = 0;
2641     int kno = 0;
2642     struct it_key key;
2643     RSFD rfd;
2644
2645     yaz_log(YLOG_DEBUG, "count_set");
2646
2647     *count = 0;
2648     rfd = rset_open (r, RSETF_READ);
2649     while (rset_read (rfd, &key,0 /* never mind terms */))
2650     {
2651         if (key.mem[0] != psysno)
2652         {
2653             psysno = key.mem[0];
2654             (*count)++;
2655         }
2656         kno++;
2657     }
2658     rset_close (rfd);
2659     yaz_log(YLOG_DEBUG, "%d keys, %d records", kno, *count);
2660 }
2661
2662 ZEBRA_RES rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2663                    oid_value attributeset,
2664                    int num_bases, char **basenames,
2665                    int *position, int *num_entries, ZebraScanEntry **list,
2666                    int *is_partial, RSET limit_set, int return_zero)
2667 {
2668     int i;
2669     int pos = *position;
2670     int num = *num_entries;
2671     int before;
2672     int after;
2673     int base_no;
2674     char termz[IT_MAX_WORD+20];
2675     AttrType use;
2676     int use_value;
2677     const char *use_string = 0;
2678     struct scan_info *scan_info_array;
2679     ZebraScanEntry *glist;
2680     int ords[32], ord_no = 0;
2681     int ptr[32];
2682
2683     int bases_ok = 0;     /* no of databases with OK attribute */
2684     int errCode = 0;      /* err code (if any is not OK) */
2685     char *errString = 0;  /* addinfo */
2686
2687     unsigned reg_id;
2688     char *search_type = NULL;
2689     char rank_type[128];
2690     int complete_flag;
2691     int sort_flag;
2692     NMEM rset_nmem = NULL;
2693
2694     *list = 0;
2695     *is_partial = 0;
2696
2697     if (attributeset == VAL_NONE)
2698         attributeset = VAL_BIB1;
2699
2700     if (!limit_set)
2701     {
2702         AttrType termset;
2703         int termset_value_numeric;
2704         const char *termset_value_string;
2705         attr_init(&termset, zapt, 8);
2706         termset_value_numeric =
2707             attr_find_ex(&termset, NULL, &termset_value_string);
2708         if (termset_value_numeric != -1)
2709         {
2710             char resname[32];
2711             const char *termset_name = 0;
2712
2713             if (termset_value_numeric != -2)
2714             {
2715
2716                 sprintf(resname, "%d", termset_value_numeric);
2717                 termset_name = resname;
2718             }
2719             else
2720                 termset_name = termset_value_string;
2721
2722             limit_set = resultSetRef (zh, termset_name);
2723         }
2724     }
2725
2726     yaz_log(YLOG_DEBUG, "position = %d, num = %d set=%d",
2727             pos, num, attributeset);
2728
2729     attr_init(&use, zapt, 1);
2730     use_value = attr_find_ex(&use, &attributeset, &use_string);
2731
2732     if (zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2733                         rank_type, &complete_flag, &sort_flag))
2734     {
2735         *num_entries = 0;
2736         zh->errCode = 113;
2737         return ZEBRA_FAIL;
2738     }
2739     yaz_log(YLOG_DEBUG, "use_value = %d", use_value);
2740
2741     if (use_value == -1)
2742         use_value = 1016;
2743     for (base_no = 0; base_no < num_bases && ord_no < 32; base_no++)
2744     {
2745         data1_local_attribute *local_attr;
2746         attent attp;
2747         int ord;
2748
2749         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2750         {
2751             zh->errString = basenames[base_no];
2752             zh->errCode = 109; /* Database unavailable */
2753             *num_entries = 0;
2754             return ZEBRA_FAIL;
2755         }
2756
2757         if (use_string &&
2758             (ord = zebraExplain_lookup_attr_str(zh->reg->zei,
2759                                                 use_string)) >= 0)
2760         {
2761             /* we have a match for a raw string attribute */
2762             if (ord > 0)
2763                 ords[ord_no++] = ord;
2764             attp.local_attributes = 0;  /* no more attributes */
2765         }
2766         else
2767         {
2768             int r;
2769
2770             if ((r = att_getentbyatt (zh, &attp, attributeset, use_value,
2771                                       use_string)))
2772             {
2773                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
2774                         attributeset, use_value);
2775                 if (r == -1)
2776                 {
2777                     errCode = 114;
2778                     if (use_string)
2779                         errString = odr_strdup(stream, use_string);
2780                     else
2781                     {
2782                         char val_str[32];
2783                         sprintf(val_str, "%d", use_value);
2784                         errString = odr_strdup(stream, val_str);
2785                     }
2786                 }
2787                 else
2788                     errCode = 121;
2789                 continue;
2790             }
2791         }
2792         bases_ok++;
2793         for (local_attr = attp.local_attributes; local_attr && ord_no < 32;
2794              local_attr = local_attr->next)
2795         {
2796             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
2797                                               attp.attset_ordinal,
2798                                               local_attr->local);
2799             if (ord > 0)
2800                 ords[ord_no++] = ord;
2801         }
2802     }
2803     if (!bases_ok && errCode)
2804     {
2805         zh->errCode = errCode;
2806         zh->errString = errString;
2807         *num_entries = 0;
2808         return ZEBRA_FAIL;
2809     }
2810     if (ord_no == 0)
2811     {
2812         *num_entries = 0;
2813         return ZEBRA_OK;
2814     }
2815     /* prepare dictionary scanning */
2816     before = pos-1;
2817     after = 1+num-pos;
2818     scan_info_array = (struct scan_info *)
2819         odr_malloc(stream, ord_no * sizeof(*scan_info_array));
2820     for (i = 0; i < ord_no; i++)
2821     {
2822         int j, prefix_len = 0;
2823         int before_tmp = before, after_tmp = after;
2824         struct scan_info *scan_info = scan_info_array + i;
2825         struct rpn_char_map_info rcmi;
2826
2827         rpn_char_map_prepare (zh->reg, reg_id, &rcmi);
2828
2829         scan_info->before = before;
2830         scan_info->after = after;
2831         scan_info->odr = stream;
2832
2833         scan_info->list = (struct scan_info_entry *)
2834             odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
2835         for (j = 0; j<before+after; j++)
2836             scan_info->list[j].term = NULL;
2837
2838         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2839         termz[prefix_len++] = reg_id;
2840         termz[prefix_len] = 0;
2841         strcpy(scan_info->prefix, termz);
2842
2843         if (trans_scan_term(zh, zapt, termz+prefix_len, reg_id) == ZEBRA_FAIL)
2844             return ZEBRA_FAIL;
2845
2846         dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
2847                   scan_info, scan_handle);
2848     }
2849     glist = (ZebraScanEntry *)
2850         odr_malloc(stream, (before+after)*sizeof(*glist));
2851
2852     rset_nmem = nmem_create();
2853
2854     /* consider terms after main term */
2855     for (i = 0; i < ord_no; i++)
2856         ptr[i] = before;
2857
2858     *is_partial = 0;
2859     for (i = 0; i<after; i++)
2860     {
2861         int j, j0 = -1;
2862         const char *mterm = NULL;
2863         const char *tst;
2864         RSET rset;
2865
2866         for (j = 0; j < ord_no; j++)
2867         {
2868             if (ptr[j] < before+after &&
2869                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2870                 (!mterm || strcmp (tst, mterm) < 0))
2871             {
2872                 j0 = j;
2873                 mterm = tst;
2874             }
2875         }
2876         if (j0 == -1)
2877             break;
2878         scan_term_untrans(zh, stream->mem, reg_id,
2879                           &glist[i+before].term, mterm);
2880         rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2881                           glist[i+before].term, strlen(glist[i+before].term),
2882                           NULL, 0, zapt->term->which, rset_nmem,
2883                           key_it_ctrl,key_it_ctrl->scope);
2884         ptr[j0]++;
2885         for (j = j0+1; j<ord_no; j++)
2886         {
2887             if (ptr[j] < before+after &&
2888                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2889                 !strcmp (tst, mterm))
2890             {
2891                 RSET rsets[2];
2892
2893                 rsets[0] = rset;
2894                 rsets[1] =
2895                     rset_trunc(zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
2896                                glist[i+before].term,
2897                                strlen(glist[i+before].term), NULL, 0,
2898                                zapt->term->which,rset_nmem,
2899                                key_it_ctrl, key_it_ctrl->scope);
2900                 rset = rsmulti_or_create(rset_nmem, key_it_ctrl,
2901                                          2, key_it_ctrl->scope, rsets);
2902                 ptr[j]++;
2903             }
2904         }
2905         if (limit_set)
2906         {
2907             RSET rsets[2];
2908             rsets[0] = rset;
2909             rsets[1] = rset_dup(limit_set);
2910
2911             rset = rsmulti_and_create(rset_nmem, key_it_ctrl,
2912                                       key_it_ctrl->scope, 2, rsets);
2913         }
2914         count_set(rset, &glist[i+before].occurrences);
2915         rset_delete(rset);
2916     }
2917     if (i < after)
2918     {
2919         *num_entries -= (after-i);
2920         *is_partial = 1;
2921     }
2922
2923     /* consider terms before main term */
2924     for (i = 0; i<ord_no; i++)
2925         ptr[i] = 0;
2926
2927     for (i = 0; i<before; i++)
2928     {
2929         int j, j0 = -1;
2930         const char *mterm = NULL;
2931         const char *tst;
2932         RSET rset;
2933
2934         for (j = 0; j <ord_no; j++)
2935         {
2936             if (ptr[j] < before &&
2937                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2938                 (!mterm || strcmp (tst, mterm) > 0))
2939             {
2940                 j0 = j;
2941                     mterm = tst;
2942             }
2943         }
2944         if (j0 == -1)
2945             break;
2946
2947         scan_term_untrans (zh, stream->mem, reg_id,
2948                            &glist[before-1-i].term, mterm);
2949
2950         rset = rset_trunc
2951             (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
2952              glist[before-1-i].term, strlen(glist[before-1-i].term),
2953              NULL, 0, zapt->term->which,rset_nmem,
2954              key_it_ctrl,key_it_ctrl->scope);
2955
2956         ptr[j0]++;
2957
2958         for (j = j0+1; j<ord_no; j++)
2959         {
2960             if (ptr[j] < before &&
2961                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2962                 !strcmp (tst, mterm))
2963             {
2964                 RSET rsets[2];
2965
2966                 rsets[0] = rset;
2967                 rsets[1] = rset_trunc(
2968                     zh,
2969                     &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
2970                     glist[before-1-i].term,
2971                     strlen(glist[before-1-i].term), NULL, 0,
2972                     zapt->term->which, rset_nmem,
2973                     key_it_ctrl, key_it_ctrl->scope);
2974                 rset = rsmulti_or_create(rset_nmem, key_it_ctrl,
2975                                          2, key_it_ctrl->scope, rsets);
2976
2977                 ptr[j]++;
2978             }
2979         }
2980         if (limit_set)
2981         {
2982             RSET rsets[2];
2983             rsets[0] = rset;
2984             rsets[1] = rset_dup(limit_set);
2985
2986             rset = rsmulti_and_create(rset_nmem, key_it_ctrl,
2987                                       key_it_ctrl->scope, 2, rsets);
2988         }
2989         count_set (rset, &glist[before-1-i].occurrences);
2990         rset_delete (rset);
2991     }
2992     i = before-i;
2993     if (i)
2994     {
2995         *is_partial = 1;
2996         *position -= i;
2997         *num_entries -= i;
2998     }
2999
3000     nmem_destroy(rset_nmem);
3001     *list = glist + i;               /* list is set to first 'real' entry */
3002
3003     yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
3004             *position, *num_entries);
3005     return ZEBRA_OK;
3006 }
3007